From d5147adf7957d0953339f5e80be144fba825f323 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Fri, 16 Oct 2026 17:33:38 +0000 Subject: [PATCH 1/4] Drop outdated TRL workarounds from the text tuning guides --- docs/notebooks/full-finetune.ipynb | 8 +- docs/notebooks/qlora-gemma-4-12b.ipynb | 100 ++++----------------- docs/notebooks/qlora-gemma-4-26b-a4b.ipynb | 100 ++++----------------- docs/notebooks/qlora-gemma-4-31b.ipynb | 100 ++++----------------- docs/notebooks/qlora-gemma-4-e2b.ipynb | 100 ++++----------------- docs/notebooks/qlora-gemma-4-e4b.ipynb | 100 ++++----------------- docs/tuning/full-finetune.md | 8 +- docs/tuning/qlora.md | 98 +++----------------- 8 files changed, 93 insertions(+), 521 deletions(-) diff --git a/docs/notebooks/full-finetune.ipynb b/docs/notebooks/full-finetune.ipynb index 3eb2f4b..804bd4b 100644 --- a/docs/notebooks/full-finetune.ipynb +++ b/docs/notebooks/full-finetune.ipynb @@ -206,7 +206,7 @@ "# Load model and tokenizer\n", "model = AutoModelForCausalLM.from_pretrained(\n", " base_model,\n", - " torch_dtype=\"auto\",\n", + " dtype=\"auto\",\n", " device_map=\"auto\",\n", " attn_implementation=\"eager\"\n", ")\n", @@ -354,10 +354,6 @@ " lr_scheduler_type=\"constant\", # use constant learning rate scheduler\n", " push_to_hub=True, # push model to hub\n", " report_to=\"tensorboard\", # report metrics to tensorboard\n", - " dataset_kwargs={\n", - " \"add_special_tokens\": False, # Template with special tokens\n", - " \"append_concat_token\": True, # Add EOS token as separator token between examples\n", - " }\n", ")\n" ] }, @@ -484,7 +480,7 @@ "# Load Model\n", "model = AutoModelForCausalLM.from_pretrained(\n", " model_id,\n", - " torch_dtype=\"auto\",\n", + " dtype=\"auto\",\n", " device_map=\"auto\",\n", " attn_implementation=\"eager\"\n", ")\n", diff --git a/docs/notebooks/qlora-gemma-4-12b.ipynb b/docs/notebooks/qlora-gemma-4-12b.ipynb index eae7108..182fb6f 100644 --- a/docs/notebooks/qlora-gemma-4-12b.ipynb +++ b/docs/notebooks/qlora-gemma-4-12b.ipynb @@ -124,12 +124,12 @@ "\n", "This guide uses an already existing dataset ([philschmid/gretel-synthetic-text-to-sql](https://huggingface.co/datasets/philschmid/gretel-synthetic-text-to-sql)), a high quality synthetic Text-to-SQL dataset including natural language instructions, schema definitions, reasoning and the corresponding SQL query.\n", "\n", - "[Hugging Face TRL](https://huggingface.co/docs/trl/en/index) supports automatic templating of conversation dataset formats. This means you only need to convert your dataset into the right json objects, and `trl` takes care of templating and putting it into the right format.\n", + "[Hugging Face TRL](https://huggingface.co/docs/trl/en/index) supports automatic templating of conversation dataset formats. This means you only need to convert your dataset into the right json objects, and `trl` takes care of templating and putting it into the right format. With the prompt-completion format below, `trl` also computes the loss on the completion only, so the model learns to write the SQL query rather than to repeat the prompt.\n", "\n", "```text\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"You are...\"}, {\"role\": \"user\", \"content\": \"...\"}, {\"role\": \"assistant\", \"content\": \"...\"}]}\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"You are...\"}, {\"role\": \"user\", \"content\": \"...\"}, {\"role\": \"assistant\", \"content\": \"...\"}]}\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"You are...\"}, {\"role\": \"user\", \"content\": \"...\"}, {\"role\": \"assistant\", \"content\": \"...\"}]}\n", + "{\"prompt\": [{\"role\": \"system\", \"content\": \"You are...\"}, {\"role\": \"user\", \"content\": \"...\"}], \"completion\": [{\"role\": \"assistant\", \"content\": \"...\"}]}\n", + "{\"prompt\": [{\"role\": \"system\", \"content\": \"You are...\"}, {\"role\": \"user\", \"content\": \"...\"}], \"completion\": [{\"role\": \"assistant\", \"content\": \"...\"}]}\n", + "{\"prompt\": [{\"role\": \"system\", \"content\": \"You are...\"}, {\"role\": \"user\", \"content\": \"...\"}], \"completion\": [{\"role\": \"assistant\", \"content\": \"...\"}]}\n", "```\n", "\n", "The [philschmid/gretel-synthetic-text-to-sql](https://huggingface.co/datasets/philschmid/gretel-synthetic-text-to-sql) contains over 100k samples. To keep the guide small, it is downsampled to only use 10,000 samples.\n", @@ -163,11 +163,13 @@ "\"\"\"\n", "def create_conversation(sample, idx):\n", " return {\n", - " \"messages\": [\n", + " \"prompt\": [\n", " {\"role\": \"system\", \"content\": system_message},\n", " {\"role\": \"user\", \"content\": user_prompt.format(question=sample[\"sql_prompt\"], context=sample[\"sql_context\"])},\n", + " ],\n", + " \"completion\": [\n", " {\"role\": \"assistant\", \"content\": sample[\"sql\"]}\n", - " ]\n", + " ],\n", " }\n", "\n", "# Load dataset from the hub\n", @@ -180,7 +182,7 @@ "dataset = dataset.train_test_split(test_size=0.2, shuffle=False)\n", "\n", "# Print formatted user prompt\n", - "for item in dataset[\"train\"][0][\"messages\"]:\n", + "for item in dataset[\"train\"][0][\"prompt\"] + dataset[\"train\"][0][\"completion\"]:\n", " print(item)\n" ] }, @@ -310,78 +312,7 @@ " lr_scheduler_type=\"constant\", # use constant learning rate scheduler\n", " push_to_hub=True, # push model to hub\n", " report_to=\"tensorboard\", # report metrics to tensorboard\n", - " dataset_kwargs={\"skip_prepare_dataset\": True}, # important for collator\n", - " remove_unused_columns = False, # important for collator\n", - ")\n", - "\n", - "# Data collator\n", - "def collate_fn(examples):\n", - " texts = []\n", - "\n", - " for example in examples:\n", - " full_text = processor.apply_chat_template(\n", - " example[\"messages\"], add_generation_prompt=False, tokenize=False\n", - " )\n", - " texts.append(full_text.strip())\n", - "\n", - " # Tokenize the texts and process the audios\n", - " batch = processor(text=texts, return_tensors=\"pt\", padding=True)\n", - "\n", - " # The labels are the input_ids, and we mask the padding tokens and audio tokens in the loss computation\n", - " labels = batch[\"input_ids\"].clone()\n", - "\n", - " target_tokens = [\n", - " processor.tokenizer.convert_tokens_to_ids(\"<|turn>\"),\n", - " processor.tokenizer.convert_tokens_to_ids(\"model\"),\n", - " processor.tokenizer.convert_tokens_to_ids(\"\\n\")\n", - " ]\n", - " target_len = len(target_tokens)\n", - "\n", - " for i in range(labels.size(0)):\n", - " row_tokens = batch[\"input_ids\"][i].tolist()\n", - "\n", - " # Find where the assistant block begins\n", - " assistant_start_idx = None\n", - " for idx in range(len(row_tokens) - target_len + 1):\n", - " if row_tokens[idx : idx + target_len] == target_tokens:\n", - " # We want to keep loss calculation on the assistant transcription tokens,\n", - " # so we move the index right past the assistant header ('<|turn>\\nmodel\\n')\n", - " assistant_start_idx = idx + target_len\n", - " break\n", - "\n", - " if assistant_start_idx is not None:\n", - " # Mask everything from index 0 up to the start of the actual Japanese text response\n", - " labels[i, :assistant_start_idx] = -100\n", - " else:\n", - " # Fallback safety: if template matching fails for an anomalous row, mask padding anyway\n", - " print(\"WARNING: maybe the sample is too long, try to increase `token_limit` value.\")\n", - " labels[i, labels[i] == processor.tokenizer.pad_token_id] = -100\n", - "\n", - " \"\"\"\n", - " # --- DEBUG PRINT CODE ---\n", - " print(f\"\\n--- Example {i} (Split index: {assistant_start_idx}) ---\")\n", - " debug_string = []\n", - " for token_id, label_id in zip(row_tokens, labels[i].tolist()):\n", - " # Decode token by token so we can see exactly what is masked\n", - " decoded_token = processor.tokenizer.decode([token_id])\n", - "\n", - " if label_id == -100:\n", - " # Red text for masked tokens (ANSI Escape Code)\n", - " debug_string.append(f\"\\033[91m{decoded_token}\\033[0m\")\n", - " else:\n", - " # Green text for active loss tokens\n", - " debug_string.append(f\"\\033[92m{decoded_token}\\033[0m\")\n", - "\n", - " print(\"\".join(debug_string))\n", - " # ---\n", - " \"\"\"\n", - "\n", - "\n", - " # Mask tokens for not being used in the loss computation\n", - " labels[labels == processor.tokenizer.pad_token_id] = -100\n", - "\n", - " batch[\"labels\"] = labels\n", - " return batch\n" + ")\n" ] }, { @@ -408,8 +339,7 @@ " train_dataset=dataset[\"train\"],\n", " eval_dataset=dataset[\"test\"],\n", " peft_config=peft_config,\n", - " processing_class=processor,\n", - " data_collator=collate_fn,\n", + " processing_class=processor.tokenizer, # the data is text only\n", ")\n" ] }, @@ -551,16 +481,16 @@ "test_sample = dataset[\"test\"][rand_idx]\n", "\n", "# Convert as test example into a prompt with the Gemma template\n", - "prompt = processor.tokenizer.apply_chat_template(test_sample[\"messages\"][:2], tokenize=False, add_generation_prompt=True)\n", + "prompt = processor.tokenizer.apply_chat_template(test_sample[\"prompt\"], tokenize=False, add_generation_prompt=True)\n", "print(prompt)\n", "\n", "# Generate our SQL query.\n", "outputs = pipe(text_inputs=prompt, generation_config=config)\n", "\n", "# Extract the user query and original answer\n", - "print(f\"Context:\\n\", re.search(r'\\n(.*?)\\n', test_sample['messages'][1]['content'], re.DOTALL).group(1).strip())\n", - "print(f\"Query:\\n\", re.search(r'\\n(.*?)\\n', test_sample['messages'][1]['content'], re.DOTALL).group(1).strip())\n", - "print(f\"Original Answer:\\n{test_sample['messages'][2]['content']}\")\n", + "print(f\"Context:\\n\", re.search(r'\\n(.*?)\\n', test_sample['prompt'][1]['content'], re.DOTALL).group(1).strip())\n", + "print(f\"Query:\\n\", re.search(r'\\n(.*?)\\n', test_sample['prompt'][1]['content'], re.DOTALL).group(1).strip())\n", + "print(f\"Original Answer:\\n{test_sample['completion'][0]['content']}\")\n", "print(f\"Generated Answer:\\n{outputs[0]['generated_text'][len(prompt):].strip().removesuffix(\"\")}\")\n" ] }, diff --git a/docs/notebooks/qlora-gemma-4-26b-a4b.ipynb b/docs/notebooks/qlora-gemma-4-26b-a4b.ipynb index a386f1d..fc1fd1a 100644 --- a/docs/notebooks/qlora-gemma-4-26b-a4b.ipynb +++ b/docs/notebooks/qlora-gemma-4-26b-a4b.ipynb @@ -124,12 +124,12 @@ "\n", "This guide uses an already existing dataset ([philschmid/gretel-synthetic-text-to-sql](https://huggingface.co/datasets/philschmid/gretel-synthetic-text-to-sql)), a high quality synthetic Text-to-SQL dataset including natural language instructions, schema definitions, reasoning and the corresponding SQL query.\n", "\n", - "[Hugging Face TRL](https://huggingface.co/docs/trl/en/index) supports automatic templating of conversation dataset formats. This means you only need to convert your dataset into the right json objects, and `trl` takes care of templating and putting it into the right format.\n", + "[Hugging Face TRL](https://huggingface.co/docs/trl/en/index) supports automatic templating of conversation dataset formats. This means you only need to convert your dataset into the right json objects, and `trl` takes care of templating and putting it into the right format. With the prompt-completion format below, `trl` also computes the loss on the completion only, so the model learns to write the SQL query rather than to repeat the prompt.\n", "\n", "```text\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"You are...\"}, {\"role\": \"user\", \"content\": \"...\"}, {\"role\": \"assistant\", \"content\": \"...\"}]}\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"You are...\"}, {\"role\": \"user\", \"content\": \"...\"}, {\"role\": \"assistant\", \"content\": \"...\"}]}\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"You are...\"}, {\"role\": \"user\", \"content\": \"...\"}, {\"role\": \"assistant\", \"content\": \"...\"}]}\n", + "{\"prompt\": [{\"role\": \"system\", \"content\": \"You are...\"}, {\"role\": \"user\", \"content\": \"...\"}], \"completion\": [{\"role\": \"assistant\", \"content\": \"...\"}]}\n", + "{\"prompt\": [{\"role\": \"system\", \"content\": \"You are...\"}, {\"role\": \"user\", \"content\": \"...\"}], \"completion\": [{\"role\": \"assistant\", \"content\": \"...\"}]}\n", + "{\"prompt\": [{\"role\": \"system\", \"content\": \"You are...\"}, {\"role\": \"user\", \"content\": \"...\"}], \"completion\": [{\"role\": \"assistant\", \"content\": \"...\"}]}\n", "```\n", "\n", "The [philschmid/gretel-synthetic-text-to-sql](https://huggingface.co/datasets/philschmid/gretel-synthetic-text-to-sql) contains over 100k samples. To keep the guide small, it is downsampled to only use 10,000 samples.\n", @@ -163,11 +163,13 @@ "\"\"\"\n", "def create_conversation(sample, idx):\n", " return {\n", - " \"messages\": [\n", + " \"prompt\": [\n", " {\"role\": \"system\", \"content\": system_message},\n", " {\"role\": \"user\", \"content\": user_prompt.format(question=sample[\"sql_prompt\"], context=sample[\"sql_context\"])},\n", + " ],\n", + " \"completion\": [\n", " {\"role\": \"assistant\", \"content\": sample[\"sql\"]}\n", - " ]\n", + " ],\n", " }\n", "\n", "# Load dataset from the hub\n", @@ -180,7 +182,7 @@ "dataset = dataset.train_test_split(test_size=0.2, shuffle=False)\n", "\n", "# Print formatted user prompt\n", - "for item in dataset[\"train\"][0][\"messages\"]:\n", + "for item in dataset[\"train\"][0][\"prompt\"] + dataset[\"train\"][0][\"completion\"]:\n", " print(item)\n" ] }, @@ -310,78 +312,7 @@ " lr_scheduler_type=\"constant\", # use constant learning rate scheduler\n", " push_to_hub=True, # push model to hub\n", " report_to=\"tensorboard\", # report metrics to tensorboard\n", - " dataset_kwargs={\"skip_prepare_dataset\": True}, # important for collator\n", - " remove_unused_columns = False, # important for collator\n", - ")\n", - "\n", - "# Data collator\n", - "def collate_fn(examples):\n", - " texts = []\n", - "\n", - " for example in examples:\n", - " full_text = processor.apply_chat_template(\n", - " example[\"messages\"], add_generation_prompt=False, tokenize=False\n", - " )\n", - " texts.append(full_text.strip())\n", - "\n", - " # Tokenize the texts and process the audios\n", - " batch = processor(text=texts, return_tensors=\"pt\", padding=True)\n", - "\n", - " # The labels are the input_ids, and we mask the padding tokens and audio tokens in the loss computation\n", - " labels = batch[\"input_ids\"].clone()\n", - "\n", - " target_tokens = [\n", - " processor.tokenizer.convert_tokens_to_ids(\"<|turn>\"),\n", - " processor.tokenizer.convert_tokens_to_ids(\"model\"),\n", - " processor.tokenizer.convert_tokens_to_ids(\"\\n\")\n", - " ]\n", - " target_len = len(target_tokens)\n", - "\n", - " for i in range(labels.size(0)):\n", - " row_tokens = batch[\"input_ids\"][i].tolist()\n", - "\n", - " # Find where the assistant block begins\n", - " assistant_start_idx = None\n", - " for idx in range(len(row_tokens) - target_len + 1):\n", - " if row_tokens[idx : idx + target_len] == target_tokens:\n", - " # We want to keep loss calculation on the assistant transcription tokens,\n", - " # so we move the index right past the assistant header ('<|turn>\\nmodel\\n')\n", - " assistant_start_idx = idx + target_len\n", - " break\n", - "\n", - " if assistant_start_idx is not None:\n", - " # Mask everything from index 0 up to the start of the actual Japanese text response\n", - " labels[i, :assistant_start_idx] = -100\n", - " else:\n", - " # Fallback safety: if template matching fails for an anomalous row, mask padding anyway\n", - " print(\"WARNING: maybe the sample is too long, try to increase `token_limit` value.\")\n", - " labels[i, labels[i] == processor.tokenizer.pad_token_id] = -100\n", - "\n", - " \"\"\"\n", - " # --- DEBUG PRINT CODE ---\n", - " print(f\"\\n--- Example {i} (Split index: {assistant_start_idx}) ---\")\n", - " debug_string = []\n", - " for token_id, label_id in zip(row_tokens, labels[i].tolist()):\n", - " # Decode token by token so we can see exactly what is masked\n", - " decoded_token = processor.tokenizer.decode([token_id])\n", - "\n", - " if label_id == -100:\n", - " # Red text for masked tokens (ANSI Escape Code)\n", - " debug_string.append(f\"\\033[91m{decoded_token}\\033[0m\")\n", - " else:\n", - " # Green text for active loss tokens\n", - " debug_string.append(f\"\\033[92m{decoded_token}\\033[0m\")\n", - "\n", - " print(\"\".join(debug_string))\n", - " # ---\n", - " \"\"\"\n", - "\n", - "\n", - " # Mask tokens for not being used in the loss computation\n", - " labels[labels == processor.tokenizer.pad_token_id] = -100\n", - "\n", - " batch[\"labels\"] = labels\n", - " return batch\n" + ")\n" ] }, { @@ -408,8 +339,7 @@ " train_dataset=dataset[\"train\"],\n", " eval_dataset=dataset[\"test\"],\n", " peft_config=peft_config,\n", - " processing_class=processor,\n", - " data_collator=collate_fn,\n", + " processing_class=processor.tokenizer, # the data is text only\n", ")\n" ] }, @@ -551,16 +481,16 @@ "test_sample = dataset[\"test\"][rand_idx]\n", "\n", "# Convert as test example into a prompt with the Gemma template\n", - "prompt = processor.tokenizer.apply_chat_template(test_sample[\"messages\"][:2], tokenize=False, add_generation_prompt=True)\n", + "prompt = processor.tokenizer.apply_chat_template(test_sample[\"prompt\"], tokenize=False, add_generation_prompt=True)\n", "print(prompt)\n", "\n", "# Generate our SQL query.\n", "outputs = pipe(text_inputs=prompt, generation_config=config)\n", "\n", "# Extract the user query and original answer\n", - "print(f\"Context:\\n\", re.search(r'\\n(.*?)\\n', test_sample['messages'][1]['content'], re.DOTALL).group(1).strip())\n", - "print(f\"Query:\\n\", re.search(r'\\n(.*?)\\n', test_sample['messages'][1]['content'], re.DOTALL).group(1).strip())\n", - "print(f\"Original Answer:\\n{test_sample['messages'][2]['content']}\")\n", + "print(f\"Context:\\n\", re.search(r'\\n(.*?)\\n', test_sample['prompt'][1]['content'], re.DOTALL).group(1).strip())\n", + "print(f\"Query:\\n\", re.search(r'\\n(.*?)\\n', test_sample['prompt'][1]['content'], re.DOTALL).group(1).strip())\n", + "print(f\"Original Answer:\\n{test_sample['completion'][0]['content']}\")\n", "print(f\"Generated Answer:\\n{outputs[0]['generated_text'][len(prompt):].strip().removesuffix(\"\")}\")\n" ] }, diff --git a/docs/notebooks/qlora-gemma-4-31b.ipynb b/docs/notebooks/qlora-gemma-4-31b.ipynb index 25e2f40..814ba25 100644 --- a/docs/notebooks/qlora-gemma-4-31b.ipynb +++ b/docs/notebooks/qlora-gemma-4-31b.ipynb @@ -124,12 +124,12 @@ "\n", "This guide uses an already existing dataset ([philschmid/gretel-synthetic-text-to-sql](https://huggingface.co/datasets/philschmid/gretel-synthetic-text-to-sql)), a high quality synthetic Text-to-SQL dataset including natural language instructions, schema definitions, reasoning and the corresponding SQL query.\n", "\n", - "[Hugging Face TRL](https://huggingface.co/docs/trl/en/index) supports automatic templating of conversation dataset formats. This means you only need to convert your dataset into the right json objects, and `trl` takes care of templating and putting it into the right format.\n", + "[Hugging Face TRL](https://huggingface.co/docs/trl/en/index) supports automatic templating of conversation dataset formats. This means you only need to convert your dataset into the right json objects, and `trl` takes care of templating and putting it into the right format. With the prompt-completion format below, `trl` also computes the loss on the completion only, so the model learns to write the SQL query rather than to repeat the prompt.\n", "\n", "```text\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"You are...\"}, {\"role\": \"user\", \"content\": \"...\"}, {\"role\": \"assistant\", \"content\": \"...\"}]}\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"You are...\"}, {\"role\": \"user\", \"content\": \"...\"}, {\"role\": \"assistant\", \"content\": \"...\"}]}\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"You are...\"}, {\"role\": \"user\", \"content\": \"...\"}, {\"role\": \"assistant\", \"content\": \"...\"}]}\n", + "{\"prompt\": [{\"role\": \"system\", \"content\": \"You are...\"}, {\"role\": \"user\", \"content\": \"...\"}], \"completion\": [{\"role\": \"assistant\", \"content\": \"...\"}]}\n", + "{\"prompt\": [{\"role\": \"system\", \"content\": \"You are...\"}, {\"role\": \"user\", \"content\": \"...\"}], \"completion\": [{\"role\": \"assistant\", \"content\": \"...\"}]}\n", + "{\"prompt\": [{\"role\": \"system\", \"content\": \"You are...\"}, {\"role\": \"user\", \"content\": \"...\"}], \"completion\": [{\"role\": \"assistant\", \"content\": \"...\"}]}\n", "```\n", "\n", "The [philschmid/gretel-synthetic-text-to-sql](https://huggingface.co/datasets/philschmid/gretel-synthetic-text-to-sql) contains over 100k samples. To keep the guide small, it is downsampled to only use 10,000 samples.\n", @@ -163,11 +163,13 @@ "\"\"\"\n", "def create_conversation(sample, idx):\n", " return {\n", - " \"messages\": [\n", + " \"prompt\": [\n", " {\"role\": \"system\", \"content\": system_message},\n", " {\"role\": \"user\", \"content\": user_prompt.format(question=sample[\"sql_prompt\"], context=sample[\"sql_context\"])},\n", + " ],\n", + " \"completion\": [\n", " {\"role\": \"assistant\", \"content\": sample[\"sql\"]}\n", - " ]\n", + " ],\n", " }\n", "\n", "# Load dataset from the hub\n", @@ -180,7 +182,7 @@ "dataset = dataset.train_test_split(test_size=0.2, shuffle=False)\n", "\n", "# Print formatted user prompt\n", - "for item in dataset[\"train\"][0][\"messages\"]:\n", + "for item in dataset[\"train\"][0][\"prompt\"] + dataset[\"train\"][0][\"completion\"]:\n", " print(item)\n" ] }, @@ -310,78 +312,7 @@ " lr_scheduler_type=\"constant\", # use constant learning rate scheduler\n", " push_to_hub=True, # push model to hub\n", " report_to=\"tensorboard\", # report metrics to tensorboard\n", - " dataset_kwargs={\"skip_prepare_dataset\": True}, # important for collator\n", - " remove_unused_columns = False, # important for collator\n", - ")\n", - "\n", - "# Data collator\n", - "def collate_fn(examples):\n", - " texts = []\n", - "\n", - " for example in examples:\n", - " full_text = processor.apply_chat_template(\n", - " example[\"messages\"], add_generation_prompt=False, tokenize=False\n", - " )\n", - " texts.append(full_text.strip())\n", - "\n", - " # Tokenize the texts and process the audios\n", - " batch = processor(text=texts, return_tensors=\"pt\", padding=True)\n", - "\n", - " # The labels are the input_ids, and we mask the padding tokens and audio tokens in the loss computation\n", - " labels = batch[\"input_ids\"].clone()\n", - "\n", - " target_tokens = [\n", - " processor.tokenizer.convert_tokens_to_ids(\"<|turn>\"),\n", - " processor.tokenizer.convert_tokens_to_ids(\"model\"),\n", - " processor.tokenizer.convert_tokens_to_ids(\"\\n\")\n", - " ]\n", - " target_len = len(target_tokens)\n", - "\n", - " for i in range(labels.size(0)):\n", - " row_tokens = batch[\"input_ids\"][i].tolist()\n", - "\n", - " # Find where the assistant block begins\n", - " assistant_start_idx = None\n", - " for idx in range(len(row_tokens) - target_len + 1):\n", - " if row_tokens[idx : idx + target_len] == target_tokens:\n", - " # We want to keep loss calculation on the assistant transcription tokens,\n", - " # so we move the index right past the assistant header ('<|turn>\\nmodel\\n')\n", - " assistant_start_idx = idx + target_len\n", - " break\n", - "\n", - " if assistant_start_idx is not None:\n", - " # Mask everything from index 0 up to the start of the actual Japanese text response\n", - " labels[i, :assistant_start_idx] = -100\n", - " else:\n", - " # Fallback safety: if template matching fails for an anomalous row, mask padding anyway\n", - " print(\"WARNING: maybe the sample is too long, try to increase `token_limit` value.\")\n", - " labels[i, labels[i] == processor.tokenizer.pad_token_id] = -100\n", - "\n", - " \"\"\"\n", - " # --- DEBUG PRINT CODE ---\n", - " print(f\"\\n--- Example {i} (Split index: {assistant_start_idx}) ---\")\n", - " debug_string = []\n", - " for token_id, label_id in zip(row_tokens, labels[i].tolist()):\n", - " # Decode token by token so we can see exactly what is masked\n", - " decoded_token = processor.tokenizer.decode([token_id])\n", - "\n", - " if label_id == -100:\n", - " # Red text for masked tokens (ANSI Escape Code)\n", - " debug_string.append(f\"\\033[91m{decoded_token}\\033[0m\")\n", - " else:\n", - " # Green text for active loss tokens\n", - " debug_string.append(f\"\\033[92m{decoded_token}\\033[0m\")\n", - "\n", - " print(\"\".join(debug_string))\n", - " # ---\n", - " \"\"\"\n", - "\n", - "\n", - " # Mask tokens for not being used in the loss computation\n", - " labels[labels == processor.tokenizer.pad_token_id] = -100\n", - "\n", - " batch[\"labels\"] = labels\n", - " return batch\n" + ")\n" ] }, { @@ -408,8 +339,7 @@ " train_dataset=dataset[\"train\"],\n", " eval_dataset=dataset[\"test\"],\n", " peft_config=peft_config,\n", - " processing_class=processor,\n", - " data_collator=collate_fn,\n", + " processing_class=processor.tokenizer, # the data is text only\n", ")\n" ] }, @@ -551,16 +481,16 @@ "test_sample = dataset[\"test\"][rand_idx]\n", "\n", "# Convert as test example into a prompt with the Gemma template\n", - "prompt = processor.tokenizer.apply_chat_template(test_sample[\"messages\"][:2], tokenize=False, add_generation_prompt=True)\n", + "prompt = processor.tokenizer.apply_chat_template(test_sample[\"prompt\"], tokenize=False, add_generation_prompt=True)\n", "print(prompt)\n", "\n", "# Generate our SQL query.\n", "outputs = pipe(text_inputs=prompt, generation_config=config)\n", "\n", "# Extract the user query and original answer\n", - "print(f\"Context:\\n\", re.search(r'\\n(.*?)\\n', test_sample['messages'][1]['content'], re.DOTALL).group(1).strip())\n", - "print(f\"Query:\\n\", re.search(r'\\n(.*?)\\n', test_sample['messages'][1]['content'], re.DOTALL).group(1).strip())\n", - "print(f\"Original Answer:\\n{test_sample['messages'][2]['content']}\")\n", + "print(f\"Context:\\n\", re.search(r'\\n(.*?)\\n', test_sample['prompt'][1]['content'], re.DOTALL).group(1).strip())\n", + "print(f\"Query:\\n\", re.search(r'\\n(.*?)\\n', test_sample['prompt'][1]['content'], re.DOTALL).group(1).strip())\n", + "print(f\"Original Answer:\\n{test_sample['completion'][0]['content']}\")\n", "print(f\"Generated Answer:\\n{outputs[0]['generated_text'][len(prompt):].strip().removesuffix(\"\")}\")\n" ] }, diff --git a/docs/notebooks/qlora-gemma-4-e2b.ipynb b/docs/notebooks/qlora-gemma-4-e2b.ipynb index 0512bd4..f490ead 100644 --- a/docs/notebooks/qlora-gemma-4-e2b.ipynb +++ b/docs/notebooks/qlora-gemma-4-e2b.ipynb @@ -124,12 +124,12 @@ "\n", "This guide uses an already existing dataset ([philschmid/gretel-synthetic-text-to-sql](https://huggingface.co/datasets/philschmid/gretel-synthetic-text-to-sql)), a high quality synthetic Text-to-SQL dataset including natural language instructions, schema definitions, reasoning and the corresponding SQL query.\n", "\n", - "[Hugging Face TRL](https://huggingface.co/docs/trl/en/index) supports automatic templating of conversation dataset formats. This means you only need to convert your dataset into the right json objects, and `trl` takes care of templating and putting it into the right format.\n", + "[Hugging Face TRL](https://huggingface.co/docs/trl/en/index) supports automatic templating of conversation dataset formats. This means you only need to convert your dataset into the right json objects, and `trl` takes care of templating and putting it into the right format. With the prompt-completion format below, `trl` also computes the loss on the completion only, so the model learns to write the SQL query rather than to repeat the prompt.\n", "\n", "```text\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"You are...\"}, {\"role\": \"user\", \"content\": \"...\"}, {\"role\": \"assistant\", \"content\": \"...\"}]}\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"You are...\"}, {\"role\": \"user\", \"content\": \"...\"}, {\"role\": \"assistant\", \"content\": \"...\"}]}\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"You are...\"}, {\"role\": \"user\", \"content\": \"...\"}, {\"role\": \"assistant\", \"content\": \"...\"}]}\n", + "{\"prompt\": [{\"role\": \"system\", \"content\": \"You are...\"}, {\"role\": \"user\", \"content\": \"...\"}], \"completion\": [{\"role\": \"assistant\", \"content\": \"...\"}]}\n", + "{\"prompt\": [{\"role\": \"system\", \"content\": \"You are...\"}, {\"role\": \"user\", \"content\": \"...\"}], \"completion\": [{\"role\": \"assistant\", \"content\": \"...\"}]}\n", + "{\"prompt\": [{\"role\": \"system\", \"content\": \"You are...\"}, {\"role\": \"user\", \"content\": \"...\"}], \"completion\": [{\"role\": \"assistant\", \"content\": \"...\"}]}\n", "```\n", "\n", "The [philschmid/gretel-synthetic-text-to-sql](https://huggingface.co/datasets/philschmid/gretel-synthetic-text-to-sql) contains over 100k samples. To keep the guide small, it is downsampled to only use 10,000 samples.\n", @@ -163,11 +163,13 @@ "\"\"\"\n", "def create_conversation(sample, idx):\n", " return {\n", - " \"messages\": [\n", + " \"prompt\": [\n", " {\"role\": \"system\", \"content\": system_message},\n", " {\"role\": \"user\", \"content\": user_prompt.format(question=sample[\"sql_prompt\"], context=sample[\"sql_context\"])},\n", + " ],\n", + " \"completion\": [\n", " {\"role\": \"assistant\", \"content\": sample[\"sql\"]}\n", - " ]\n", + " ],\n", " }\n", "\n", "# Load dataset from the hub\n", @@ -180,7 +182,7 @@ "dataset = dataset.train_test_split(test_size=0.2, shuffle=False)\n", "\n", "# Print formatted user prompt\n", - "for item in dataset[\"train\"][0][\"messages\"]:\n", + "for item in dataset[\"train\"][0][\"prompt\"] + dataset[\"train\"][0][\"completion\"]:\n", " print(item)\n" ] }, @@ -310,78 +312,7 @@ " lr_scheduler_type=\"constant\", # use constant learning rate scheduler\n", " push_to_hub=True, # push model to hub\n", " report_to=\"tensorboard\", # report metrics to tensorboard\n", - " dataset_kwargs={\"skip_prepare_dataset\": True}, # important for collator\n", - " remove_unused_columns = False, # important for collator\n", - ")\n", - "\n", - "# Data collator\n", - "def collate_fn(examples):\n", - " texts = []\n", - "\n", - " for example in examples:\n", - " full_text = processor.apply_chat_template(\n", - " example[\"messages\"], add_generation_prompt=False, tokenize=False\n", - " )\n", - " texts.append(full_text.strip())\n", - "\n", - " # Tokenize the texts and process the audios\n", - " batch = processor(text=texts, return_tensors=\"pt\", padding=True)\n", - "\n", - " # The labels are the input_ids, and we mask the padding tokens and audio tokens in the loss computation\n", - " labels = batch[\"input_ids\"].clone()\n", - "\n", - " target_tokens = [\n", - " processor.tokenizer.convert_tokens_to_ids(\"<|turn>\"),\n", - " processor.tokenizer.convert_tokens_to_ids(\"model\"),\n", - " processor.tokenizer.convert_tokens_to_ids(\"\\n\")\n", - " ]\n", - " target_len = len(target_tokens)\n", - "\n", - " for i in range(labels.size(0)):\n", - " row_tokens = batch[\"input_ids\"][i].tolist()\n", - "\n", - " # Find where the assistant block begins\n", - " assistant_start_idx = None\n", - " for idx in range(len(row_tokens) - target_len + 1):\n", - " if row_tokens[idx : idx + target_len] == target_tokens:\n", - " # We want to keep loss calculation on the assistant transcription tokens,\n", - " # so we move the index right past the assistant header ('<|turn>\\nmodel\\n')\n", - " assistant_start_idx = idx + target_len\n", - " break\n", - "\n", - " if assistant_start_idx is not None:\n", - " # Mask everything from index 0 up to the start of the actual Japanese text response\n", - " labels[i, :assistant_start_idx] = -100\n", - " else:\n", - " # Fallback safety: if template matching fails for an anomalous row, mask padding anyway\n", - " print(\"WARNING: maybe the sample is too long, try to increase `token_limit` value.\")\n", - " labels[i, labels[i] == processor.tokenizer.pad_token_id] = -100\n", - "\n", - " \"\"\"\n", - " # --- DEBUG PRINT CODE ---\n", - " print(f\"\\n--- Example {i} (Split index: {assistant_start_idx}) ---\")\n", - " debug_string = []\n", - " for token_id, label_id in zip(row_tokens, labels[i].tolist()):\n", - " # Decode token by token so we can see exactly what is masked\n", - " decoded_token = processor.tokenizer.decode([token_id])\n", - "\n", - " if label_id == -100:\n", - " # Red text for masked tokens (ANSI Escape Code)\n", - " debug_string.append(f\"\\033[91m{decoded_token}\\033[0m\")\n", - " else:\n", - " # Green text for active loss tokens\n", - " debug_string.append(f\"\\033[92m{decoded_token}\\033[0m\")\n", - "\n", - " print(\"\".join(debug_string))\n", - " # ---\n", - " \"\"\"\n", - "\n", - "\n", - " # Mask tokens for not being used in the loss computation\n", - " labels[labels == processor.tokenizer.pad_token_id] = -100\n", - "\n", - " batch[\"labels\"] = labels\n", - " return batch\n" + ")\n" ] }, { @@ -408,8 +339,7 @@ " train_dataset=dataset[\"train\"],\n", " eval_dataset=dataset[\"test\"],\n", " peft_config=peft_config,\n", - " processing_class=processor,\n", - " data_collator=collate_fn,\n", + " processing_class=processor.tokenizer, # the data is text only\n", ")\n" ] }, @@ -551,16 +481,16 @@ "test_sample = dataset[\"test\"][rand_idx]\n", "\n", "# Convert as test example into a prompt with the Gemma template\n", - "prompt = processor.tokenizer.apply_chat_template(test_sample[\"messages\"][:2], tokenize=False, add_generation_prompt=True)\n", + "prompt = processor.tokenizer.apply_chat_template(test_sample[\"prompt\"], tokenize=False, add_generation_prompt=True)\n", "print(prompt)\n", "\n", "# Generate our SQL query.\n", "outputs = pipe(text_inputs=prompt, generation_config=config)\n", "\n", "# Extract the user query and original answer\n", - "print(f\"Context:\\n\", re.search(r'\\n(.*?)\\n', test_sample['messages'][1]['content'], re.DOTALL).group(1).strip())\n", - "print(f\"Query:\\n\", re.search(r'\\n(.*?)\\n', test_sample['messages'][1]['content'], re.DOTALL).group(1).strip())\n", - "print(f\"Original Answer:\\n{test_sample['messages'][2]['content']}\")\n", + "print(f\"Context:\\n\", re.search(r'\\n(.*?)\\n', test_sample['prompt'][1]['content'], re.DOTALL).group(1).strip())\n", + "print(f\"Query:\\n\", re.search(r'\\n(.*?)\\n', test_sample['prompt'][1]['content'], re.DOTALL).group(1).strip())\n", + "print(f\"Original Answer:\\n{test_sample['completion'][0]['content']}\")\n", "print(f\"Generated Answer:\\n{outputs[0]['generated_text'][len(prompt):].strip().removesuffix(\"\")}\")\n" ] }, diff --git a/docs/notebooks/qlora-gemma-4-e4b.ipynb b/docs/notebooks/qlora-gemma-4-e4b.ipynb index 13088ba..317debc 100644 --- a/docs/notebooks/qlora-gemma-4-e4b.ipynb +++ b/docs/notebooks/qlora-gemma-4-e4b.ipynb @@ -124,12 +124,12 @@ "\n", "This guide uses an already existing dataset ([philschmid/gretel-synthetic-text-to-sql](https://huggingface.co/datasets/philschmid/gretel-synthetic-text-to-sql)), a high quality synthetic Text-to-SQL dataset including natural language instructions, schema definitions, reasoning and the corresponding SQL query.\n", "\n", - "[Hugging Face TRL](https://huggingface.co/docs/trl/en/index) supports automatic templating of conversation dataset formats. This means you only need to convert your dataset into the right json objects, and `trl` takes care of templating and putting it into the right format.\n", + "[Hugging Face TRL](https://huggingface.co/docs/trl/en/index) supports automatic templating of conversation dataset formats. This means you only need to convert your dataset into the right json objects, and `trl` takes care of templating and putting it into the right format. With the prompt-completion format below, `trl` also computes the loss on the completion only, so the model learns to write the SQL query rather than to repeat the prompt.\n", "\n", "```text\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"You are...\"}, {\"role\": \"user\", \"content\": \"...\"}, {\"role\": \"assistant\", \"content\": \"...\"}]}\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"You are...\"}, {\"role\": \"user\", \"content\": \"...\"}, {\"role\": \"assistant\", \"content\": \"...\"}]}\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"You are...\"}, {\"role\": \"user\", \"content\": \"...\"}, {\"role\": \"assistant\", \"content\": \"...\"}]}\n", + "{\"prompt\": [{\"role\": \"system\", \"content\": \"You are...\"}, {\"role\": \"user\", \"content\": \"...\"}], \"completion\": [{\"role\": \"assistant\", \"content\": \"...\"}]}\n", + "{\"prompt\": [{\"role\": \"system\", \"content\": \"You are...\"}, {\"role\": \"user\", \"content\": \"...\"}], \"completion\": [{\"role\": \"assistant\", \"content\": \"...\"}]}\n", + "{\"prompt\": [{\"role\": \"system\", \"content\": \"You are...\"}, {\"role\": \"user\", \"content\": \"...\"}], \"completion\": [{\"role\": \"assistant\", \"content\": \"...\"}]}\n", "```\n", "\n", "The [philschmid/gretel-synthetic-text-to-sql](https://huggingface.co/datasets/philschmid/gretel-synthetic-text-to-sql) contains over 100k samples. To keep the guide small, it is downsampled to only use 10,000 samples.\n", @@ -163,11 +163,13 @@ "\"\"\"\n", "def create_conversation(sample, idx):\n", " return {\n", - " \"messages\": [\n", + " \"prompt\": [\n", " {\"role\": \"system\", \"content\": system_message},\n", " {\"role\": \"user\", \"content\": user_prompt.format(question=sample[\"sql_prompt\"], context=sample[\"sql_context\"])},\n", + " ],\n", + " \"completion\": [\n", " {\"role\": \"assistant\", \"content\": sample[\"sql\"]}\n", - " ]\n", + " ],\n", " }\n", "\n", "# Load dataset from the hub\n", @@ -180,7 +182,7 @@ "dataset = dataset.train_test_split(test_size=0.2, shuffle=False)\n", "\n", "# Print formatted user prompt\n", - "for item in dataset[\"train\"][0][\"messages\"]:\n", + "for item in dataset[\"train\"][0][\"prompt\"] + dataset[\"train\"][0][\"completion\"]:\n", " print(item)\n" ] }, @@ -310,78 +312,7 @@ " lr_scheduler_type=\"constant\", # use constant learning rate scheduler\n", " push_to_hub=True, # push model to hub\n", " report_to=\"tensorboard\", # report metrics to tensorboard\n", - " dataset_kwargs={\"skip_prepare_dataset\": True}, # important for collator\n", - " remove_unused_columns = False, # important for collator\n", - ")\n", - "\n", - "# Data collator\n", - "def collate_fn(examples):\n", - " texts = []\n", - "\n", - " for example in examples:\n", - " full_text = processor.apply_chat_template(\n", - " example[\"messages\"], add_generation_prompt=False, tokenize=False\n", - " )\n", - " texts.append(full_text.strip())\n", - "\n", - " # Tokenize the texts and process the audios\n", - " batch = processor(text=texts, return_tensors=\"pt\", padding=True)\n", - "\n", - " # The labels are the input_ids, and we mask the padding tokens and audio tokens in the loss computation\n", - " labels = batch[\"input_ids\"].clone()\n", - "\n", - " target_tokens = [\n", - " processor.tokenizer.convert_tokens_to_ids(\"<|turn>\"),\n", - " processor.tokenizer.convert_tokens_to_ids(\"model\"),\n", - " processor.tokenizer.convert_tokens_to_ids(\"\\n\")\n", - " ]\n", - " target_len = len(target_tokens)\n", - "\n", - " for i in range(labels.size(0)):\n", - " row_tokens = batch[\"input_ids\"][i].tolist()\n", - "\n", - " # Find where the assistant block begins\n", - " assistant_start_idx = None\n", - " for idx in range(len(row_tokens) - target_len + 1):\n", - " if row_tokens[idx : idx + target_len] == target_tokens:\n", - " # We want to keep loss calculation on the assistant transcription tokens,\n", - " # so we move the index right past the assistant header ('<|turn>\\nmodel\\n')\n", - " assistant_start_idx = idx + target_len\n", - " break\n", - "\n", - " if assistant_start_idx is not None:\n", - " # Mask everything from index 0 up to the start of the actual Japanese text response\n", - " labels[i, :assistant_start_idx] = -100\n", - " else:\n", - " # Fallback safety: if template matching fails for an anomalous row, mask padding anyway\n", - " print(\"WARNING: maybe the sample is too long, try to increase `token_limit` value.\")\n", - " labels[i, labels[i] == processor.tokenizer.pad_token_id] = -100\n", - "\n", - " \"\"\"\n", - " # --- DEBUG PRINT CODE ---\n", - " print(f\"\\n--- Example {i} (Split index: {assistant_start_idx}) ---\")\n", - " debug_string = []\n", - " for token_id, label_id in zip(row_tokens, labels[i].tolist()):\n", - " # Decode token by token so we can see exactly what is masked\n", - " decoded_token = processor.tokenizer.decode([token_id])\n", - "\n", - " if label_id == -100:\n", - " # Red text for masked tokens (ANSI Escape Code)\n", - " debug_string.append(f\"\\033[91m{decoded_token}\\033[0m\")\n", - " else:\n", - " # Green text for active loss tokens\n", - " debug_string.append(f\"\\033[92m{decoded_token}\\033[0m\")\n", - "\n", - " print(\"\".join(debug_string))\n", - " # ---\n", - " \"\"\"\n", - "\n", - "\n", - " # Mask tokens for not being used in the loss computation\n", - " labels[labels == processor.tokenizer.pad_token_id] = -100\n", - "\n", - " batch[\"labels\"] = labels\n", - " return batch\n" + ")\n" ] }, { @@ -408,8 +339,7 @@ " train_dataset=dataset[\"train\"],\n", " eval_dataset=dataset[\"test\"],\n", " peft_config=peft_config,\n", - " processing_class=processor,\n", - " data_collator=collate_fn,\n", + " processing_class=processor.tokenizer, # the data is text only\n", ")\n" ] }, @@ -551,16 +481,16 @@ "test_sample = dataset[\"test\"][rand_idx]\n", "\n", "# Convert as test example into a prompt with the Gemma template\n", - "prompt = processor.tokenizer.apply_chat_template(test_sample[\"messages\"][:2], tokenize=False, add_generation_prompt=True)\n", + "prompt = processor.tokenizer.apply_chat_template(test_sample[\"prompt\"], tokenize=False, add_generation_prompt=True)\n", "print(prompt)\n", "\n", "# Generate our SQL query.\n", "outputs = pipe(text_inputs=prompt, generation_config=config)\n", "\n", "# Extract the user query and original answer\n", - "print(f\"Context:\\n\", re.search(r'\\n(.*?)\\n', test_sample['messages'][1]['content'], re.DOTALL).group(1).strip())\n", - "print(f\"Query:\\n\", re.search(r'\\n(.*?)\\n', test_sample['messages'][1]['content'], re.DOTALL).group(1).strip())\n", - "print(f\"Original Answer:\\n{test_sample['messages'][2]['content']}\")\n", + "print(f\"Context:\\n\", re.search(r'\\n(.*?)\\n', test_sample['prompt'][1]['content'], re.DOTALL).group(1).strip())\n", + "print(f\"Query:\\n\", re.search(r'\\n(.*?)\\n', test_sample['prompt'][1]['content'], re.DOTALL).group(1).strip())\n", + "print(f\"Original Answer:\\n{test_sample['completion'][0]['content']}\")\n", "print(f\"Generated Answer:\\n{outputs[0]['generated_text'][len(prompt):].strip().removesuffix(\"\")}\")\n" ] }, diff --git a/docs/tuning/full-finetune.md b/docs/tuning/full-finetune.md index d84d0ca..fa6017e 100644 --- a/docs/tuning/full-finetune.md +++ b/docs/tuning/full-finetune.md @@ -123,7 +123,7 @@ from transformers import AutoTokenizer, AutoModelForCausalLM # Load model and tokenizer model = AutoModelForCausalLM.from_pretrained( base_model, - torch_dtype="auto", + dtype="auto", device_map="auto", attn_implementation="eager" ) @@ -250,10 +250,6 @@ args = SFTConfig( lr_scheduler_type="constant", # use constant learning rate scheduler push_to_hub=True, # push model to hub report_to="tensorboard", # report metrics to tensorboard - dataset_kwargs={ - "add_special_tokens": False, # Template with special tokens - "append_concat_token": True, # Add EOS token as separator token between examples - } ) ``` @@ -353,7 +349,7 @@ model_id = checkpoint_dir # Load Model model = AutoModelForCausalLM.from_pretrained( model_id, - torch_dtype="auto", + dtype="auto", device_map="auto", attn_implementation="eager" ) diff --git a/docs/tuning/qlora.md b/docs/tuning/qlora.md index ba65630..c5e0dbf 100644 --- a/docs/tuning/qlora.md +++ b/docs/tuning/qlora.md @@ -75,12 +75,12 @@ Each of the methods has its own advantages and disadvantages and depends on the This guide uses an already existing dataset ([philschmid/gretel-synthetic-text-to-sql](https://huggingface.co/datasets/philschmid/gretel-synthetic-text-to-sql)), a high quality synthetic Text-to-SQL dataset including natural language instructions, schema definitions, reasoning and the corresponding SQL query. -[Hugging Face TRL](https://huggingface.co/docs/trl/en/index) supports automatic templating of conversation dataset formats. This means you only need to convert your dataset into the right json objects, and `trl` takes care of templating and putting it into the right format. +[Hugging Face TRL](https://huggingface.co/docs/trl/en/index) supports automatic templating of conversation dataset formats. This means you only need to convert your dataset into the right json objects, and `trl` takes care of templating and putting it into the right format. With the prompt-completion format below, `trl` also computes the loss on the completion only, so the model learns to write the SQL query rather than to repeat the prompt. ```text -{"messages": [{"role": "system", "content": "You are..."}, {"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}]} -{"messages": [{"role": "system", "content": "You are..."}, {"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}]} -{"messages": [{"role": "system", "content": "You are..."}, {"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}]} +{"prompt": [{"role": "system", "content": "You are..."}, {"role": "user", "content": "..."}], "completion": [{"role": "assistant", "content": "..."}]} +{"prompt": [{"role": "system", "content": "You are..."}, {"role": "user", "content": "..."}], "completion": [{"role": "assistant", "content": "..."}]} +{"prompt": [{"role": "system", "content": "You are..."}, {"role": "user", "content": "..."}], "completion": [{"role": "assistant", "content": "..."}]} ``` The [philschmid/gretel-synthetic-text-to-sql](https://huggingface.co/datasets/philschmid/gretel-synthetic-text-to-sql) contains over 100k samples. To keep the guide small, it is downsampled to only use 10,000 samples. @@ -106,11 +106,13 @@ user_prompt = """Given the and the , generate the correspon """ def create_conversation(sample, idx): return { - "messages": [ + "prompt": [ {"role": "system", "content": system_message}, {"role": "user", "content": user_prompt.format(question=sample["sql_prompt"], context=sample["sql_context"])}, + ], + "completion": [ {"role": "assistant", "content": sample["sql"]} - ] + ], } # Load dataset from the hub @@ -123,7 +125,7 @@ dataset = dataset.map(create_conversation, with_indices=True, remove_columns=dat dataset = dataset.train_test_split(test_size=0.2, shuffle=False) # Print formatted user prompt -for item in dataset["train"][0]["messages"]: +for item in dataset["train"][0]["prompt"] + dataset["train"][0]["completion"]: print(item) ``` @@ -223,78 +225,7 @@ args = SFTConfig( lr_scheduler_type="constant", # use constant learning rate scheduler push_to_hub=True, # push model to hub report_to="tensorboard", # report metrics to tensorboard - dataset_kwargs={"skip_prepare_dataset": True}, # important for collator - remove_unused_columns = False, # important for collator ) - -# Data collator -def collate_fn(examples): - texts = [] - - for example in examples: - full_text = processor.apply_chat_template( - example["messages"], add_generation_prompt=False, tokenize=False - ) - texts.append(full_text.strip()) - - # Tokenize the texts and process the audios - batch = processor(text=texts, return_tensors="pt", padding=True) - - # The labels are the input_ids, and we mask the padding tokens and audio tokens in the loss computation - labels = batch["input_ids"].clone() - - target_tokens = [ - processor.tokenizer.convert_tokens_to_ids("<|turn>"), - processor.tokenizer.convert_tokens_to_ids("model"), - processor.tokenizer.convert_tokens_to_ids("\n") - ] - target_len = len(target_tokens) - - for i in range(labels.size(0)): - row_tokens = batch["input_ids"][i].tolist() - - # Find where the assistant block begins - assistant_start_idx = None - for idx in range(len(row_tokens) - target_len + 1): - if row_tokens[idx : idx + target_len] == target_tokens: - # We want to keep loss calculation on the assistant transcription tokens, - # so we move the index right past the assistant header ('<|turn>\nmodel\n') - assistant_start_idx = idx + target_len - break - - if assistant_start_idx is not None: - # Mask everything from index 0 up to the start of the actual Japanese text response - labels[i, :assistant_start_idx] = -100 - else: - # Fallback safety: if template matching fails for an anomalous row, mask padding anyway - print("WARNING: maybe the sample is too long, try to increase `token_limit` value.") - labels[i, labels[i] == processor.tokenizer.pad_token_id] = -100 - - """ - # --- DEBUG PRINT CODE --- - print(f"\n--- Example {i} (Split index: {assistant_start_idx}) ---") - debug_string = [] - for token_id, label_id in zip(row_tokens, labels[i].tolist()): - # Decode token by token so we can see exactly what is masked - decoded_token = processor.tokenizer.decode([token_id]) - - if label_id == -100: - # Red text for masked tokens (ANSI Escape Code) - debug_string.append(f"\033[91m{decoded_token}\033[0m") - else: - # Green text for active loss tokens - debug_string.append(f"\033[92m{decoded_token}\033[0m") - - print("".join(debug_string)) - # --- - """ - - - # Mask tokens for not being used in the loss computation - labels[labels == processor.tokenizer.pad_token_id] = -100 - - batch["labels"] = labels - return batch ``` You now have every building block you need to create your `SFTTrainer` to start the training of your model. @@ -309,8 +240,7 @@ trainer = SFTTrainer( train_dataset=dataset["train"], eval_dataset=dataset["test"], peft_config=peft_config, - processing_class=processor, - data_collator=collate_fn, + processing_class=processor.tokenizer, # the data is text only ) ``` @@ -416,16 +346,16 @@ rand_idx = randint(0, len(dataset["test"])) test_sample = dataset["test"][rand_idx] # Convert as test example into a prompt with the Gemma template -prompt = processor.tokenizer.apply_chat_template(test_sample["messages"][:2], tokenize=False, add_generation_prompt=True) +prompt = processor.tokenizer.apply_chat_template(test_sample["prompt"], tokenize=False, add_generation_prompt=True) print(prompt) # Generate our SQL query. outputs = pipe(text_inputs=prompt, generation_config=config) # Extract the user query and original answer -print(f"Context:\n", re.search(r'\n(.*?)\n', test_sample['messages'][1]['content'], re.DOTALL).group(1).strip()) -print(f"Query:\n", re.search(r'\n(.*?)\n', test_sample['messages'][1]['content'], re.DOTALL).group(1).strip()) -print(f"Original Answer:\n{test_sample['messages'][2]['content']}") +print(f"Context:\n", re.search(r'\n(.*?)\n', test_sample['prompt'][1]['content'], re.DOTALL).group(1).strip()) +print(f"Query:\n", re.search(r'\n(.*?)\n', test_sample['prompt'][1]['content'], re.DOTALL).group(1).strip()) +print(f"Original Answer:\n{test_sample['completion'][0]['content']}") print(f"Generated Answer:\n{outputs[0]['generated_text'][len(prompt):].strip().removesuffix("")}") ``` From 58cfb5311f7d603f2a4d8996c5c8b09e8a74799e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Fri, 16 Oct 2026 17:52:31 +0000 Subject: [PATCH 2/4] Let SFTTrainer and the pipelines load the model in the text tuning guides --- docs/notebooks/full-finetune.ipynb | 79 +++------------------- docs/notebooks/qlora-gemma-4-12b.ipynb | 43 ++++-------- docs/notebooks/qlora-gemma-4-26b-a4b.ipynb | 43 ++++-------- docs/notebooks/qlora-gemma-4-31b.ipynb | 43 ++++-------- docs/notebooks/qlora-gemma-4-e2b.ipynb | 43 ++++-------- docs/notebooks/qlora-gemma-4-e4b.ipynb | 43 ++++-------- docs/tuning/full-finetune.md | 58 ++-------------- docs/tuning/qlora.md | 43 ++++-------- 8 files changed, 105 insertions(+), 290 deletions(-) diff --git a/docs/notebooks/full-finetune.ipynb b/docs/notebooks/full-finetune.ipynb index 804bd4b..d35bf83 100644 --- a/docs/notebooks/full-finetune.ipynb +++ b/docs/notebooks/full-finetune.ipynb @@ -189,37 +189,6 @@ "\n", "You are now ready to fine-tune your model. Hugging Face TRL [SFTTrainer](https://huggingface.co/docs/trl/sft_trainer) makes it straightforward to supervise fine-tune open LLMs. The `SFTTrainer` is a subclass of the `Trainer` from the `transformers` library and supports all the same features,\n", "\n", - "The following code loads the Gemma model and tokenizer from Hugging Face." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "language": "python" - }, - "outputs": [], - "source": [ - "import torch\n", - "from transformers import AutoTokenizer, AutoModelForCausalLM\n", - "\n", - "# Load model and tokenizer\n", - "model = AutoModelForCausalLM.from_pretrained(\n", - " base_model,\n", - " dtype=\"auto\",\n", - " device_map=\"auto\",\n", - " attn_implementation=\"eager\"\n", - ")\n", - "tokenizer = AutoTokenizer.from_pretrained(base_model)\n", - "\n", - "print(f\"Device: {model.device}\")\n", - "print(f\"DType: {model.dtype}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ "## Before fine-tune\n", "\n", "The output below shows that the out-of-the-box capabilities may not be good enough for this use case." @@ -238,8 +207,8 @@ "from random import randint\n", "import re\n", "\n", - "# Load the model and tokenizer into the pipeline\n", - "pipe = pipeline(\"text-generation\", model=model, tokenizer=tokenizer)\n", + "# Load the base model into the pipeline\n", + "pipe = pipeline(\"text-generation\", model=base_model, dtype=\"auto\")\n", "\n", "# Load a random sample from the test dataset\n", "rand_idx = randint(0, len(dataset[\"test\"])-1)\n", @@ -335,22 +304,18 @@ "source": [ "from trl import SFTConfig\n", "\n", - "torch_dtype = model.dtype\n", - "\n", "args = SFTConfig(\n", " output_dir=checkpoint_dir, # directory to save and repository id\n", " max_length=512, # max sequence length for model and packing of the dataset\n", " packing=False, # Groups multiple samples in the dataset into a single sequence\n", " num_train_epochs=5, # number of training epochs\n", " per_device_train_batch_size=4, # batch size per device during training\n", - " gradient_checkpointing=False, # Caching is incompatible with gradient checkpointing\n", + " gradient_checkpointing=False, # the model is small, so skip gradient checkpointing for speed\n", " optim=\"adamw_torch_fused\", # use fused adamw optimizer\n", " logging_steps=1, # log every step\n", " save_strategy=\"epoch\", # save checkpoint every epoch\n", " eval_strategy=\"epoch\", # evaluate checkpoint every epoch\n", " learning_rate=learning_rate, # learning rate\n", - " fp16=True if torch_dtype == torch.float16 else False, # use float16 precision\n", - " bf16=True if torch_dtype == torch.bfloat16 else False, # use bfloat16 precision\n", " lr_scheduler_type=\"constant\", # use constant learning rate scheduler\n", " push_to_hub=True, # push model to hub\n", " report_to=\"tensorboard\", # report metrics to tensorboard\n", @@ -376,11 +341,10 @@ "\n", "# Create Trainer object\n", "trainer = SFTTrainer(\n", - " model=model,\n", + " model=base_model,\n", " args=args,\n", " train_dataset=dataset['train'],\n", " eval_dataset=dataset['test'],\n", - " processing_class=tokenizer,\n", ")\n" ] }, @@ -462,36 +426,9 @@ "\n", "After the training is done, you'll want to evaluate and test your model. You can load different samples from the test dataset and evaluate the model on those samples.\n", "\n", - "For this particular use case, the best model is a matter of preference. Interestingly, what we'd normally call 'overfitting' can be very useful for a game NPC. It forces the model to forget general information and instead lock onto the specific persona and characteristics it was trained on, ensuring it stays consistently in character." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "language": "python" - }, - "outputs": [], - "source": [ - "from transformers import AutoTokenizer, AutoModelForCausalLM\n", - "\n", - "model_id = checkpoint_dir\n", + "For this particular use case, the best model is a matter of preference. Interestingly, what we'd normally call 'overfitting' can be very useful for a game NPC. It forces the model to forget general information and instead lock onto the specific persona and characteristics it was trained on, ensuring it stays consistently in character.\n", "\n", - "# Load Model\n", - "model = AutoModelForCausalLM.from_pretrained(\n", - " model_id,\n", - " dtype=\"auto\",\n", - " device_map=\"auto\",\n", - " attn_implementation=\"eager\"\n", - ")\n", - "tokenizer = AutoTokenizer.from_pretrained(model_id)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's load all questions from the test dataset and generate outputs." + "Let's load the fine-tuned model, then generate outputs for all questions from the test dataset." ] }, { @@ -504,8 +441,8 @@ "source": [ "from transformers import pipeline\n", "\n", - "# Load the model and tokenizer into the pipeline\n", - "pipe = pipeline(\"text-generation\", model=model, tokenizer=tokenizer)\n", + "# Load the fine-tuned model into the pipeline\n", + "pipe = pipeline(\"text-generation\", model=checkpoint_dir, dtype=\"auto\")\n", "\n", "def test(test_sample):\n", " # Convert as test example into a prompt with the Gemma template\n", diff --git a/docs/notebooks/qlora-gemma-4-12b.ipynb b/docs/notebooks/qlora-gemma-4-12b.ipynb index 182fb6f..65585d4 100644 --- a/docs/notebooks/qlora-gemma-4-12b.ipynb +++ b/docs/notebooks/qlora-gemma-4-12b.ipynb @@ -200,7 +200,7 @@ "- Parameter-efficient fine-tuning (PEFT) support including QloRA\n", "- Preparing the model and tokenizer for conversational fine-tuning (such as adding special tokens)\n", "\n", - "The following code loads the Gemma model and tokenizer from Hugging Face and initializes the quantization configuration." + "The following code defines the quantization configuration and loads the tokenizer. The `SFTTrainer` loads and quantizes the model for you." ] }, { @@ -212,41 +212,28 @@ "outputs": [], "source": [ "import torch\n", - "from transformers import AutoProcessor, AutoModelForMultimodalLM, BitsAndBytesConfig\n", - "from peft import prepare_model_for_kbit_training\n", + "from transformers import AutoTokenizer, BitsAndBytesConfig\n", "\n", "# Hugging Face model id\n", "model_id = \"google/gemma-4-12B\"\n", "\n", "# Check if GPU supports bfloat16\n", "if torch.cuda.is_bf16_supported():\n", - " torch_dtype = torch.bfloat16\n", + " dtype = \"bfloat16\"\n", "else:\n", - " torch_dtype = torch.float16\n", - "\n", - "# Define model init arguments\n", - "model_kwargs = dict(\n", - " dtype=torch_dtype, # What torch dtype to use\n", - " device_map=\"auto\", # Let torch decide how to load the model\n", - ")\n", + " dtype = \"float16\"\n", "\n", "# BitsAndBytesConfig: Enables 4-bit quantization to reduce model size/memory usage\n", - "model_kwargs[\"quantization_config\"] = BitsAndBytesConfig(\n", + "quantization_config = BitsAndBytesConfig(\n", " load_in_4bit=True,\n", " bnb_4bit_use_double_quant=True,\n", " bnb_4bit_quant_type='nf4',\n", - " bnb_4bit_compute_dtype=torch_dtype,\n", - " bnb_4bit_quant_storage=torch_dtype,\n", + " bnb_4bit_compute_dtype=dtype,\n", + " bnb_4bit_quant_storage=dtype,\n", ")\n", "\n", - "# Load model and processor\n", - "model = AutoModelForMultimodalLM.from_pretrained(model_id, **model_kwargs)\n", - "processor = AutoProcessor.from_pretrained(\"google/gemma-4-12B-it\") # Load the Instruction Processor to use the official Gemma template\n", - "\n", - "# NOTE: You should call the prepare_model_for_kbit_training() function to preprocess the quantized model for training.\n", - "# On T4, we are skipping this step purely due to VRAM limitation and for a quick demonstration.\n", - "if (torch.cuda.get_device_properties(0).total_memory/1024**3) > 16:\n", - " model = prepare_model_for_kbit_training(model)\n" + "# The base model ships no chat template, so load the tokenizer of the instruction-tuned model\n", + "tokenizer = AutoTokenizer.from_pretrained(\"google/gemma-4-12B-it\")\n" ] }, { @@ -293,7 +280,6 @@ }, "outputs": [], "source": [ - "import torch\n", "from trl import SFTConfig\n", "\n", "args = SFTConfig(\n", @@ -307,8 +293,9 @@ " save_strategy=\"epoch\", # save checkpoint every epoch\n", " eval_strategy=\"epoch\", # evaluate checkpoint every epoch\n", " learning_rate=2e-4, # learning rate\n", - " fp16=True if torch_dtype == torch.float16 else False, # use float16 precision\n", - " bf16=True if torch_dtype == torch.bfloat16 else False, # use bfloat16 precision\n", + " fp16=dtype == \"float16\", # use float16 precision\n", + " bf16=dtype == \"bfloat16\", # use bfloat16 precision\n", + " model_init_kwargs={\"dtype\": dtype}, # dtype of the weights that are not quantized\n", " lr_scheduler_type=\"constant\", # use constant learning rate scheduler\n", " push_to_hub=True, # push model to hub\n", " report_to=\"tensorboard\", # report metrics to tensorboard\n", @@ -334,12 +321,13 @@ "\n", "# Create Trainer object\n", "trainer = SFTTrainer(\n", - " model=model,\n", + " model=model_id,\n", " args=args,\n", " train_dataset=dataset[\"train\"],\n", " eval_dataset=dataset[\"test\"],\n", + " processing_class=tokenizer,\n", + " quantization_config=quantization_config,\n", " peft_config=peft_config,\n", - " processing_class=processor.tokenizer, # the data is text only\n", ")\n" ] }, @@ -381,7 +369,6 @@ "outputs": [], "source": [ "# free the memory again\n", - "del model\n", "del trainer\n", "torch.cuda.empty_cache()\n" ] diff --git a/docs/notebooks/qlora-gemma-4-26b-a4b.ipynb b/docs/notebooks/qlora-gemma-4-26b-a4b.ipynb index fc1fd1a..f5887c0 100644 --- a/docs/notebooks/qlora-gemma-4-26b-a4b.ipynb +++ b/docs/notebooks/qlora-gemma-4-26b-a4b.ipynb @@ -200,7 +200,7 @@ "- Parameter-efficient fine-tuning (PEFT) support including QloRA\n", "- Preparing the model and tokenizer for conversational fine-tuning (such as adding special tokens)\n", "\n", - "The following code loads the Gemma model and tokenizer from Hugging Face and initializes the quantization configuration." + "The following code defines the quantization configuration and loads the tokenizer. The `SFTTrainer` loads and quantizes the model for you." ] }, { @@ -212,41 +212,28 @@ "outputs": [], "source": [ "import torch\n", - "from transformers import AutoProcessor, AutoModelForMultimodalLM, BitsAndBytesConfig\n", - "from peft import prepare_model_for_kbit_training\n", + "from transformers import AutoTokenizer, BitsAndBytesConfig\n", "\n", "# Hugging Face model id\n", "model_id = \"google/gemma-4-26B-A4B\"\n", "\n", "# Check if GPU supports bfloat16\n", "if torch.cuda.is_bf16_supported():\n", - " torch_dtype = torch.bfloat16\n", + " dtype = \"bfloat16\"\n", "else:\n", - " torch_dtype = torch.float16\n", - "\n", - "# Define model init arguments\n", - "model_kwargs = dict(\n", - " dtype=torch_dtype, # What torch dtype to use\n", - " device_map=\"auto\", # Let torch decide how to load the model\n", - ")\n", + " dtype = \"float16\"\n", "\n", "# BitsAndBytesConfig: Enables 4-bit quantization to reduce model size/memory usage\n", - "model_kwargs[\"quantization_config\"] = BitsAndBytesConfig(\n", + "quantization_config = BitsAndBytesConfig(\n", " load_in_4bit=True,\n", " bnb_4bit_use_double_quant=True,\n", " bnb_4bit_quant_type='nf4',\n", - " bnb_4bit_compute_dtype=torch_dtype,\n", - " bnb_4bit_quant_storage=torch_dtype,\n", + " bnb_4bit_compute_dtype=dtype,\n", + " bnb_4bit_quant_storage=dtype,\n", ")\n", "\n", - "# Load model and processor\n", - "model = AutoModelForMultimodalLM.from_pretrained(model_id, **model_kwargs)\n", - "processor = AutoProcessor.from_pretrained(\"google/gemma-4-26B-A4B-it\") # Load the Instruction Processor to use the official Gemma template\n", - "\n", - "# NOTE: You should call the prepare_model_for_kbit_training() function to preprocess the quantized model for training.\n", - "# On T4, we are skipping this step purely due to VRAM limitation and for a quick demonstration.\n", - "if (torch.cuda.get_device_properties(0).total_memory/1024**3) > 16:\n", - " model = prepare_model_for_kbit_training(model)\n" + "# The base model ships no chat template, so load the tokenizer of the instruction-tuned model\n", + "tokenizer = AutoTokenizer.from_pretrained(\"google/gemma-4-26B-A4B-it\")\n" ] }, { @@ -293,7 +280,6 @@ }, "outputs": [], "source": [ - "import torch\n", "from trl import SFTConfig\n", "\n", "args = SFTConfig(\n", @@ -307,8 +293,9 @@ " save_strategy=\"epoch\", # save checkpoint every epoch\n", " eval_strategy=\"epoch\", # evaluate checkpoint every epoch\n", " learning_rate=2e-4, # learning rate\n", - " fp16=True if torch_dtype == torch.float16 else False, # use float16 precision\n", - " bf16=True if torch_dtype == torch.bfloat16 else False, # use bfloat16 precision\n", + " fp16=dtype == \"float16\", # use float16 precision\n", + " bf16=dtype == \"bfloat16\", # use bfloat16 precision\n", + " model_init_kwargs={\"dtype\": dtype}, # dtype of the weights that are not quantized\n", " lr_scheduler_type=\"constant\", # use constant learning rate scheduler\n", " push_to_hub=True, # push model to hub\n", " report_to=\"tensorboard\", # report metrics to tensorboard\n", @@ -334,12 +321,13 @@ "\n", "# Create Trainer object\n", "trainer = SFTTrainer(\n", - " model=model,\n", + " model=model_id,\n", " args=args,\n", " train_dataset=dataset[\"train\"],\n", " eval_dataset=dataset[\"test\"],\n", + " processing_class=tokenizer,\n", + " quantization_config=quantization_config,\n", " peft_config=peft_config,\n", - " processing_class=processor.tokenizer, # the data is text only\n", ")\n" ] }, @@ -381,7 +369,6 @@ "outputs": [], "source": [ "# free the memory again\n", - "del model\n", "del trainer\n", "torch.cuda.empty_cache()\n" ] diff --git a/docs/notebooks/qlora-gemma-4-31b.ipynb b/docs/notebooks/qlora-gemma-4-31b.ipynb index 814ba25..374da8f 100644 --- a/docs/notebooks/qlora-gemma-4-31b.ipynb +++ b/docs/notebooks/qlora-gemma-4-31b.ipynb @@ -200,7 +200,7 @@ "- Parameter-efficient fine-tuning (PEFT) support including QloRA\n", "- Preparing the model and tokenizer for conversational fine-tuning (such as adding special tokens)\n", "\n", - "The following code loads the Gemma model and tokenizer from Hugging Face and initializes the quantization configuration." + "The following code defines the quantization configuration and loads the tokenizer. The `SFTTrainer` loads and quantizes the model for you." ] }, { @@ -212,41 +212,28 @@ "outputs": [], "source": [ "import torch\n", - "from transformers import AutoProcessor, AutoModelForMultimodalLM, BitsAndBytesConfig\n", - "from peft import prepare_model_for_kbit_training\n", + "from transformers import AutoTokenizer, BitsAndBytesConfig\n", "\n", "# Hugging Face model id\n", "model_id = \"google/gemma-4-31B\"\n", "\n", "# Check if GPU supports bfloat16\n", "if torch.cuda.is_bf16_supported():\n", - " torch_dtype = torch.bfloat16\n", + " dtype = \"bfloat16\"\n", "else:\n", - " torch_dtype = torch.float16\n", - "\n", - "# Define model init arguments\n", - "model_kwargs = dict(\n", - " dtype=torch_dtype, # What torch dtype to use\n", - " device_map=\"auto\", # Let torch decide how to load the model\n", - ")\n", + " dtype = \"float16\"\n", "\n", "# BitsAndBytesConfig: Enables 4-bit quantization to reduce model size/memory usage\n", - "model_kwargs[\"quantization_config\"] = BitsAndBytesConfig(\n", + "quantization_config = BitsAndBytesConfig(\n", " load_in_4bit=True,\n", " bnb_4bit_use_double_quant=True,\n", " bnb_4bit_quant_type='nf4',\n", - " bnb_4bit_compute_dtype=torch_dtype,\n", - " bnb_4bit_quant_storage=torch_dtype,\n", + " bnb_4bit_compute_dtype=dtype,\n", + " bnb_4bit_quant_storage=dtype,\n", ")\n", "\n", - "# Load model and processor\n", - "model = AutoModelForMultimodalLM.from_pretrained(model_id, **model_kwargs)\n", - "processor = AutoProcessor.from_pretrained(\"google/gemma-4-31B-it\") # Load the Instruction Processor to use the official Gemma template\n", - "\n", - "# NOTE: You should call the prepare_model_for_kbit_training() function to preprocess the quantized model for training.\n", - "# On T4, we are skipping this step purely due to VRAM limitation and for a quick demonstration.\n", - "if (torch.cuda.get_device_properties(0).total_memory/1024**3) > 16:\n", - " model = prepare_model_for_kbit_training(model)\n" + "# The base model ships no chat template, so load the tokenizer of the instruction-tuned model\n", + "tokenizer = AutoTokenizer.from_pretrained(\"google/gemma-4-31B-it\")\n" ] }, { @@ -293,7 +280,6 @@ }, "outputs": [], "source": [ - "import torch\n", "from trl import SFTConfig\n", "\n", "args = SFTConfig(\n", @@ -307,8 +293,9 @@ " save_strategy=\"epoch\", # save checkpoint every epoch\n", " eval_strategy=\"epoch\", # evaluate checkpoint every epoch\n", " learning_rate=2e-4, # learning rate\n", - " fp16=True if torch_dtype == torch.float16 else False, # use float16 precision\n", - " bf16=True if torch_dtype == torch.bfloat16 else False, # use bfloat16 precision\n", + " fp16=dtype == \"float16\", # use float16 precision\n", + " bf16=dtype == \"bfloat16\", # use bfloat16 precision\n", + " model_init_kwargs={\"dtype\": dtype}, # dtype of the weights that are not quantized\n", " lr_scheduler_type=\"constant\", # use constant learning rate scheduler\n", " push_to_hub=True, # push model to hub\n", " report_to=\"tensorboard\", # report metrics to tensorboard\n", @@ -334,12 +321,13 @@ "\n", "# Create Trainer object\n", "trainer = SFTTrainer(\n", - " model=model,\n", + " model=model_id,\n", " args=args,\n", " train_dataset=dataset[\"train\"],\n", " eval_dataset=dataset[\"test\"],\n", + " processing_class=tokenizer,\n", + " quantization_config=quantization_config,\n", " peft_config=peft_config,\n", - " processing_class=processor.tokenizer, # the data is text only\n", ")\n" ] }, @@ -381,7 +369,6 @@ "outputs": [], "source": [ "# free the memory again\n", - "del model\n", "del trainer\n", "torch.cuda.empty_cache()\n" ] diff --git a/docs/notebooks/qlora-gemma-4-e2b.ipynb b/docs/notebooks/qlora-gemma-4-e2b.ipynb index f490ead..fceb269 100644 --- a/docs/notebooks/qlora-gemma-4-e2b.ipynb +++ b/docs/notebooks/qlora-gemma-4-e2b.ipynb @@ -200,7 +200,7 @@ "- Parameter-efficient fine-tuning (PEFT) support including QloRA\n", "- Preparing the model and tokenizer for conversational fine-tuning (such as adding special tokens)\n", "\n", - "The following code loads the Gemma model and tokenizer from Hugging Face and initializes the quantization configuration." + "The following code defines the quantization configuration and loads the tokenizer. The `SFTTrainer` loads and quantizes the model for you." ] }, { @@ -212,41 +212,28 @@ "outputs": [], "source": [ "import torch\n", - "from transformers import AutoProcessor, AutoModelForMultimodalLM, BitsAndBytesConfig\n", - "from peft import prepare_model_for_kbit_training\n", + "from transformers import AutoTokenizer, BitsAndBytesConfig\n", "\n", "# Hugging Face model id\n", "model_id = \"google/gemma-4-E2B\"\n", "\n", "# Check if GPU supports bfloat16\n", "if torch.cuda.is_bf16_supported():\n", - " torch_dtype = torch.bfloat16\n", + " dtype = \"bfloat16\"\n", "else:\n", - " torch_dtype = torch.float16\n", - "\n", - "# Define model init arguments\n", - "model_kwargs = dict(\n", - " dtype=torch_dtype, # What torch dtype to use\n", - " device_map=\"auto\", # Let torch decide how to load the model\n", - ")\n", + " dtype = \"float16\"\n", "\n", "# BitsAndBytesConfig: Enables 4-bit quantization to reduce model size/memory usage\n", - "model_kwargs[\"quantization_config\"] = BitsAndBytesConfig(\n", + "quantization_config = BitsAndBytesConfig(\n", " load_in_4bit=True,\n", " bnb_4bit_use_double_quant=True,\n", " bnb_4bit_quant_type='nf4',\n", - " bnb_4bit_compute_dtype=torch_dtype,\n", - " bnb_4bit_quant_storage=torch_dtype,\n", + " bnb_4bit_compute_dtype=dtype,\n", + " bnb_4bit_quant_storage=dtype,\n", ")\n", "\n", - "# Load model and processor\n", - "model = AutoModelForMultimodalLM.from_pretrained(model_id, **model_kwargs)\n", - "processor = AutoProcessor.from_pretrained(\"google/gemma-4-E2B-it\") # Load the Instruction Processor to use the official Gemma template\n", - "\n", - "# NOTE: You should call the prepare_model_for_kbit_training() function to preprocess the quantized model for training.\n", - "# On T4, we are skipping this step purely due to VRAM limitation and for a quick demonstration.\n", - "if (torch.cuda.get_device_properties(0).total_memory/1024**3) > 16:\n", - " model = prepare_model_for_kbit_training(model)\n" + "# The base model ships no chat template, so load the tokenizer of the instruction-tuned model\n", + "tokenizer = AutoTokenizer.from_pretrained(\"google/gemma-4-E2B-it\")\n" ] }, { @@ -293,7 +280,6 @@ }, "outputs": [], "source": [ - "import torch\n", "from trl import SFTConfig\n", "\n", "args = SFTConfig(\n", @@ -307,8 +293,9 @@ " save_strategy=\"epoch\", # save checkpoint every epoch\n", " eval_strategy=\"epoch\", # evaluate checkpoint every epoch\n", " learning_rate=2e-4, # learning rate\n", - " fp16=True if torch_dtype == torch.float16 else False, # use float16 precision\n", - " bf16=True if torch_dtype == torch.bfloat16 else False, # use bfloat16 precision\n", + " fp16=dtype == \"float16\", # use float16 precision\n", + " bf16=dtype == \"bfloat16\", # use bfloat16 precision\n", + " model_init_kwargs={\"dtype\": dtype}, # dtype of the weights that are not quantized\n", " lr_scheduler_type=\"constant\", # use constant learning rate scheduler\n", " push_to_hub=True, # push model to hub\n", " report_to=\"tensorboard\", # report metrics to tensorboard\n", @@ -334,12 +321,13 @@ "\n", "# Create Trainer object\n", "trainer = SFTTrainer(\n", - " model=model,\n", + " model=model_id,\n", " args=args,\n", " train_dataset=dataset[\"train\"],\n", " eval_dataset=dataset[\"test\"],\n", + " processing_class=tokenizer,\n", + " quantization_config=quantization_config,\n", " peft_config=peft_config,\n", - " processing_class=processor.tokenizer, # the data is text only\n", ")\n" ] }, @@ -381,7 +369,6 @@ "outputs": [], "source": [ "# free the memory again\n", - "del model\n", "del trainer\n", "torch.cuda.empty_cache()\n" ] diff --git a/docs/notebooks/qlora-gemma-4-e4b.ipynb b/docs/notebooks/qlora-gemma-4-e4b.ipynb index 317debc..2b7babf 100644 --- a/docs/notebooks/qlora-gemma-4-e4b.ipynb +++ b/docs/notebooks/qlora-gemma-4-e4b.ipynb @@ -200,7 +200,7 @@ "- Parameter-efficient fine-tuning (PEFT) support including QloRA\n", "- Preparing the model and tokenizer for conversational fine-tuning (such as adding special tokens)\n", "\n", - "The following code loads the Gemma model and tokenizer from Hugging Face and initializes the quantization configuration." + "The following code defines the quantization configuration and loads the tokenizer. The `SFTTrainer` loads and quantizes the model for you." ] }, { @@ -212,41 +212,28 @@ "outputs": [], "source": [ "import torch\n", - "from transformers import AutoProcessor, AutoModelForMultimodalLM, BitsAndBytesConfig\n", - "from peft import prepare_model_for_kbit_training\n", + "from transformers import AutoTokenizer, BitsAndBytesConfig\n", "\n", "# Hugging Face model id\n", "model_id = \"google/gemma-4-E4B\"\n", "\n", "# Check if GPU supports bfloat16\n", "if torch.cuda.is_bf16_supported():\n", - " torch_dtype = torch.bfloat16\n", + " dtype = \"bfloat16\"\n", "else:\n", - " torch_dtype = torch.float16\n", - "\n", - "# Define model init arguments\n", - "model_kwargs = dict(\n", - " dtype=torch_dtype, # What torch dtype to use\n", - " device_map=\"auto\", # Let torch decide how to load the model\n", - ")\n", + " dtype = \"float16\"\n", "\n", "# BitsAndBytesConfig: Enables 4-bit quantization to reduce model size/memory usage\n", - "model_kwargs[\"quantization_config\"] = BitsAndBytesConfig(\n", + "quantization_config = BitsAndBytesConfig(\n", " load_in_4bit=True,\n", " bnb_4bit_use_double_quant=True,\n", " bnb_4bit_quant_type='nf4',\n", - " bnb_4bit_compute_dtype=torch_dtype,\n", - " bnb_4bit_quant_storage=torch_dtype,\n", + " bnb_4bit_compute_dtype=dtype,\n", + " bnb_4bit_quant_storage=dtype,\n", ")\n", "\n", - "# Load model and processor\n", - "model = AutoModelForMultimodalLM.from_pretrained(model_id, **model_kwargs)\n", - "processor = AutoProcessor.from_pretrained(\"google/gemma-4-E4B-it\") # Load the Instruction Processor to use the official Gemma template\n", - "\n", - "# NOTE: You should call the prepare_model_for_kbit_training() function to preprocess the quantized model for training.\n", - "# On T4, we are skipping this step purely due to VRAM limitation and for a quick demonstration.\n", - "if (torch.cuda.get_device_properties(0).total_memory/1024**3) > 16:\n", - " model = prepare_model_for_kbit_training(model)\n" + "# The base model ships no chat template, so load the tokenizer of the instruction-tuned model\n", + "tokenizer = AutoTokenizer.from_pretrained(\"google/gemma-4-E4B-it\")\n" ] }, { @@ -293,7 +280,6 @@ }, "outputs": [], "source": [ - "import torch\n", "from trl import SFTConfig\n", "\n", "args = SFTConfig(\n", @@ -307,8 +293,9 @@ " save_strategy=\"epoch\", # save checkpoint every epoch\n", " eval_strategy=\"epoch\", # evaluate checkpoint every epoch\n", " learning_rate=2e-4, # learning rate\n", - " fp16=True if torch_dtype == torch.float16 else False, # use float16 precision\n", - " bf16=True if torch_dtype == torch.bfloat16 else False, # use bfloat16 precision\n", + " fp16=dtype == \"float16\", # use float16 precision\n", + " bf16=dtype == \"bfloat16\", # use bfloat16 precision\n", + " model_init_kwargs={\"dtype\": dtype}, # dtype of the weights that are not quantized\n", " lr_scheduler_type=\"constant\", # use constant learning rate scheduler\n", " push_to_hub=True, # push model to hub\n", " report_to=\"tensorboard\", # report metrics to tensorboard\n", @@ -334,12 +321,13 @@ "\n", "# Create Trainer object\n", "trainer = SFTTrainer(\n", - " model=model,\n", + " model=model_id,\n", " args=args,\n", " train_dataset=dataset[\"train\"],\n", " eval_dataset=dataset[\"test\"],\n", + " processing_class=tokenizer,\n", + " quantization_config=quantization_config,\n", " peft_config=peft_config,\n", - " processing_class=processor.tokenizer, # the data is text only\n", ")\n" ] }, @@ -381,7 +369,6 @@ "outputs": [], "source": [ "# free the memory again\n", - "del model\n", "del trainer\n", "torch.cuda.empty_cache()\n" ] diff --git a/docs/tuning/full-finetune.md b/docs/tuning/full-finetune.md index fa6017e..061b450 100644 --- a/docs/tuning/full-finetune.md +++ b/docs/tuning/full-finetune.md @@ -114,30 +114,6 @@ Map: 0%| | 0/25 [00:00 16: - model = prepare_model_for_kbit_training(model) +# The base model ships no chat template, so load the tokenizer of the instruction-tuned model +tokenizer = AutoTokenizer.from_pretrained("google/gemma-4-E2B-it") ``` The `SFTTrainer` supports a built-in integration with `peft`, which makes it straightforward to efficiently tune LLMs using QLoRA. You only need to create a `LoraConfig` and provide it to the trainer. @@ -206,7 +193,6 @@ peft_config = LoraConfig( Before you can start your training, you need to define the hyperparameter you want to use in a `SFTConfig` instance. ```python { .notebook-cell } -import torch from trl import SFTConfig args = SFTConfig( @@ -220,8 +206,9 @@ args = SFTConfig( save_strategy="epoch", # save checkpoint every epoch eval_strategy="epoch", # evaluate checkpoint every epoch learning_rate=2e-4, # learning rate - fp16=True if torch_dtype == torch.float16 else False, # use float16 precision - bf16=True if torch_dtype == torch.bfloat16 else False, # use bfloat16 precision + fp16=dtype == "float16", # use float16 precision + bf16=dtype == "bfloat16", # use bfloat16 precision + model_init_kwargs={"dtype": dtype}, # dtype of the weights that are not quantized lr_scheduler_type="constant", # use constant learning rate scheduler push_to_hub=True, # push model to hub report_to="tensorboard", # report metrics to tensorboard @@ -235,12 +222,13 @@ from trl import SFTTrainer # Create Trainer object trainer = SFTTrainer( - model=model, + model=model_id, args=args, train_dataset=dataset["train"], eval_dataset=dataset["test"], + processing_class=tokenizer, + quantization_config=quantization_config, peft_config=peft_config, - processing_class=processor.tokenizer, # the data is text only ) ``` @@ -268,7 +256,6 @@ Before you can test your model, make sure to free the memory. ```python { .notebook-cell } # free the memory again -del model del trainer torch.cuda.empty_cache() ``` From 92fd907b49779401cb2f9ea86bad9d1789e718b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Fri, 16 Oct 2026 19:21:43 +0000 Subject: [PATCH 3/4] Keep the default max_length in the QLoRA guide so no sample is truncated --- docs/notebooks/qlora-gemma-4-12b.ipynb | 1 - docs/notebooks/qlora-gemma-4-26b-a4b.ipynb | 1 - docs/notebooks/qlora-gemma-4-31b.ipynb | 1 - docs/notebooks/qlora-gemma-4-e2b.ipynb | 1 - docs/notebooks/qlora-gemma-4-e4b.ipynb | 1 - docs/tuning/qlora.md | 1 - 6 files changed, 6 deletions(-) diff --git a/docs/notebooks/qlora-gemma-4-12b.ipynb b/docs/notebooks/qlora-gemma-4-12b.ipynb index 65585d4..6d3afd0 100644 --- a/docs/notebooks/qlora-gemma-4-12b.ipynb +++ b/docs/notebooks/qlora-gemma-4-12b.ipynb @@ -284,7 +284,6 @@ "\n", "args = SFTConfig(\n", " output_dir=\"gemma-text-to-sql\", # directory to save and repository id\n", - " max_length=512, # max length for model and packing of the dataset\n", " num_train_epochs=3, # number of training epochs\n", " per_device_train_batch_size=1, # batch size per device during training\n", " per_device_eval_batch_size=1, # batch size per device during evaluation\n", diff --git a/docs/notebooks/qlora-gemma-4-26b-a4b.ipynb b/docs/notebooks/qlora-gemma-4-26b-a4b.ipynb index f5887c0..16076ab 100644 --- a/docs/notebooks/qlora-gemma-4-26b-a4b.ipynb +++ b/docs/notebooks/qlora-gemma-4-26b-a4b.ipynb @@ -284,7 +284,6 @@ "\n", "args = SFTConfig(\n", " output_dir=\"gemma-text-to-sql\", # directory to save and repository id\n", - " max_length=512, # max length for model and packing of the dataset\n", " num_train_epochs=3, # number of training epochs\n", " per_device_train_batch_size=1, # batch size per device during training\n", " per_device_eval_batch_size=1, # batch size per device during evaluation\n", diff --git a/docs/notebooks/qlora-gemma-4-31b.ipynb b/docs/notebooks/qlora-gemma-4-31b.ipynb index 374da8f..917376f 100644 --- a/docs/notebooks/qlora-gemma-4-31b.ipynb +++ b/docs/notebooks/qlora-gemma-4-31b.ipynb @@ -284,7 +284,6 @@ "\n", "args = SFTConfig(\n", " output_dir=\"gemma-text-to-sql\", # directory to save and repository id\n", - " max_length=512, # max length for model and packing of the dataset\n", " num_train_epochs=3, # number of training epochs\n", " per_device_train_batch_size=1, # batch size per device during training\n", " per_device_eval_batch_size=1, # batch size per device during evaluation\n", diff --git a/docs/notebooks/qlora-gemma-4-e2b.ipynb b/docs/notebooks/qlora-gemma-4-e2b.ipynb index fceb269..e5f502a 100644 --- a/docs/notebooks/qlora-gemma-4-e2b.ipynb +++ b/docs/notebooks/qlora-gemma-4-e2b.ipynb @@ -284,7 +284,6 @@ "\n", "args = SFTConfig(\n", " output_dir=\"gemma-text-to-sql\", # directory to save and repository id\n", - " max_length=512, # max length for model and packing of the dataset\n", " num_train_epochs=3, # number of training epochs\n", " per_device_train_batch_size=1, # batch size per device during training\n", " per_device_eval_batch_size=1, # batch size per device during evaluation\n", diff --git a/docs/notebooks/qlora-gemma-4-e4b.ipynb b/docs/notebooks/qlora-gemma-4-e4b.ipynb index 2b7babf..e47f9e9 100644 --- a/docs/notebooks/qlora-gemma-4-e4b.ipynb +++ b/docs/notebooks/qlora-gemma-4-e4b.ipynb @@ -284,7 +284,6 @@ "\n", "args = SFTConfig(\n", " output_dir=\"gemma-text-to-sql\", # directory to save and repository id\n", - " max_length=512, # max length for model and packing of the dataset\n", " num_train_epochs=3, # number of training epochs\n", " per_device_train_batch_size=1, # batch size per device during training\n", " per_device_eval_batch_size=1, # batch size per device during evaluation\n", diff --git a/docs/tuning/qlora.md b/docs/tuning/qlora.md index 9877645..c620626 100644 --- a/docs/tuning/qlora.md +++ b/docs/tuning/qlora.md @@ -197,7 +197,6 @@ from trl import SFTConfig args = SFTConfig( output_dir="gemma-text-to-sql", # directory to save and repository id - max_length=512, # max length for model and packing of the dataset num_train_epochs=3, # number of training epochs per_device_train_batch_size=1, # batch size per device during training per_device_eval_batch_size=1, # batch size per device during evaluation From 4e3471f1ee7d765c4e01efe34d393fcffce238f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Fri, 16 Oct 2026 19:50:45 +0000 Subject: [PATCH 4/4] Let SFTTrainer load the model in the FunctionGemma fine-tuning guide --- docs/models/functiongemma/fine-tuning.md | 135 ++++++++---------- .../notebooks/functiongemma-fine-tuning.ipynb | 59 ++++---- 2 files changed, 91 insertions(+), 103 deletions(-) diff --git a/docs/models/functiongemma/fine-tuning.md b/docs/models/functiongemma/fine-tuning.md index c474638..ccb1d20 100644 --- a/docs/models/functiongemma/fine-tuning.md +++ b/docs/models/functiongemma/fine-tuning.md @@ -197,23 +197,13 @@ When using `shuffle=False` to your own custom datasets, ensure your source data You are now ready to fine-tune your model. Hugging Face TRL [SFTTrainer](https://huggingface.co/docs/trl/sft_trainer) makes it straightforward to supervise fine-tune open LLMs. The `SFTTrainer` is a subclass of the `Trainer` from the `transformers` library and supports all the same features, -The following code loads the FunctionGemma model and tokenizer from Hugging Face. +The following code loads the FunctionGemma tokenizer from Hugging Face and shows how the chat template formats a training example. The `SFTTrainer` loads the model for you. ```python { .notebook-cell } -import torch -from transformers import AutoTokenizer, AutoModelForCausalLM - -# Load model and tokenizer -model = AutoModelForCausalLM.from_pretrained( - base_model, - dtype="auto", - device_map="auto", - attn_implementation="eager" -) -tokenizer = AutoTokenizer.from_pretrained(base_model) +from transformers import AutoTokenizer -print(f"Device: {model.device}") -print(f"DType: {model.dtype}") +# Load the tokenizer +tokenizer = AutoTokenizer.from_pretrained(base_model) # Print formatted user prompt print("--- dataset input ---") @@ -224,8 +214,6 @@ print(debug_msg) ``` ``` { .no-copy .output } -Device: cuda:0 -DType: torch.bfloat16 --- dataset input --- { "messages": [ @@ -311,12 +299,10 @@ What is the reimbursement limit for travel meals? call:search_knowledge_base{query:travel meal reimbursement limit policy} ``` -## Before fine-tune - -The output below shows that the out-of-the-box capabilities may not be good enough for this use case. +The following function measures how often a model calls the expected tool on the test set. You will use it before and after training. ```python { .notebook-cell } -def check_success_rate(): +def check_success_rate(model): success_count = 0 for idx, item in enumerate(dataset['test']): messages = [ @@ -347,8 +333,61 @@ def check_success_rate(): print(f" -> ❌ wrong (hallucinated '{other_tool}')") print(f"Success : {success_count} / {len(dataset['test'])}") +``` + +## Training + +Before you can start your training, you need to define the hyperparameters you want to use in a `SFTConfig` instance. + +```python { .notebook-cell } +from trl import SFTConfig -check_success_rate() +args = SFTConfig( + output_dir=checkpoint_dir, # directory to save and repository id + max_length=512, # max sequence length for model and packing of the dataset + packing=False, # Groups multiple samples in the dataset into a single sequence + num_train_epochs=8, # number of training epochs + per_device_train_batch_size=4, # batch size per device during training + gradient_checkpointing=False, # the model is small, so skip gradient checkpointing for speed + optim="adamw_torch_fused", # use fused adamw optimizer + logging_steps=1, # log every step + #save_strategy="epoch", # save checkpoint every epoch + eval_strategy="epoch", # evaluate checkpoint every epoch + learning_rate=learning_rate, # learning rate + lr_scheduler_type="constant", # use constant learning rate scheduler + push_to_hub=True, # push model to hub + report_to="tensorboard", # report metrics to tensorboard +) +``` + +You now have every building block you need to create your `SFTTrainer` to start the training of your model. + +```python { .notebook-cell } +from trl import SFTTrainer + +# Create Trainer object +trainer = SFTTrainer( + model=base_model, + args=args, + train_dataset=dataset['train'], + eval_dataset=dataset['test'], +) +``` + +``` { .no-copy .output } +Tokenizing train dataset: 0%| | 0/20 [00:00 ❌ wrong (hallucinated '{other_tool}')\")\n", "\n", - " print(f\"Success : {success_count} / {len(dataset['test'])}\")\n", - "\n", - "check_success_rate()\n" + " print(f\"Success : {success_count} / {len(dataset['test'])}\")\n" ] }, { @@ -378,22 +364,18 @@ "source": [ "from trl import SFTConfig\n", "\n", - "torch_dtype = model.dtype\n", - "\n", "args = SFTConfig(\n", " output_dir=checkpoint_dir, # directory to save and repository id\n", " max_length=512, # max sequence length for model and packing of the dataset\n", " packing=False, # Groups multiple samples in the dataset into a single sequence\n", " num_train_epochs=8, # number of training epochs\n", " per_device_train_batch_size=4, # batch size per device during training\n", - " gradient_checkpointing=False, # Caching is incompatible with gradient checkpointing\n", + " gradient_checkpointing=False, # the model is small, so skip gradient checkpointing for speed\n", " optim=\"adamw_torch_fused\", # use fused adamw optimizer\n", " logging_steps=1, # log every step\n", " #save_strategy=\"epoch\", # save checkpoint every epoch\n", " eval_strategy=\"epoch\", # evaluate checkpoint every epoch\n", " learning_rate=learning_rate, # learning rate\n", - " fp16=True if torch_dtype == torch.float16 else False, # use float16 precision\n", - " bf16=True if torch_dtype == torch.bfloat16 else False, # use bfloat16 precision\n", " lr_scheduler_type=\"constant\", # use constant learning rate scheduler\n", " push_to_hub=True, # push model to hub\n", " report_to=\"tensorboard\", # report metrics to tensorboard\n", @@ -419,14 +401,33 @@ "\n", "# Create Trainer object\n", "trainer = SFTTrainer(\n", - " model=model,\n", + " model=base_model,\n", " args=args,\n", " train_dataset=dataset['train'],\n", " eval_dataset=dataset['test'],\n", - " processing_class=tokenizer,\n", ")\n" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Before fine-tune\n", + "\n", + "The trainer has loaded the base model. The output below shows that its out-of-the-box capabilities may not be good enough for this use case." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "language": "python" + }, + "outputs": [], + "source": [ + "check_success_rate(trainer.model)\n" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -505,7 +506,7 @@ }, "outputs": [], "source": [ - "check_success_rate()\n" + "check_success_rate(trainer.model)\n" ] }, {