We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Tutorial script missing eval_batch_size
Following this tutorial https://huggingface.co/blog/dpo-trl, I ran into out of memory issues with the DPO script. The problem was that the script missed copying per_device_eval_batch_size to Training Arguments [here](https://github.com/huggingface/trl/blob/main/examples/research_projects/stack_llama_2/scripts/dpo_llama2.py#L166C12-L166C12). This makes it use the default size of 8 for eval_batch_size instead of 1 as set in this script. The below addition fixes the issue.

```
training_args = TrainingArguments(
        per_device_train_batch_size=script_args.per_device_train_batch_size,
-->        per_device_eval_batch_size=script_args.per_device_eval_batch_size,
```

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `examples/research_projects/stack_llama_2/scripts/dpo_llama2.py`
Content:
```
1 # 0. imports
2 import os
3 from dataclasses import dataclass, field
4 from typing import Dict, Optional
5 
6 import torch
7 from datasets import Dataset, load_dataset
8 from peft import AutoPeftModelForCausalLM, LoraConfig
9 from transformers import AutoTokenizer, HfArgumentParser, TrainingArguments
10 
11 from trl import DPOTrainer
12 
13 
14 # Define and parse arguments.
15 @dataclass
16 class ScriptArguments:
17     """
18     The arguments for the DPO training script.
19     """
20 
21     # data parameters
22     beta: Optional[float] = field(default=0.1, metadata={"help": "the beta parameter for DPO loss"})
23 
24     # training parameters
25     model_name_or_path: Optional[str] = field(
26         default="../sft/results/final_checkpoint",
27         metadata={"help": "the location of the SFT model name or path"},
28     )
29     learning_rate: Optional[float] = field(default=5e-4, metadata={"help": "optimizer learning rate"})
30     lr_scheduler_type: Optional[str] = field(default="cosine", metadata={"help": "the lr scheduler type"})
31     warmup_steps: Optional[int] = field(default=100, metadata={"help": "the number of warmup steps"})
32     weight_decay: Optional[float] = field(default=0.05, metadata={"help": "the weight decay"})
33     optimizer_type: Optional[str] = field(default="paged_adamw_32bit", metadata={"help": "the optimizer type"})
34 
35     per_device_train_batch_size: Optional[int] = field(default=4, metadata={"help": "train batch size per device"})
36     per_device_eval_batch_size: Optional[int] = field(default=1, metadata={"help": "eval batch size per device"})
37     gradient_accumulation_steps: Optional[int] = field(
38         default=4, metadata={"help": "the number of gradient accumulation steps"}
39     )
40     gradient_checkpointing: Optional[bool] = field(
41         default=True, metadata={"help": "whether to use gradient checkpointing"}
42     )
43 
44     lora_alpha: Optional[float] = field(default=16, metadata={"help": "the lora alpha parameter"})
45     lora_dropout: Optional[float] = field(default=0.05, metadata={"help": "the lora dropout parameter"})
46     lora_r: Optional[int] = field(default=8, metadata={"help": "the lora r parameter"})
47 
48     max_prompt_length: Optional[int] = field(default=512, metadata={"help": "the maximum prompt length"})
49     max_length: Optional[int] = field(default=1024, metadata={"help": "the maximum sequence length"})
50     max_steps: Optional[int] = field(default=1000, metadata={"help": "max number of training steps"})
51     logging_steps: Optional[int] = field(default=10, metadata={"help": "the logging frequency"})
52     save_steps: Optional[int] = field(default=100, metadata={"help": "the saving frequency"})
53     eval_steps: Optional[int] = field(default=100, metadata={"help": "the evaluation frequency"})
54 
55     output_dir: Optional[str] = field(default="./results", metadata={"help": "the output directory"})
56     log_freq: Optional[int] = field(default=1, metadata={"help": "the logging frequency"})
57 
58     # instrumentation
59     sanity_check: Optional[bool] = field(default=False, metadata={"help": "only train on 1000 samples"})
60     report_to: Optional[str] = field(
61         default="wandb",
62         metadata={
63             "help": 'The list of integrations to report the results and logs to. Supported platforms are `"azure_ml"`,'
64             '`"comet_ml"`, `"mlflow"`, `"neptune"`, `"tensorboard"`,`"clearml"` and `"wandb"`. '
65             'Use `"all"` to report to all integrations installed, `"none"` for no integrations.'
66         },
67     )
68     # debug argument for distributed training
69     ignore_bias_buffers: Optional[bool] = field(
70         default=False,
71         metadata={
72             "help": "fix for DDP issues with LM bias/mask buffers - invalid scalar type,`inplace operation. See"
73             "https://github.com/huggingface/transformers/issues/22482#issuecomment-1595790992"
74         },
75     )
76 
77 
78 def get_stack_exchange_paired(
79     data_dir: str = "data/rl",
80     sanity_check: bool = False,
81     cache_dir: str = None,
82     num_proc=24,
83 ) -> Dataset:
84     """Load the stack-exchange-paired dataset from Hugging Face and convert it to the necessary format.
85 
86     The dataset is converted to a dictionary with the following structure:
87     {
88         'prompt': List[str],
89         'chosen': List[str],
90         'rejected': List[str],
91     }
92 
93     Prompts are structured as follows:
94       "Question: " + <prompt> + "\n\nAnswer: "
95     """
96     dataset = load_dataset(
97         "lvwerra/stack-exchange-paired",
98         split="train",
99         cache_dir=cache_dir,
100         data_dir=data_dir,
101     )
102     original_columns = dataset.column_names
103 
104     if sanity_check:
105         dataset = dataset.select(range(min(len(dataset), 1000)))
106 
107     def return_prompt_and_responses(samples) -> Dict[str, str]:
108         return {
109             "prompt": ["Question: " + question + "\n\nAnswer: " for question in samples["question"]],
110             "chosen": samples["response_j"],
111             "rejected": samples["response_k"],
112         }
113 
114     return dataset.map(
115         return_prompt_and_responses,
116         batched=True,
117         num_proc=num_proc,
118         remove_columns=original_columns,
119     )
120 
121 
122 if __name__ == "__main__":
123     parser = HfArgumentParser(ScriptArguments)
124     script_args = parser.parse_args_into_dataclasses()[0]
125 
126     # 1. load a pretrained model
127     model = AutoPeftModelForCausalLM.from_pretrained(
128         script_args.model_name_or_path,
129         low_cpu_mem_usage=True,
130         torch_dtype=torch.float16,
131         load_in_4bit=True,
132     )
133     model.config.use_cache = False
134 
135     if script_args.ignore_bias_buffers:
136         # torch distributed hack
137         model._ddp_params_and_buffers_to_ignore = [
138             name for name, buffer in model.named_buffers() if buffer.dtype == torch.bool
139         ]
140 
141     model_ref = AutoPeftModelForCausalLM.from_pretrained(
142         script_args.model_name_or_path,
143         low_cpu_mem_usage=True,
144         torch_dtype=torch.float16,
145         load_in_4bit=True,
146     )
147     tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf")
148     tokenizer.pad_token = tokenizer.eos_token
149 
150     # 2. Load the Stack-exchange paired dataset
151     train_dataset = get_stack_exchange_paired(data_dir="data/rl", sanity_check=script_args.sanity_check)
152     train_dataset = train_dataset.filter(
153         lambda x: len(x["prompt"]) + len(x["chosen"]) <= script_args.max_length
154         and len(x["prompt"]) + len(x["rejected"]) <= script_args.max_length
155     )
156 
157     # 3. Load evaluation dataset
158     eval_dataset = get_stack_exchange_paired(data_dir="data/evaluation", sanity_check=True)
159     eval_dataset = eval_dataset.filter(
160         lambda x: len(x["prompt"]) + len(x["chosen"]) <= script_args.max_length
161         and len(x["prompt"]) + len(x["rejected"]) <= script_args.max_length
162     )
163 
164     # 4. initialize training arguments:
165     training_args = TrainingArguments(
166         per_device_train_batch_size=script_args.per_device_train_batch_size,
167         max_steps=script_args.max_steps,
168         logging_steps=script_args.logging_steps,
169         save_steps=script_args.save_steps,
170         gradient_accumulation_steps=script_args.gradient_accumulation_steps,
171         gradient_checkpointing=script_args.gradient_checkpointing,
172         learning_rate=script_args.learning_rate,
173         evaluation_strategy="steps",
174         eval_steps=script_args.eval_steps,
175         output_dir=script_args.output_dir,
176         report_to=script_args.report_to,
177         lr_scheduler_type=script_args.lr_scheduler_type,
178         warmup_steps=script_args.warmup_steps,
179         optim=script_args.optimizer_type,
180         bf16=True,
181         remove_unused_columns=False,
182         run_name="dpo_llama2",
183     )
184 
185     peft_config = LoraConfig(
186         r=script_args.lora_r,
187         lora_alpha=script_args.lora_alpha,
188         lora_dropout=script_args.lora_dropout,
189         target_modules=[
190             "q_proj",
191             "v_proj",
192             "k_proj",
193             "out_proj",
194             "fc_in",
195             "fc_out",
196             "wte",
197         ],
198         bias="none",
199         task_type="CAUSAL_LM",
200     )
201 
202     # 5. initialize the DPO trainer
203     dpo_trainer = DPOTrainer(
204         model,
205         model_ref,
206         args=training_args,
207         beta=script_args.beta,
208         train_dataset=train_dataset,
209         eval_dataset=eval_dataset,
210         tokenizer=tokenizer,
211         peft_config=peft_config,
212         max_prompt_length=script_args.max_prompt_length,
213         max_length=script_args.max_length,
214     )
215 
216     # 6. train
217     dpo_trainer.train()
218     dpo_trainer.save_model(script_args.output_dir)
219 
220     # 7. save
221     output_dir = os.path.join(script_args.output_dir, "final_checkpoint")
222     dpo_trainer.model.save_pretrained(output_dir)
223
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/examples/research_projects/stack_llama_2/scripts/dpo_llama2.py b/examples/research_projects/stack_llama_2/scripts/dpo_llama2.py
--- a/examples/research_projects/stack_llama_2/scripts/dpo_llama2.py
+++ b/examples/research_projects/stack_llama_2/scripts/dpo_llama2.py
@@ -164,6 +164,7 @@
     # 4. initialize training arguments:
     training_args = TrainingArguments(
         per_device_train_batch_size=script_args.per_device_train_batch_size,
+        per_device_eval_batch_size=script_args.per_device_eval_batch_size,
         max_steps=script_args.max_steps,
         logging_steps=script_args.logging_steps,
         save_steps=script_args.save_steps,

{"golden_diff": "diff --git a/examples/research_projects/stack_llama_2/scripts/dpo_llama2.py b/examples/research_projects/stack_llama_2/scripts/dpo_llama2.py\n--- a/examples/research_projects/stack_llama_2/scripts/dpo_llama2.py\n+++ b/examples/research_projects/stack_llama_2/scripts/dpo_llama2.py\n@@ -164,6 +164,7 @@\n     # 4. initialize training arguments:\n     training_args = TrainingArguments(\n         per_device_train_batch_size=script_args.per_device_train_batch_size,\n+        per_device_eval_batch_size=script_args.per_device_eval_batch_size,\n         max_steps=script_args.max_steps,\n         logging_steps=script_args.logging_steps,\n         save_steps=script_args.save_steps,\n", "issue": "Tutorial script missing eval_batch_size\nFollowing this tutorial https://huggingface.co/blog/dpo-trl, I ran into out of memory issues with the DPO script. The problem was that the script missed copying per_device_eval_batch_size to Training Arguments [here](https://github.com/huggingface/trl/blob/main/examples/research_projects/stack_llama_2/scripts/dpo_llama2.py#L166C12-L166C12). This makes it use the default size of 8 for eval_batch_size instead of 1 as set in this script. The below addition fixes the issue.\r\n\r\n```\r\ntraining_args = TrainingArguments(\r\n        per_device_train_batch_size=script_args.per_device_train_batch_size,\r\n-->        per_device_eval_batch_size=script_args.per_device_eval_batch_size,\r\n```\n", "before_files": [{"content": "# 0. imports\nimport os\nfrom dataclasses import dataclass, field\nfrom typing import Dict, Optional\n\nimport torch\nfrom datasets import Dataset, load_dataset\nfrom peft import AutoPeftModelForCausalLM, LoraConfig\nfrom transformers import AutoTokenizer, HfArgumentParser, TrainingArguments\n\nfrom trl import DPOTrainer\n\n\n# Define and parse arguments.\n@dataclass\nclass ScriptArguments:\n    \"\"\"\n    The arguments for the DPO training script.\n    \"\"\"\n\n    # data parameters\n    beta: Optional[float] = field(default=0.1, metadata={\"help\": \"the beta parameter for DPO loss\"})\n\n    # training parameters\n    model_name_or_path: Optional[str] = field(\n        default=\"../sft/results/final_checkpoint\",\n        metadata={\"help\": \"the location of the SFT model name or path\"},\n    )\n    learning_rate: Optional[float] = field(default=5e-4, metadata={\"help\": \"optimizer learning rate\"})\n    lr_scheduler_type: Optional[str] = field(default=\"cosine\", metadata={\"help\": \"the lr scheduler type\"})\n    warmup_steps: Optional[int] = field(default=100, metadata={\"help\": \"the number of warmup steps\"})\n    weight_decay: Optional[float] = field(default=0.05, metadata={\"help\": \"the weight decay\"})\n    optimizer_type: Optional[str] = field(default=\"paged_adamw_32bit\", metadata={\"help\": \"the optimizer type\"})\n\n    per_device_train_batch_size: Optional[int] = field(default=4, metadata={\"help\": \"train batch size per device\"})\n    per_device_eval_batch_size: Optional[int] = field(default=1, metadata={\"help\": \"eval batch size per device\"})\n    gradient_accumulation_steps: Optional[int] = field(\n        default=4, metadata={\"help\": \"the number of gradient accumulation steps\"}\n    )\n    gradient_checkpointing: Optional[bool] = field(\n        default=True, metadata={\"help\": \"whether to use gradient checkpointing\"}\n    )\n\n    lora_alpha: Optional[float] = field(default=16, metadata={\"help\": \"the lora alpha parameter\"})\n    lora_dropout: Optional[float] = field(default=0.05, metadata={\"help\": \"the lora dropout parameter\"})\n    lora_r: Optional[int] = field(default=8, metadata={\"help\": \"the lora r parameter\"})\n\n    max_prompt_length: Optional[int] = field(default=512, metadata={\"help\": \"the maximum prompt length\"})\n    max_length: Optional[int] = field(default=1024, metadata={\"help\": \"the maximum sequence length\"})\n    max_steps: Optional[int] = field(default=1000, metadata={\"help\": \"max number of training steps\"})\n    logging_steps: Optional[int] = field(default=10, metadata={\"help\": \"the logging frequency\"})\n    save_steps: Optional[int] = field(default=100, metadata={\"help\": \"the saving frequency\"})\n    eval_steps: Optional[int] = field(default=100, metadata={\"help\": \"the evaluation frequency\"})\n\n    output_dir: Optional[str] = field(default=\"./results\", metadata={\"help\": \"the output directory\"})\n    log_freq: Optional[int] = field(default=1, metadata={\"help\": \"the logging frequency\"})\n\n    # instrumentation\n    sanity_check: Optional[bool] = field(default=False, metadata={\"help\": \"only train on 1000 samples\"})\n    report_to: Optional[str] = field(\n        default=\"wandb\",\n        metadata={\n            \"help\": 'The list of integrations to report the results and logs to. Supported platforms are `\"azure_ml\"`,'\n            '`\"comet_ml\"`, `\"mlflow\"`, `\"neptune\"`, `\"tensorboard\"`,`\"clearml\"` and `\"wandb\"`. '\n            'Use `\"all\"` to report to all integrations installed, `\"none\"` for no integrations.'\n        },\n    )\n    # debug argument for distributed training\n    ignore_bias_buffers: Optional[bool] = field(\n        default=False,\n        metadata={\n            \"help\": \"fix for DDP issues with LM bias/mask buffers - invalid scalar type,`inplace operation. See\"\n            \"https://github.com/huggingface/transformers/issues/22482#issuecomment-1595790992\"\n        },\n    )\n\n\ndef get_stack_exchange_paired(\n    data_dir: str = \"data/rl\",\n    sanity_check: bool = False,\n    cache_dir: str = None,\n    num_proc=24,\n) -> Dataset:\n    \"\"\"Load the stack-exchange-paired dataset from Hugging Face and convert it to the necessary format.\n\n    The dataset is converted to a dictionary with the following structure:\n    {\n        'prompt': List[str],\n        'chosen': List[str],\n        'rejected': List[str],\n    }\n\n    Prompts are structured as follows:\n      \"Question: \" + <prompt> + \"\\n\\nAnswer: \"\n    \"\"\"\n    dataset = load_dataset(\n        \"lvwerra/stack-exchange-paired\",\n        split=\"train\",\n        cache_dir=cache_dir,\n        data_dir=data_dir,\n    )\n    original_columns = dataset.column_names\n\n    if sanity_check:\n        dataset = dataset.select(range(min(len(dataset), 1000)))\n\n    def return_prompt_and_responses(samples) -> Dict[str, str]:\n        return {\n            \"prompt\": [\"Question: \" + question + \"\\n\\nAnswer: \" for question in samples[\"question\"]],\n            \"chosen\": samples[\"response_j\"],\n            \"rejected\": samples[\"response_k\"],\n        }\n\n    return dataset.map(\n        return_prompt_and_responses,\n        batched=True,\n        num_proc=num_proc,\n        remove_columns=original_columns,\n    )\n\n\nif __name__ == \"__main__\":\n    parser = HfArgumentParser(ScriptArguments)\n    script_args = parser.parse_args_into_dataclasses()[0]\n\n    # 1. load a pretrained model\n    model = AutoPeftModelForCausalLM.from_pretrained(\n        script_args.model_name_or_path,\n        low_cpu_mem_usage=True,\n        torch_dtype=torch.float16,\n        load_in_4bit=True,\n    )\n    model.config.use_cache = False\n\n    if script_args.ignore_bias_buffers:\n        # torch distributed hack\n        model._ddp_params_and_buffers_to_ignore = [\n            name for name, buffer in model.named_buffers() if buffer.dtype == torch.bool\n        ]\n\n    model_ref = AutoPeftModelForCausalLM.from_pretrained(\n        script_args.model_name_or_path,\n        low_cpu_mem_usage=True,\n        torch_dtype=torch.float16,\n        load_in_4bit=True,\n    )\n    tokenizer = AutoTokenizer.from_pretrained(\"meta-llama/Llama-2-7b-hf\")\n    tokenizer.pad_token = tokenizer.eos_token\n\n    # 2. Load the Stack-exchange paired dataset\n    train_dataset = get_stack_exchange_paired(data_dir=\"data/rl\", sanity_check=script_args.sanity_check)\n    train_dataset = train_dataset.filter(\n        lambda x: len(x[\"prompt\"]) + len(x[\"chosen\"]) <= script_args.max_length\n        and len(x[\"prompt\"]) + len(x[\"rejected\"]) <= script_args.max_length\n    )\n\n    # 3. Load evaluation dataset\n    eval_dataset = get_stack_exchange_paired(data_dir=\"data/evaluation\", sanity_check=True)\n    eval_dataset = eval_dataset.filter(\n        lambda x: len(x[\"prompt\"]) + len(x[\"chosen\"]) <= script_args.max_length\n        and len(x[\"prompt\"]) + len(x[\"rejected\"]) <= script_args.max_length\n    )\n\n    # 4. initialize training arguments:\n    training_args = TrainingArguments(\n        per_device_train_batch_size=script_args.per_device_train_batch_size,\n        max_steps=script_args.max_steps,\n        logging_steps=script_args.logging_steps,\n        save_steps=script_args.save_steps,\n        gradient_accumulation_steps=script_args.gradient_accumulation_steps,\n        gradient_checkpointing=script_args.gradient_checkpointing,\n        learning_rate=script_args.learning_rate,\n        evaluation_strategy=\"steps\",\n        eval_steps=script_args.eval_steps,\n        output_dir=script_args.output_dir,\n        report_to=script_args.report_to,\n        lr_scheduler_type=script_args.lr_scheduler_type,\n        warmup_steps=script_args.warmup_steps,\n        optim=script_args.optimizer_type,\n        bf16=True,\n        remove_unused_columns=False,\n        run_name=\"dpo_llama2\",\n    )\n\n    peft_config = LoraConfig(\n        r=script_args.lora_r,\n        lora_alpha=script_args.lora_alpha,\n        lora_dropout=script_args.lora_dropout,\n        target_modules=[\n            \"q_proj\",\n            \"v_proj\",\n            \"k_proj\",\n            \"out_proj\",\n            \"fc_in\",\n            \"fc_out\",\n            \"wte\",\n        ],\n        bias=\"none\",\n        task_type=\"CAUSAL_LM\",\n    )\n\n    # 5. initialize the DPO trainer\n    dpo_trainer = DPOTrainer(\n        model,\n        model_ref,\n        args=training_args,\n        beta=script_args.beta,\n        train_dataset=train_dataset,\n        eval_dataset=eval_dataset,\n        tokenizer=tokenizer,\n        peft_config=peft_config,\n        max_prompt_length=script_args.max_prompt_length,\n        max_length=script_args.max_length,\n    )\n\n    # 6. train\n    dpo_trainer.train()\n    dpo_trainer.save_model(script_args.output_dir)\n\n    # 7. save\n    output_dir = os.path.join(script_args.output_dir, \"final_checkpoint\")\n    dpo_trainer.model.save_pretrained(output_dir)\n", "path": "examples/research_projects/stack_llama_2/scripts/dpo_llama2.py"}], "after_files": [{"content": "# 0. imports\nimport os\nfrom dataclasses import dataclass, field\nfrom typing import Dict, Optional\n\nimport torch\nfrom datasets import Dataset, load_dataset\nfrom peft import AutoPeftModelForCausalLM, LoraConfig\nfrom transformers import AutoTokenizer, HfArgumentParser, TrainingArguments\n\nfrom trl import DPOTrainer\n\n\n# Define and parse arguments.\n@dataclass\nclass ScriptArguments:\n    \"\"\"\n    The arguments for the DPO training script.\n    \"\"\"\n\n    # data parameters\n    beta: Optional[float] = field(default=0.1, metadata={\"help\": \"the beta parameter for DPO loss\"})\n\n    # training parameters\n    model_name_or_path: Optional[str] = field(\n        default=\"../sft/results/final_checkpoint\",\n        metadata={\"help\": \"the location of the SFT model name or path\"},\n    )\n    learning_rate: Optional[float] = field(default=5e-4, metadata={\"help\": \"optimizer learning rate\"})\n    lr_scheduler_type: Optional[str] = field(default=\"cosine\", metadata={\"help\": \"the lr scheduler type\"})\n    warmup_steps: Optional[int] = field(default=100, metadata={\"help\": \"the number of warmup steps\"})\n    weight_decay: Optional[float] = field(default=0.05, metadata={\"help\": \"the weight decay\"})\n    optimizer_type: Optional[str] = field(default=\"paged_adamw_32bit\", metadata={\"help\": \"the optimizer type\"})\n\n    per_device_train_batch_size: Optional[int] = field(default=4, metadata={\"help\": \"train batch size per device\"})\n    per_device_eval_batch_size: Optional[int] = field(default=1, metadata={\"help\": \"eval batch size per device\"})\n    gradient_accumulation_steps: Optional[int] = field(\n        default=4, metadata={\"help\": \"the number of gradient accumulation steps\"}\n    )\n    gradient_checkpointing: Optional[bool] = field(\n        default=True, metadata={\"help\": \"whether to use gradient checkpointing\"}\n    )\n\n    lora_alpha: Optional[float] = field(default=16, metadata={\"help\": \"the lora alpha parameter\"})\n    lora_dropout: Optional[float] = field(default=0.05, metadata={\"help\": \"the lora dropout parameter\"})\n    lora_r: Optional[int] = field(default=8, metadata={\"help\": \"the lora r parameter\"})\n\n    max_prompt_length: Optional[int] = field(default=512, metadata={\"help\": \"the maximum prompt length\"})\n    max_length: Optional[int] = field(default=1024, metadata={\"help\": \"the maximum sequence length\"})\n    max_steps: Optional[int] = field(default=1000, metadata={\"help\": \"max number of training steps\"})\n    logging_steps: Optional[int] = field(default=10, metadata={\"help\": \"the logging frequency\"})\n    save_steps: Optional[int] = field(default=100, metadata={\"help\": \"the saving frequency\"})\n    eval_steps: Optional[int] = field(default=100, metadata={\"help\": \"the evaluation frequency\"})\n\n    output_dir: Optional[str] = field(default=\"./results\", metadata={\"help\": \"the output directory\"})\n    log_freq: Optional[int] = field(default=1, metadata={\"help\": \"the logging frequency\"})\n\n    # instrumentation\n    sanity_check: Optional[bool] = field(default=False, metadata={\"help\": \"only train on 1000 samples\"})\n    report_to: Optional[str] = field(\n        default=\"wandb\",\n        metadata={\n            \"help\": 'The list of integrations to report the results and logs to. Supported platforms are `\"azure_ml\"`,'\n            '`\"comet_ml\"`, `\"mlflow\"`, `\"neptune\"`, `\"tensorboard\"`,`\"clearml\"` and `\"wandb\"`. '\n            'Use `\"all\"` to report to all integrations installed, `\"none\"` for no integrations.'\n        },\n    )\n    # debug argument for distributed training\n    ignore_bias_buffers: Optional[bool] = field(\n        default=False,\n        metadata={\n            \"help\": \"fix for DDP issues with LM bias/mask buffers - invalid scalar type,`inplace operation. See\"\n            \"https://github.com/huggingface/transformers/issues/22482#issuecomment-1595790992\"\n        },\n    )\n\n\ndef get_stack_exchange_paired(\n    data_dir: str = \"data/rl\",\n    sanity_check: bool = False,\n    cache_dir: str = None,\n    num_proc=24,\n) -> Dataset:\n    \"\"\"Load the stack-exchange-paired dataset from Hugging Face and convert it to the necessary format.\n\n    The dataset is converted to a dictionary with the following structure:\n    {\n        'prompt': List[str],\n        'chosen': List[str],\n        'rejected': List[str],\n    }\n\n    Prompts are structured as follows:\n      \"Question: \" + <prompt> + \"\\n\\nAnswer: \"\n    \"\"\"\n    dataset = load_dataset(\n        \"lvwerra/stack-exchange-paired\",\n        split=\"train\",\n        cache_dir=cache_dir,\n        data_dir=data_dir,\n    )\n    original_columns = dataset.column_names\n\n    if sanity_check:\n        dataset = dataset.select(range(min(len(dataset), 1000)))\n\n    def return_prompt_and_responses(samples) -> Dict[str, str]:\n        return {\n            \"prompt\": [\"Question: \" + question + \"\\n\\nAnswer: \" for question in samples[\"question\"]],\n            \"chosen\": samples[\"response_j\"],\n            \"rejected\": samples[\"response_k\"],\n        }\n\n    return dataset.map(\n        return_prompt_and_responses,\n        batched=True,\n        num_proc=num_proc,\n        remove_columns=original_columns,\n    )\n\n\nif __name__ == \"__main__\":\n    parser = HfArgumentParser(ScriptArguments)\n    script_args = parser.parse_args_into_dataclasses()[0]\n\n    # 1. load a pretrained model\n    model = AutoPeftModelForCausalLM.from_pretrained(\n        script_args.model_name_or_path,\n        low_cpu_mem_usage=True,\n        torch_dtype=torch.float16,\n        load_in_4bit=True,\n    )\n    model.config.use_cache = False\n\n    if script_args.ignore_bias_buffers:\n        # torch distributed hack\n        model._ddp_params_and_buffers_to_ignore = [\n            name for name, buffer in model.named_buffers() if buffer.dtype == torch.bool\n        ]\n\n    model_ref = AutoPeftModelForCausalLM.from_pretrained(\n        script_args.model_name_or_path,\n        low_cpu_mem_usage=True,\n        torch_dtype=torch.float16,\n        load_in_4bit=True,\n    )\n    tokenizer = AutoTokenizer.from_pretrained(\"meta-llama/Llama-2-7b-hf\")\n    tokenizer.pad_token = tokenizer.eos_token\n\n    # 2. Load the Stack-exchange paired dataset\n    train_dataset = get_stack_exchange_paired(data_dir=\"data/rl\", sanity_check=script_args.sanity_check)\n    train_dataset = train_dataset.filter(\n        lambda x: len(x[\"prompt\"]) + len(x[\"chosen\"]) <= script_args.max_length\n        and len(x[\"prompt\"]) + len(x[\"rejected\"]) <= script_args.max_length\n    )\n\n    # 3. Load evaluation dataset\n    eval_dataset = get_stack_exchange_paired(data_dir=\"data/evaluation\", sanity_check=True)\n    eval_dataset = eval_dataset.filter(\n        lambda x: len(x[\"prompt\"]) + len(x[\"chosen\"]) <= script_args.max_length\n        and len(x[\"prompt\"]) + len(x[\"rejected\"]) <= script_args.max_length\n    )\n\n    # 4. initialize training arguments:\n    training_args = TrainingArguments(\n        per_device_train_batch_size=script_args.per_device_train_batch_size,\n        per_device_eval_batch_size=script_args.per_device_eval_batch_size,\n        max_steps=script_args.max_steps,\n        logging_steps=script_args.logging_steps,\n        save_steps=script_args.save_steps,\n        gradient_accumulation_steps=script_args.gradient_accumulation_steps,\n        gradient_checkpointing=script_args.gradient_checkpointing,\n        learning_rate=script_args.learning_rate,\n        evaluation_strategy=\"steps\",\n        eval_steps=script_args.eval_steps,\n        output_dir=script_args.output_dir,\n        report_to=script_args.report_to,\n        lr_scheduler_type=script_args.lr_scheduler_type,\n        warmup_steps=script_args.warmup_steps,\n        optim=script_args.optimizer_type,\n        bf16=True,\n        remove_unused_columns=False,\n        run_name=\"dpo_llama2\",\n    )\n\n    peft_config = LoraConfig(\n        r=script_args.lora_r,\n        lora_alpha=script_args.lora_alpha,\n        lora_dropout=script_args.lora_dropout,\n        target_modules=[\n            \"q_proj\",\n            \"v_proj\",\n            \"k_proj\",\n            \"out_proj\",\n            \"fc_in\",\n            \"fc_out\",\n            \"wte\",\n        ],\n        bias=\"none\",\n        task_type=\"CAUSAL_LM\",\n    )\n\n    # 5. initialize the DPO trainer\n    dpo_trainer = DPOTrainer(\n        model,\n        model_ref,\n        args=training_args,\n        beta=script_args.beta,\n        train_dataset=train_dataset,\n        eval_dataset=eval_dataset,\n        tokenizer=tokenizer,\n        peft_config=peft_config,\n        max_prompt_length=script_args.max_prompt_length,\n        max_length=script_args.max_length,\n    )\n\n    # 6. train\n    dpo_trainer.train()\n    dpo_trainer.save_model(script_args.output_dir)\n\n    # 7. save\n    output_dir = os.path.join(script_args.output_dir, \"final_checkpoint\")\n    dpo_trainer.model.save_pretrained(output_dir)\n", "path": "examples/research_projects/stack_llama_2/scripts/dpo_llama2.py"}]}