






import json
import logging
import os
import sys
from pathlib import Path

import finetune_rag

from transformers.file_utils import is_apex_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    require_ray,
    require_torch_gpu,
    require_torch_multi_gpu,
)


logging.basicConfig(level=logging.DEBUG)
__A      =				logging.getLogger()

__A      =				logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)





class 						SCREAMING_SNAKE_CASE				( snake_case					):





							"""simple docstring"""

							def       __A (  self:   Any		,     __A:   Union[str, Any]   )    ->				Optional[Any]:
												os.makedirs(__A		,     exist_ok=__A   )
												_A								= {'''source''': '''What is love ?''', '''target''': '''life'''}
												_A								= {'''train''': 12, '''val''': 2, '''test''': 2}
												for split in ["train", "test", "val"]:
																	for field in ["source", "target"]:
																						_A								= '''\n'''.join([contents[field]] * n_lines[split]   )
																						with open(os.path.join(__A		,     f"""{split}.{field}"""   )		,     '''w'''   ) as f:
																											f.write(__A   )

							def       __A (  self:   Dict		,     __A:   int		,     __A:   str = "pytorch"   )    ->				Union[str, Any]:
												_A								= self.get_auto_remove_tmp_dir()
												_A								= os.path.join(__A		,     '''output'''   )
												_A								= os.path.join(__A		,     '''data'''   )
												self._create_dummy_data(data_dir=__A   )

												_A								= f"""
                --data_dir {data_dir} \
                --output_dir {output_dir} \
                --model_name_or_path facebook/rag-sequence-base \
                --model_type rag_sequence \
                --do_train \
                --do_predict \
                --n_val -1 \
                --val_check_interval 1.0 \
                --train_batch_size 2 \
                --eval_batch_size 1 \
                --max_source_length 25 \
                --max_target_length 25 \
                --val_max_target_length 25 \
                --test_max_target_length 25 \
                --label_smoothing 0.1 \
                --dropout 0.1 \
                --attention_dropout 0.1 \
                --weight_decay 0.001 \
                --adam_epsilon 1e-08 \
                --max_grad_norm 0.1 \
                --lr_scheduler polynomial \
                --learning_rate 3e-04 \
                --num_train_epochs 1 \
                --warmup_steps 4 \
                --gradient_accumulation_steps 1 \
                --distributed-port 8787 \
                --use_dummy_dataset 1 \
                --distributed_retriever {distributed_retriever} \
            """.split()

												if gpus > 0:
																	testargs.append(f"""--gpus={gpus}"""   )
																	if is_apex_available():
																						testargs.append('''--fp16'''   )
												else:
																	testargs.append('''--gpus=0'''   )
																	testargs.append('''--distributed_backend=ddp_cpu'''   )
																	testargs.append('''--num_processes=2'''   )

												_A								= [sys.executable, str(Path(finetune_rag.__file__   ).resolve()   )] + testargs
												execute_subprocess_async(__A		,     env=self.get_env()   )

												_A								= os.path.join(__A		,     '''metrics.json'''   )
												with open(__A   ) as f:
																	_A								= json.load(__A   )
												return result

							@require_torch_gpu
							def       __A (  self:   Any   )    ->				Any:
												_A								= self._run_finetune(gpus=1   )
												self.assertGreaterEqual(result['''test'''][0]['''test_avg_em''']		,     0.2   )

							@require_torch_multi_gpu
							def       __A (  self:   Optional[Any]   )    ->				Optional[int]:
												_A								= self._run_finetune(gpus=2   )
												self.assertGreaterEqual(result['''test'''][0]['''test_avg_em''']		,     0.2   )

							@require_torch_gpu
							@require_ray
							def       __A (  self:   str   )    ->				Tuple:
												_A								= self._run_finetune(gpus=1		,     distributed_retriever='''ray'''   )
												self.assertGreaterEqual(result['''test'''][0]['''test_avg_em''']		,     0.2   )



							@require_torch_multi_gpu
							@require_ray
							def       __A (  self:   List[Any]   )    ->				Optional[int]:
												_A								= self._run_finetune(gpus=1		,     distributed_retriever='''ray'''   )
												self.assertGreaterEqual(result['''test'''][0]['''test_avg_em''']		,     0.2   )











import json
import os
import unittest

from transformers.models.blenderbot_small.tokenization_blenderbot_small import (
    VOCAB_FILES_NAMES,
    BlenderbotSmallTokenizer,
)

from ...test_tokenization_common import TokenizerTesterMixin





class 						SCREAMING_SNAKE_CASE				( snake_case  ,	unittest.TestCase					):





							"""simple docstring"""

							A_						  =					BlenderbotSmallTokenizer
							A_						  =					False

							def       __A (  self:   List[str]   )    ->				int:
												super().setUp()

												_A								= ['''__start__''', '''adapt''', '''act''', '''ap@@''', '''te''', '''__end__''', '''__unk__''']
												_A								= dict(zip(__A		,     range(len(__A   )   )   )   )

												_A								= ['''#version: 0.2''', '''a p''', '''t e</w>''', '''ap t</w>''', '''a d''', '''ad apt</w>''', '''a c''', '''ac t</w>''', '''''']
												_A								= {'''unk_token''': '''__unk__''', '''bos_token''': '''__start__''', '''eos_token''': '''__end__'''}

												_A								= os.path.join(self.tmpdirname		,     VOCAB_FILES_NAMES['''vocab_file''']   )
												_A								= os.path.join(self.tmpdirname		,     VOCAB_FILES_NAMES['''merges_file''']   )
												with open(self.vocab_file		,     '''w'''		,     encoding='''utf-8'''   ) as fp:
																	fp.write(json.dumps(__A   ) + '''\n'''   )
												with open(self.merges_file		,     '''w'''		,     encoding='''utf-8'''   ) as fp:
																	fp.write('''\n'''.join(__A   )   )

							def       __A (  self:   str		,     **__A:   Optional[Any]   )    ->				Dict:
												kwargs.update(self.special_tokens_map   )
												return BlenderbotSmallTokenizer.from_pretrained(self.tmpdirname		,     **__A   )

							def       __A (  self:   str		,     __A:   List[str]   )    ->				int:
												_A								= '''adapt act apte'''
												_A								= '''adapt act apte'''
												return input_text, output_text

							def       __A (  self:   Union[str, Any]   )    ->				Any:
												_A								= BlenderbotSmallTokenizer(self.vocab_file		,     self.merges_file		,     **self.special_tokens_map   )
												_A								= '''adapt act apte'''
												_A								= ['''adapt''', '''act''', '''ap@@''', '''te''']
												_A								= tokenizer.tokenize(__A   )
												self.assertListEqual(__A		,     __A   )

												_A								= [tokenizer.bos_token] + tokens + [tokenizer.eos_token]

												_A								= [0, 1, 2, 3, 4, 5]
												self.assertListEqual(tokenizer.convert_tokens_to_ids(__A   )		,     __A   )

							def       __A (  self:   Any   )    ->				List[str]:
												_A								= BlenderbotSmallTokenizer.from_pretrained('''facebook/blenderbot-90M'''   )
												assert tok('''sam'''   ).input_ids == [13_84]
												_A								= '''I am a small frog.'''
												_A								= tok([src_text]		,     padding=__A		,     truncation=__A   )['''input_ids''']
												_A								= tok.batch_decode(__A		,     skip_special_tokens=__A		,     clean_up_tokenization_spaces=__A   )[0]
												assert src_text != decoded  # I wish it did!
												assert decoded == "i am a small frog ."



							def       __A (  self:   Any   )    ->				int:
												_A								= BlenderbotSmallTokenizer.from_pretrained('''facebook/blenderbot-90M'''   )
												_A								= '''I am a small frog .'''
												_A								= '''.'''
												_A								= tok(__A   )['''input_ids''']
												_A								= tok(__A   )['''input_ids''']

												assert encoded[-1] == encoded_dot[0]



