import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import BatchEncoding, MarianTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import is_sentencepiece_available, is_tf_available, is_torch_available


if is_sentencepiece_available():
					from transformers.models.marian.tokenization_marian import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


__UpperCamelCase	: List[str]    		=  get_tests_dir('fixtures/test_sentencepiece.model')

__UpperCamelCase	: Tuple    		=  {'target_lang': 'fi', 'source_lang': 'en'}
__UpperCamelCase	: Dict    		=  '>>zh<<'
__UpperCamelCase	: int    		=  'Helsinki-NLP/'

if is_torch_available():
					__UpperCamelCase	: List[str]    		=  'pt'
elif is_tf_available():
					__UpperCamelCase	: Union[str, Any]    		=  'tf'
else:
					__UpperCamelCase	: Union[str, Any]    		=  'jax'





@require_sentencepiece
class       lowercase__		(     UpperCamelCase_			,							unittest.TestCase):
						UpperCamelCase_  =	MarianTokenizer
						UpperCamelCase_  =	False
						UpperCamelCase_  =	True


						def  __A ( self			:       Optional[int]		):





												'''simple docstring'''





												super().setUp()
												SCREAMING_SNAKE_CASE							:					Dict           =							['''</s>''', '''<unk>''', '''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''', '''\u0120''', '''<pad>''']
												SCREAMING_SNAKE_CASE							:					List[Any]           =							dict(zip(UpperCamelCase__					,	range(len(UpperCamelCase__		)		)		)		)
												SCREAMING_SNAKE_CASE							:					Optional[int]           =							Path(self.tmpdirname		)
												save_json(UpperCamelCase__					,	save_dir / VOCAB_FILES_NAMES['''vocab''']		)
												save_json(UpperCamelCase__					,	save_dir / VOCAB_FILES_NAMES['''tokenizer_config_file''']		)
												if not (save_dir / VOCAB_FILES_NAMES["source_spm"]).exists():
																		copyfile(UpperCamelCase__					,	save_dir / VOCAB_FILES_NAMES['''source_spm''']		)
																		copyfile(UpperCamelCase__					,	save_dir / VOCAB_FILES_NAMES['''target_spm''']		)

												SCREAMING_SNAKE_CASE							:					int           =							MarianTokenizer.from_pretrained(self.tmpdirname		)
												tokenizer.save_pretrained(self.tmpdirname		)


						def  __A ( self			:       Any					,	**UpperCamelCase__			:       List[str]		):





												'''simple docstring'''





												return MarianTokenizer.from_pretrained(self.tmpdirname					,	**UpperCamelCase__		)


						def  __A ( self			:       Dict					,	UpperCamelCase__			:       Optional[int]		):





												'''simple docstring'''





												return (
												    "This is a test",
												    "This is a test",
												)


						def  __A ( self			:       Any		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							'''</s>'''
												SCREAMING_SNAKE_CASE							:					Any           =							0

												self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase__		)					,	UpperCamelCase__		)
												self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase__		)					,	UpperCamelCase__		)


						def  __A ( self			:       Tuple		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Any           =							list(self.get_tokenizer().get_vocab().keys()		)

												self.assertEqual(vocab_keys[0]					,	'''</s>'''		)
												self.assertEqual(vocab_keys[1]					,	'''<unk>'''		)
												self.assertEqual(vocab_keys[-1]					,	'''<pad>'''		)
												self.assertEqual(len(UpperCamelCase__		)					,	9		)


						def  __A ( self			:       str		):





												'''simple docstring'''





												self.assertEqual(self.get_tokenizer().vocab_size					,	9		)


						def  __A ( self			:       Union[str, Any]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					int           =							MarianTokenizer.from_pretrained(f"""{ORG_NAME}opus-mt-en-de"""		)
												SCREAMING_SNAKE_CASE							:					List[str]           =							en_de_tokenizer(['''I am a small frog''']					,	return_tensors=UpperCamelCase__		)
												self.assertIsInstance(UpperCamelCase__					,	UpperCamelCase__		)
												SCREAMING_SNAKE_CASE							:					int           =							[38, 121, 14, 697, 3_8848, 0]
												self.assertListEqual(UpperCamelCase__					,	batch.input_ids[0]		)

												SCREAMING_SNAKE_CASE							:					Dict           =							tempfile.mkdtemp()
												en_de_tokenizer.save_pretrained(UpperCamelCase__		)
												SCREAMING_SNAKE_CASE							:					str           =							[x.name for x in Path(UpperCamelCase__		).glob('''*'''		)]
												self.assertIn('''source.spm'''					,	UpperCamelCase__		)
												MarianTokenizer.from_pretrained(UpperCamelCase__		)


						def  __A ( self			:       Union[str, Any]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Tuple           =							self.get_tokenizer()

												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							tok(
												    ['''I am a small frog''' * 1000, '''I am a small frog''']					,	padding=UpperCamelCase__					,	truncation=UpperCamelCase__					,	return_tensors=UpperCamelCase__		)
												self.assertIsInstance(UpperCamelCase__					,	UpperCamelCase__		)
												self.assertEqual(batch.input_ids.shape					,	(2, 512)		)


						def  __A ( self			:       Any		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Tuple           =							self.get_tokenizer()
												SCREAMING_SNAKE_CASE							:					int           =							tok(['''I am a tiny frog''', '''I am a small frog''']					,	padding=UpperCamelCase__					,	return_tensors=UpperCamelCase__		)
												self.assertIsInstance(UpperCamelCase__					,	UpperCamelCase__		)
												self.assertEqual(batch_smaller.input_ids.shape					,	(2, 10)		)


						@slow
						def  __A ( self			:       Optional[Any]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Any           =							{'''input_ids''': [[4_3495, 462, 20, 4_2164, 1369, 52, 464, 132, 1703, 492, 13, 7491, 3_8999, 6, 8, 464, 132, 1703, 492, 13, 4669, 3_7867, 13, 7525, 27, 1593, 988, 13, 3_3972, 7029, 6, 20, 8251, 383, 2, 270, 5866, 3788, 2, 2353, 8251, 1_2338, 2, 1_3958, 387, 2, 3629, 6953, 188, 2900, 2, 1_3958, 8011, 1_1501, 23, 8460, 4073, 3_4009, 20, 435, 1_1439, 27, 8, 8460, 4073, 6004, 20, 9988, 375, 27, 33, 266, 1945, 1076, 1350, 3_7867, 3288, 5, 577, 1076, 4374, 8, 5082, 5, 2_6453, 257, 556, 403, 2, 242, 132, 383, 316, 492, 8, 1_0767, 6, 316, 304, 4239, 3, 0], [148, 1_5722, 19, 1839, 12, 1350, 13, 2_2327, 5082, 5418, 4_7567, 3_5938, 59, 318, 1_9552, 108, 2183, 54, 1_4976, 4835, 32, 547, 1114, 8, 315, 2417, 5, 92, 1_9088, 3, 0, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100], [36, 6395, 1_2570, 3_9147, 1_1597, 6, 266, 4, 4_5405, 7296, 3, 0, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
												# fmt: on

												self.tokenizer_integration_test_util(
												    expected_encoding=UpperCamelCase__					,	model_name='''Helsinki-NLP/opus-mt-en-de'''					,	revision='''1a8c2263da11e68e50938f97e10cd57820bd504c'''					,	decode_kwargs={'''use_source_tokenizer''': True}					,	)



						def  __A ( self			:       List[str]		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							MarianTokenizer.from_pretrained('''hf-internal-testing/test-marian-two-vocabs'''		)

												SCREAMING_SNAKE_CASE							:					List[Any]           =							'''Tämä on testi'''
												SCREAMING_SNAKE_CASE							:					Dict           =							'''This is a test'''

												SCREAMING_SNAKE_CASE							:					List[Any]           =							[76, 7, 2047, 2]
												SCREAMING_SNAKE_CASE							:					int           =							[69, 12, 11, 940, 2]

												SCREAMING_SNAKE_CASE							:					List[Any]           =							tokenizer(UpperCamelCase__		).input_ids
												self.assertListEqual(UpperCamelCase__					,	UpperCamelCase__		)

												SCREAMING_SNAKE_CASE							:					Tuple           =							tokenizer(text_target=UpperCamelCase__		).input_ids
												self.assertListEqual(UpperCamelCase__					,	UpperCamelCase__		)

												SCREAMING_SNAKE_CASE							:					Any           =							tokenizer.decode(UpperCamelCase__					,	skip_special_tokens=UpperCamelCase__		)
												self.assertEqual(UpperCamelCase__					,	UpperCamelCase__		)






# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse

from ...utils.dataclasses import (
    ComputeEnvironment,
    DistributedType,
    DynamoBackend,
    PrecisionType,
    SageMakerDistributedType,
)
from ..menu import BulletMenu


__UpperCamelCase	: Dict    		=  [
    'EAGER',
    'AOT_EAGER',
    'INDUCTOR',
    'NVFUSER',
    'AOT_NVFUSER',
    'AOT_CUDAGRAPHS',
    'OFI',
    'FX2TRT',
    'ONNXRT',
    'IPEX',
]
def    A			(					_lowercase	,						_lowercase=None	,						_lowercase=None	,						_lowercase=None		):
						SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							True
						while ask_again:
												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							input(_lowercase		)
												try:
																		if default is not None and len(_lowercase		) == 0:
																								return default
																		return convert_value(_lowercase		) if convert_value is not None else result
												except Exception:
																		if error_message is not None:
																								print(_lowercase		)
def    A			(					_lowercase	,						_lowercase=[]	,						_lowercase=None	,						_lowercase=0		):
						SCREAMING_SNAKE_CASE							:					Dict           =							BulletMenu(_lowercase	,						_lowercase		)
						SCREAMING_SNAKE_CASE							:					str           =							menu.run(default_choice=_lowercase		)
						return convert_value(_lowercase		) if convert_value is not None else result
def    A			(					_lowercase		):
						SCREAMING_SNAKE_CASE							:					Dict           =							int(_lowercase		)
						return ComputeEnvironment(['''LOCAL_MACHINE''', '''AMAZON_SAGEMAKER'''][value]		)
def    A			(					_lowercase		):
						SCREAMING_SNAKE_CASE							:					Any           =							int(_lowercase		)
						return DistributedType(['''NO''', '''MULTI_CPU''', '''MULTI_XPU''', '''MULTI_GPU''', '''MULTI_NPU''', '''TPU'''][value]		)
def    A			(					_lowercase		):
						SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							int(_lowercase		)
						return DynamoBackend(DYNAMO_BACKENDS[value]		).value
def    A			(					_lowercase		):
						SCREAMING_SNAKE_CASE							:					Union[str, Any]           =							int(_lowercase		)
						return PrecisionType(['''no''', '''fp16''', '''bf16''', '''fp8'''][value]		)
def    A			(					_lowercase		):
						SCREAMING_SNAKE_CASE							:					Dict           =							int(_lowercase		)
						return SageMakerDistributedType(['''NO''', '''DATA_PARALLEL''', '''MODEL_PARALLEL'''][value]		)
def    A			(					_lowercase		):
						return {"yes": True, "no": False}[value.lower()]





class       lowercase__		(     argparse.RawDescriptionHelpFormatter):


						def  __A ( self			:       List[str]					,	UpperCamelCase__			:       List[str]					,	UpperCamelCase__			:       Dict					,	UpperCamelCase__			:       Dict					,	UpperCamelCase__			:       Dict		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:					str           =							super()._format_usage(UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__		)
												SCREAMING_SNAKE_CASE							:					Optional[Any]           =							usage.replace('''<command> [<args>] '''					,	''''''		)
												return usage





