


'''simple docstring'''



import itertools
import json
import os
import unittest

from transformers import AddedToken, RobertaTokenizer, RobertaTokenizerFast
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class  UpperCamelCase__(     lowerCAmelCase		,       unittest.TestCase ):
				__magic_name__       :   Optional[Any]							= RobertaTokenizer
				__magic_name__       :   Union[str, Any]							= RobertaTokenizerFast
				__magic_name__       :   Union[str, Any]							= True
				__magic_name__       :   Tuple							= {"cls_token": "<s>"}




				def   a__(	self    :				int   )->  Optional[int]:

										"""simple docstring"""




										super().setUp()

										# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
										UpperCAmelCase         =			[
										    '''l''',
										    '''o''',
										    '''w''',
										    '''e''',
										    '''r''',
										    '''s''',
										    '''t''',
										    '''i''',
										    '''d''',
										    '''n''',
										    '''\u0120''',
										    '''\u0120l''',
										    '''\u0120n''',
										    '''\u0120lo''',
										    '''\u0120low''',
										    '''er''',
										    '''\u0120lowest''',
										    '''\u0120newer''',
										    '''\u0120wider''',
										    '''<unk>''',
										]
										UpperCAmelCase         =			dict(zip(lowerCAmelCase   ,					range(len(lowerCAmelCase   )   )   )   )
										UpperCAmelCase         =			['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
										UpperCAmelCase         =			{'''unk_token''': '''<unk>'''}

										UpperCAmelCase         =			os.path.join(self.tmpdirname   ,					VOCAB_FILES_NAMES['''vocab_file''']   )
										UpperCAmelCase         =			os.path.join(self.tmpdirname   ,					VOCAB_FILES_NAMES['''merges_file''']   )
										with open(self.vocab_file   ,					'''w'''   ,					encoding='''utf-8'''   ) as fp:
																fp.write(json.dumps(lowerCAmelCase   ) + '''\n'''   )
										with open(self.merges_file   ,					'''w'''   ,					encoding='''utf-8'''   ) as fp:
																fp.write('''\n'''.join(lowerCAmelCase   )   )




				def   a__(	self    :				Tuple   ,					**lowerCAmelCase    :				Optional[Any]   )->  Union[str, Any]:

										"""simple docstring"""




										kwargs.update(self.special_tokens_map   )
										return self.tokenizer_class.from_pretrained(self.tmpdirname   ,					**lowerCAmelCase   )




				def   a__(	self    :				Union[str, Any]   ,					**lowerCAmelCase    :				List[Any]   )->  List[str]:

										"""simple docstring"""




										kwargs.update(self.special_tokens_map   )
										return RobertaTokenizerFast.from_pretrained(self.tmpdirname   ,					**lowerCAmelCase   )




				def   a__(	self    :				str   ,					lowerCAmelCase    :				Union[str, Any]   )->  List[str]:

										"""simple docstring"""




										UpperCAmelCase         =			'''lower newer'''
										UpperCAmelCase         =			'''lower newer'''
										return input_text, output_text




				def   a__(	self    :				Tuple   )->  Optional[int]:

										"""simple docstring"""




										UpperCAmelCase         =			self.tokenizer_class(self.vocab_file   ,					self.merges_file   ,					**self.special_tokens_map   )
										UpperCAmelCase         =			'''lower newer'''
										UpperCAmelCase         =			['''l''', '''o''', '''w''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
										UpperCAmelCase         =			tokenizer.tokenize(lowerCAmelCase   )  # , add_prefix_space=True)
										self.assertListEqual(lowerCAmelCase   ,					lowerCAmelCase   )

										UpperCAmelCase         =			tokens + [tokenizer.unk_token]
										UpperCAmelCase         =			[0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
										self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase   )   ,					lowerCAmelCase   )




				def   a__(	self    :				Tuple   )->  str:

										"""simple docstring"""




										UpperCAmelCase         =			self.get_tokenizer()

										self.assertListEqual(tokenizer.encode('''Hello world!'''   ,					add_special_tokens=lowerCAmelCase   )   ,					[0, 31414, 232, 328, 2]   )
										self.assertListEqual(
										    tokenizer.encode('''Hello world! cécé herlolip 418'''   ,					add_special_tokens=lowerCAmelCase   )   ,					[0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]   ,					)




				@slow
				def   a__(	self    :				List[Any]   )->  Optional[Any]:

										"""simple docstring"""




										UpperCAmelCase         =			self.tokenizer_class.from_pretrained('''roberta-base'''   )

										UpperCAmelCase         =			tokenizer.encode('''sequence builders'''   ,					add_special_tokens=lowerCAmelCase   )
										UpperCAmelCase         =			tokenizer.encode('''multi-sequence build'''   ,					add_special_tokens=lowerCAmelCase   )

										UpperCAmelCase         =			tokenizer.encode(
										    '''sequence builders'''   ,					add_special_tokens=lowerCAmelCase   ,					add_prefix_space=lowerCAmelCase   )
										UpperCAmelCase         =			tokenizer.encode(
										    '''sequence builders'''   ,					'''multi-sequence build'''   ,					add_special_tokens=lowerCAmelCase   ,					add_prefix_space=lowerCAmelCase   )

										UpperCAmelCase         =			tokenizer.build_inputs_with_special_tokens(lowerCAmelCase   )
										UpperCAmelCase         =			tokenizer.build_inputs_with_special_tokens(lowerCAmelCase   ,					lowerCAmelCase   )

										assert encoded_sentence == encoded_text_from_decode
										assert encoded_pair == encoded_pair_from_decode




				def   a__(	self    :				Union[str, Any]   )->  Tuple:

										"""simple docstring"""




										UpperCAmelCase         =			self.get_tokenizer()

										UpperCAmelCase         =			'''Encode this sequence.'''
										UpperCAmelCase         =			tokenizer.byte_encoder[''' '''.encode('''utf-8'''   )[0]]

										# Testing encoder arguments
										UpperCAmelCase         =			tokenizer.encode(lowerCAmelCase   ,					add_special_tokens=lowerCAmelCase   ,					add_prefix_space=lowerCAmelCase   )
										UpperCAmelCase         =			tokenizer.convert_ids_to_tokens(encoded[0]   )[0]
										self.assertNotEqual(lowerCAmelCase   ,					lowerCAmelCase   )

										UpperCAmelCase         =			tokenizer.encode(lowerCAmelCase   ,					add_special_tokens=lowerCAmelCase   ,					add_prefix_space=lowerCAmelCase   )
										UpperCAmelCase         =			tokenizer.convert_ids_to_tokens(encoded[0]   )[0]
										self.assertEqual(lowerCAmelCase   ,					lowerCAmelCase   )

										tokenizer.add_special_tokens({'''bos_token''': '''<s>'''}   )
										UpperCAmelCase         =			tokenizer.encode(lowerCAmelCase   ,					add_special_tokens=lowerCAmelCase   )
										UpperCAmelCase         =			tokenizer.convert_ids_to_tokens(encoded[1]   )[0]
										self.assertNotEqual(lowerCAmelCase   ,					lowerCAmelCase   )

										# Testing spaces after special tokens
										UpperCAmelCase         =			'''<mask>'''
										tokenizer.add_special_tokens(
										    {'''mask_token''': AddedToken(lowerCAmelCase   ,					lstrip=lowerCAmelCase   ,					rstrip=lowerCAmelCase   )}   )  # mask token has a left space
										UpperCAmelCase         =			tokenizer.convert_tokens_to_ids(lowerCAmelCase   )

										UpperCAmelCase         =			'''Encode <mask> sequence'''
										UpperCAmelCase         =			'''Encode <mask>sequence'''

										UpperCAmelCase         =			tokenizer.encode(lowerCAmelCase   )
										UpperCAmelCase         =			encoded.index(lowerCAmelCase   )
										UpperCAmelCase         =			tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]   )[0]
										self.assertEqual(lowerCAmelCase   ,					lowerCAmelCase   )

										UpperCAmelCase         =			tokenizer.encode(lowerCAmelCase   )
										UpperCAmelCase         =			encoded.index(lowerCAmelCase   )
										UpperCAmelCase         =			tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]   )[0]
										self.assertNotEqual(lowerCAmelCase   ,					lowerCAmelCase   )




				def   a__(	self    :				List[Any]   )->  int:

										"""simple docstring"""




										pass




				def   a__(	self    :				Optional[Any]   )->  Union[str, Any]:

										"""simple docstring"""




										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""   ):
																						UpperCAmelCase         =			self.rust_tokenizer_class.from_pretrained(lowerCAmelCase   ,					**lowerCAmelCase   )
																						UpperCAmelCase         =			self.tokenizer_class.from_pretrained(lowerCAmelCase   ,					**lowerCAmelCase   )
																						UpperCAmelCase         =			'''A, <mask> AllenNLP sentence.'''
																						UpperCAmelCase         =			tokenizer_r.encode_plus(lowerCAmelCase   ,					add_special_tokens=lowerCAmelCase   ,					return_token_type_ids=lowerCAmelCase   )
																						UpperCAmelCase         =			tokenizer_p.encode_plus(lowerCAmelCase   ,					add_special_tokens=lowerCAmelCase   ,					return_token_type_ids=lowerCAmelCase   )

																						# token_type_ids should put 0 everywhere
																						self.assertEqual(sum(tokens_r['''token_type_ids''']   )   ,					sum(tokens_p['''token_type_ids''']   )   )

																						# attention_mask should put 1 everywhere, so sum over length should be 1
																						self.assertEqual(
																						    sum(tokens_r['''attention_mask''']   ) / len(tokens_r['''attention_mask''']   )   ,					sum(tokens_p['''attention_mask''']   ) / len(tokens_p['''attention_mask''']   )   ,					)

																						UpperCAmelCase         =			tokenizer_r.convert_ids_to_tokens(tokens_r['''input_ids''']   )
																						UpperCAmelCase         =			tokenizer_p.convert_ids_to_tokens(tokens_p['''input_ids''']   )

																						# Rust correctly handles the space before the mask while python doesnt
																						self.assertSequenceEqual(tokens_p['''input_ids''']   ,					[0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]   )
																						self.assertSequenceEqual(tokens_r['''input_ids''']   ,					[0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]   )

																						self.assertSequenceEqual(
																						    lowerCAmelCase   ,					['''<s>''', '''A''', ''',''', '''<mask>''', '''ĠAllen''', '''N''', '''LP''', '''Ġsentence''', '''.''', '''</s>''']   )
																						self.assertSequenceEqual(
																						    lowerCAmelCase   ,					['''<s>''', '''A''', ''',''', '''<mask>''', '''ĠAllen''', '''N''', '''LP''', '''Ġsentence''', '''.''', '''</s>''']   )




				def   a__(	self    :				Dict   )->  Dict:

										"""simple docstring"""




										for trim_offsets, add_prefix_space in itertools.product([True, False]   ,					repeat=2   ):
																UpperCAmelCase         =			self.rust_tokenizer_class.from_pretrained(
																    self.tmpdirname   ,					use_fast=lowerCAmelCase   ,					add_prefix_space=lowerCAmelCase   ,					trim_offsets=lowerCAmelCase   )

																UpperCAmelCase         =			json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()   )
																UpperCAmelCase         =			json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()   )

																self.assertEqual(pre_tokenizer_state['''add_prefix_space''']   ,					lowerCAmelCase   )

																self.assertEqual(post_processor_state['''add_prefix_space''']   ,					lowerCAmelCase   )
																self.assertEqual(post_processor_state['''trim_offsets''']   ,					lowerCAmelCase   )





				def   a__(	self    :				Dict   )->  Union[str, Any]:

										"""simple docstring"""




										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""   ):
																						UpperCAmelCase         =			'''hello'''  # `hello` is a token in the vocabulary of `pretrained_name`
																						UpperCAmelCase         =			F"""{text_of_1_token} {text_of_1_token}"""

																						UpperCAmelCase         =			self.rust_tokenizer_class.from_pretrained(
																						    lowerCAmelCase   ,					use_fast=lowerCAmelCase   ,					add_prefix_space=lowerCAmelCase   ,					trim_offsets=lowerCAmelCase   )
																						UpperCAmelCase         =			tokenizer_r(lowerCAmelCase   ,					return_offsets_mapping=lowerCAmelCase   ,					add_special_tokens=lowerCAmelCase   )
																						self.assertEqual(encoding.offset_mapping[0]   ,					(0, len(lowerCAmelCase   ))   )
																						self.assertEqual(
																						    encoding.offset_mapping[1]   ,					(len(lowerCAmelCase   ) + 1, len(lowerCAmelCase   ) + 1 + len(lowerCAmelCase   ))   ,					)

																						UpperCAmelCase         =			self.rust_tokenizer_class.from_pretrained(
																						    lowerCAmelCase   ,					use_fast=lowerCAmelCase   ,					add_prefix_space=lowerCAmelCase   ,					trim_offsets=lowerCAmelCase   )
																						UpperCAmelCase         =			tokenizer_r(lowerCAmelCase   ,					return_offsets_mapping=lowerCAmelCase   ,					add_special_tokens=lowerCAmelCase   )
																						self.assertEqual(encoding.offset_mapping[0]   ,					(0, len(lowerCAmelCase   ))   )
																						self.assertEqual(
																						    encoding.offset_mapping[1]   ,					(len(lowerCAmelCase   ) + 1, len(lowerCAmelCase   ) + 1 + len(lowerCAmelCase   ))   ,					)

																						UpperCAmelCase         =			self.rust_tokenizer_class.from_pretrained(
																						    lowerCAmelCase   ,					use_fast=lowerCAmelCase   ,					add_prefix_space=lowerCAmelCase   ,					trim_offsets=lowerCAmelCase   )
																						UpperCAmelCase         =			tokenizer_r(lowerCAmelCase   ,					return_offsets_mapping=lowerCAmelCase   ,					add_special_tokens=lowerCAmelCase   )
																						self.assertEqual(encoding.offset_mapping[0]   ,					(0, len(lowerCAmelCase   ))   )
																						self.assertEqual(
																						    encoding.offset_mapping[1]   ,					(len(lowerCAmelCase   ), len(lowerCAmelCase   ) + 1 + len(lowerCAmelCase   ))   ,					)

																						UpperCAmelCase         =			self.rust_tokenizer_class.from_pretrained(
																						    lowerCAmelCase   ,					use_fast=lowerCAmelCase   ,					add_prefix_space=lowerCAmelCase   ,					trim_offsets=lowerCAmelCase   )
																						UpperCAmelCase         =			tokenizer_r(lowerCAmelCase   ,					return_offsets_mapping=lowerCAmelCase   ,					add_special_tokens=lowerCAmelCase   )
																						self.assertEqual(encoding.offset_mapping[0]   ,					(0, len(lowerCAmelCase   ))   )
																						self.assertEqual(
																						    encoding.offset_mapping[1]   ,					(len(lowerCAmelCase   ), len(lowerCAmelCase   ) + 1 + len(lowerCAmelCase   ))   ,					)

																						UpperCAmelCase         =			F""" {text}"""

																						# tokenizer_r = self.rust_tokenizer_class.from_pretrained(
																						#     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
																						# )
																						# encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
																						# self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
																						# self.assertEqual(
																						#     encoding.offset_mapping[1],
																						#     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
																						# )

																						UpperCAmelCase         =			self.rust_tokenizer_class.from_pretrained(
																						    lowerCAmelCase   ,					use_fast=lowerCAmelCase   ,					add_prefix_space=lowerCAmelCase   ,					trim_offsets=lowerCAmelCase   )
																						UpperCAmelCase         =			tokenizer_r(lowerCAmelCase   ,					return_offsets_mapping=lowerCAmelCase   ,					add_special_tokens=lowerCAmelCase   )
																						self.assertEqual(encoding.offset_mapping[0]   ,					(1, 1 + len(lowerCAmelCase   ))   )
																						self.assertEqual(
																						    encoding.offset_mapping[1]   ,					(1 + len(lowerCAmelCase   ) + 1, 1 + len(lowerCAmelCase   ) + 1 + len(lowerCAmelCase   ))   ,					)

																						UpperCAmelCase         =			self.rust_tokenizer_class.from_pretrained(
																						    lowerCAmelCase   ,					use_fast=lowerCAmelCase   ,					add_prefix_space=lowerCAmelCase   ,					trim_offsets=lowerCAmelCase   )
																						UpperCAmelCase         =			tokenizer_r(lowerCAmelCase   ,					return_offsets_mapping=lowerCAmelCase   ,					add_special_tokens=lowerCAmelCase   )
																						self.assertEqual(encoding.offset_mapping[0]   ,					(0, 1 + len(lowerCAmelCase   ))   )
																						self.assertEqual(
																						    encoding.offset_mapping[1]   ,					(1 + len(lowerCAmelCase   ), 1 + len(lowerCAmelCase   ) + 1 + len(lowerCAmelCase   ))   ,					)

																						UpperCAmelCase         =			self.rust_tokenizer_class.from_pretrained(
																						    lowerCAmelCase   ,					use_fast=lowerCAmelCase   ,					add_prefix_space=lowerCAmelCase   ,					trim_offsets=lowerCAmelCase   )
																						UpperCAmelCase         =			tokenizer_r(lowerCAmelCase   ,					return_offsets_mapping=lowerCAmelCase   ,					add_special_tokens=lowerCAmelCase   )
																						self.assertEqual(encoding.offset_mapping[0]   ,					(0, 1 + len(lowerCAmelCase   ))   )
																						self.assertEqual(
																						    encoding.offset_mapping[1]   ,					(1 + len(lowerCAmelCase   ), 1 + len(lowerCAmelCase   ) + 1 + len(lowerCAmelCase   ))   ,					)









'''simple docstring'''



# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


_lowercase    :    int			     =  {
    """configuration_xmod""": [
        """XMOD_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """XmodConfig""",
        """XmodOnnxConfig""",
    ],
}

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_lowercase    :    str			     =  [
	    """XMOD_PRETRAINED_MODEL_ARCHIVE_LIST""",
	    """XmodForCausalLM""",
	    """XmodForMaskedLM""",
	    """XmodForMultipleChoice""",
	    """XmodForQuestionAnswering""",
	    """XmodForSequenceClassification""",
	    """XmodForTokenClassification""",
	    """XmodModel""",
	    """XmodPreTrainedModel""",
	]

if TYPE_CHECKING:
	from .configuration_xmod import XMOD_PRETRAINED_CONFIG_ARCHIVE_MAP, XmodConfig, XmodOnnxConfig

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_xmod import (
		    XMOD_PRETRAINED_MODEL_ARCHIVE_LIST,
		    XmodForCausalLM,
		    XmodForMaskedLM,
		    XmodForMultipleChoice,
		    XmodForQuestionAnswering,
		    XmodForSequenceClassification,
		    XmodForTokenClassification,
		    XmodModel,
		    XmodPreTrainedModel,
		)

else:
	import sys

	_lowercase    :    List[Any]			     =  _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)





