






"""simple docstring"""
import itertools
import json
import os
import unittest

from transformers import AddedToken, RobertaTokenizer, RobertaTokenizerFast
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class     UpperCamelCase_ (__A       ,		unittest.TestCase  ):
	__magic_name__			   =       RobertaTokenizer
	__magic_name__			   =       RobertaTokenizerFast
	__magic_name__			   =       True
	__magic_name__			   =       {'''cls_token''': '''<s>'''}





	def   _SCREAMING_SNAKE_CASE   (   self      :				Tuple		)		->    Dict:
				super().setUp()

				# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
				UpperCAmelCase_  :				str      =      [
				    "l",
				    "o",
				    "w",
				    "e",
				    "r",
				    "s",
				    "t",
				    "i",
				    "d",
				    "n",
				    "\u0120",
				    "\u0120l",
				    "\u0120n",
				    "\u0120lo",
				    "\u0120low",
				    "er",
				    "\u0120lowest",
				    "\u0120newer",
				    "\u0120wider",
				    "<unk>",
				]
				UpperCAmelCase_  :				Optional[int]      =      dict(zip(lowerCAmelCase_			,			range(len(lowerCAmelCase_		)		)		)		)
				UpperCAmelCase_  :				str      =      ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
				UpperCAmelCase_  :				str      =      {"unk_token": "<unk>"}

				UpperCAmelCase_  :				Union[str, Any]      =      os.path.join(self.tmpdirname			,			VOCAB_FILES_NAMES["vocab_file"]		)
				UpperCAmelCase_  :				Any      =      os.path.join(self.tmpdirname			,			VOCAB_FILES_NAMES["merges_file"]		)
				with open(self.vocab_file			,			"w"			,			encoding="utf-8"		) as fp:
							fp.write(json.dumps(lowerCAmelCase_		) + "\n"		)
				with open(self.merges_file			,			"w"			,			encoding="utf-8"		) as fp:
							fp.write("\n".join(lowerCAmelCase_		)		)





	def   _SCREAMING_SNAKE_CASE   (   self      :				Tuple			,			**lowerCAmelCase_      :				List[str]		)		->    Optional[int]:
				kwargs.update(self.special_tokens_map		)
				return self.tokenizer_class.from_pretrained(self.tmpdirname			,			**lowerCAmelCase_		)





	def   _SCREAMING_SNAKE_CASE   (   self      :				Optional[Any]			,			**lowerCAmelCase_      :				List[str]		)		->    Optional[Any]:
				kwargs.update(self.special_tokens_map		)
				return RobertaTokenizerFast.from_pretrained(self.tmpdirname			,			**lowerCAmelCase_		)





	def   _SCREAMING_SNAKE_CASE   (   self      :				Any			,			lowerCAmelCase_      :				int		)		->    Optional[int]:
				UpperCAmelCase_  :				Optional[int]      =      "lower newer"
				UpperCAmelCase_  :				Tuple      =      "lower newer"
				return input_text, output_text





	def   _SCREAMING_SNAKE_CASE   (   self      :				List[str]		)		->    Any:
				UpperCAmelCase_  :				Dict      =      self.tokenizer_class(self.vocab_file			,			self.merges_file			,			**self.special_tokens_map		)
				UpperCAmelCase_  :				Tuple      =      "lower newer"
				UpperCAmelCase_  :				str      =      ["l", "o", "w", "er", "\u0120", "n", "e", "w", "er"]
				UpperCAmelCase_  :				Any      =      tokenizer.tokenize(lowerCAmelCase_		)  # , add_prefix_space=True)
				self.assertListEqual(lowerCAmelCase_			,			lowerCAmelCase_		)

				UpperCAmelCase_  :				Optional[int]      =      tokens + [tokenizer.unk_token]
				UpperCAmelCase_  :				Optional[Any]      =      [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
				self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_		)			,			lowerCAmelCase_		)





	def   _SCREAMING_SNAKE_CASE   (   self      :				str		)		->    Optional[int]:
				UpperCAmelCase_  :				Tuple      =      self.get_tokenizer()

				self.assertListEqual(tokenizer.encode("Hello world!"			,			add_special_tokens=lowerCAmelCase_		)			,			[0, 31_414, 232, 328, 2]		)
				self.assertListEqual(
				    tokenizer.encode("Hello world! cécé herlolip 418"			,			add_special_tokens=lowerCAmelCase_		)			,			[0, 31_414, 232, 328, 740, 1_140, 12_695, 69, 46_078, 1_588, 2]			,			)





	@slow
	def   _SCREAMING_SNAKE_CASE   (   self      :				str		)		->    List[Any]:
				UpperCAmelCase_  :				Any      =      self.tokenizer_class.from_pretrained("roberta-base"		)

				UpperCAmelCase_  :				List[str]      =      tokenizer.encode("sequence builders"			,			add_special_tokens=lowerCAmelCase_		)
				UpperCAmelCase_  :				List[str]      =      tokenizer.encode("multi-sequence build"			,			add_special_tokens=lowerCAmelCase_		)

				UpperCAmelCase_  :				Union[str, Any]      =      tokenizer.encode(
				    "sequence builders"			,			add_special_tokens=lowerCAmelCase_			,			add_prefix_space=lowerCAmelCase_		)
				UpperCAmelCase_  :				Any      =      tokenizer.encode(
				    "sequence builders"			,			"multi-sequence build"			,			add_special_tokens=lowerCAmelCase_			,			add_prefix_space=lowerCAmelCase_		)

				UpperCAmelCase_  :				str      =      tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_		)
				UpperCAmelCase_  :				int      =      tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_			,			lowerCAmelCase_		)

				assert encoded_sentence == encoded_text_from_decode
				assert encoded_pair == encoded_pair_from_decode





	def   _SCREAMING_SNAKE_CASE   (   self      :				Any		)		->    Union[str, Any]:
				UpperCAmelCase_  :				Dict      =      self.get_tokenizer()

				UpperCAmelCase_  :				Dict      =      "Encode this sequence."
				UpperCAmelCase_  :				int      =      tokenizer.byte_encoder[" ".encode("utf-8"		)[0]]

				# Testing encoder arguments
				UpperCAmelCase_  :				Dict      =      tokenizer.encode(lowerCAmelCase_			,			add_special_tokens=lowerCAmelCase_			,			add_prefix_space=lowerCAmelCase_		)
				UpperCAmelCase_  :				Optional[int]      =      tokenizer.convert_ids_to_tokens(encoded[0]		)[0]
				self.assertNotEqual(lowerCAmelCase_			,			lowerCAmelCase_		)

				UpperCAmelCase_  :				Optional[Any]      =      tokenizer.encode(lowerCAmelCase_			,			add_special_tokens=lowerCAmelCase_			,			add_prefix_space=lowerCAmelCase_		)
				UpperCAmelCase_  :				str      =      tokenizer.convert_ids_to_tokens(encoded[0]		)[0]
				self.assertEqual(lowerCAmelCase_			,			lowerCAmelCase_		)

				tokenizer.add_special_tokens({"bos_token": "<s>"}		)
				UpperCAmelCase_  :				List[str]      =      tokenizer.encode(lowerCAmelCase_			,			add_special_tokens=lowerCAmelCase_		)
				UpperCAmelCase_  :				List[str]      =      tokenizer.convert_ids_to_tokens(encoded[1]		)[0]
				self.assertNotEqual(lowerCAmelCase_			,			lowerCAmelCase_		)

				# Testing spaces after special tokens
				UpperCAmelCase_  :				Union[str, Any]      =      "<mask>"
				tokenizer.add_special_tokens(
				    {"mask_token": AddedToken(lowerCAmelCase_			,			lstrip=lowerCAmelCase_			,			rstrip=lowerCAmelCase_		)}		)  # mask token has a left space
				UpperCAmelCase_  :				Optional[int]      =      tokenizer.convert_tokens_to_ids(lowerCAmelCase_		)

				UpperCAmelCase_  :				Any      =      "Encode <mask> sequence"
				UpperCAmelCase_  :				Any      =      "Encode <mask>sequence"

				UpperCAmelCase_  :				Optional[Any]      =      tokenizer.encode(lowerCAmelCase_		)
				UpperCAmelCase_  :				Any      =      encoded.index(lowerCAmelCase_		)
				UpperCAmelCase_  :				List[Any]      =      tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]		)[0]
				self.assertEqual(lowerCAmelCase_			,			lowerCAmelCase_		)

				UpperCAmelCase_  :				Dict      =      tokenizer.encode(lowerCAmelCase_		)
				UpperCAmelCase_  :				Tuple      =      encoded.index(lowerCAmelCase_		)
				UpperCAmelCase_  :				Optional[Any]      =      tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]		)[0]
				self.assertNotEqual(lowerCAmelCase_			,			lowerCAmelCase_		)





	def   _SCREAMING_SNAKE_CASE   (   self      :				Any		)		->    List[str]:
				pass





	def   _SCREAMING_SNAKE_CASE   (   self      :				List[Any]		)		->    List[str]:
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
										UpperCAmelCase_  :				Optional[Any]      =      self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_			,			**lowerCAmelCase_		)
										UpperCAmelCase_  :				Optional[Any]      =      self.tokenizer_class.from_pretrained(lowerCAmelCase_			,			**lowerCAmelCase_		)
										UpperCAmelCase_  :				List[Any]      =      "A, <mask> AllenNLP sentence."
										UpperCAmelCase_  :				List[str]      =      tokenizer_r.encode_plus(lowerCAmelCase_			,			add_special_tokens=lowerCAmelCase_			,			return_token_type_ids=lowerCAmelCase_		)
										UpperCAmelCase_  :				Dict      =      tokenizer_p.encode_plus(lowerCAmelCase_			,			add_special_tokens=lowerCAmelCase_			,			return_token_type_ids=lowerCAmelCase_		)

										# token_type_ids should put 0 everywhere
										self.assertEqual(sum(tokens_r["token_type_ids"]		)			,			sum(tokens_p["token_type_ids"]		)		)

										# attention_mask should put 1 everywhere, so sum over length should be 1
										self.assertEqual(
										    sum(tokens_r["attention_mask"]		) / len(tokens_r["attention_mask"]		)			,			sum(tokens_p["attention_mask"]		) / len(tokens_p["attention_mask"]		)			,			)

										UpperCAmelCase_  :				Any      =      tokenizer_r.convert_ids_to_tokens(tokens_r["input_ids"]		)
										UpperCAmelCase_  :				int      =      tokenizer_p.convert_ids_to_tokens(tokens_p["input_ids"]		)

										# Rust correctly handles the space before the mask while python doesnt
										self.assertSequenceEqual(tokens_p["input_ids"]			,			[0, 250, 6, 50_264, 3_823, 487, 21_992, 3_645, 4, 2]		)
										self.assertSequenceEqual(tokens_r["input_ids"]			,			[0, 250, 6, 50_264, 3_823, 487, 21_992, 3_645, 4, 2]		)

										self.assertSequenceEqual(
										    lowerCAmelCase_			,			["<s>", "A", ",", "<mask>", "ĠAllen", "N", "LP", "Ġsentence", ".", "</s>"]		)
										self.assertSequenceEqual(
										    lowerCAmelCase_			,			["<s>", "A", ",", "<mask>", "ĠAllen", "N", "LP", "Ġsentence", ".", "</s>"]		)





	def   _SCREAMING_SNAKE_CASE   (   self      :				Optional[Any]		)		->    int:
				for trim_offsets, add_prefix_space in itertools.product([True, False]			,			repeat=2		):
							UpperCAmelCase_  :				Tuple      =      self.rust_tokenizer_class.from_pretrained(
							    self.tmpdirname			,			use_fast=lowerCAmelCase_			,			add_prefix_space=lowerCAmelCase_			,			trim_offsets=lowerCAmelCase_		)

							UpperCAmelCase_  :				Optional[Any]      =      json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()		)
							UpperCAmelCase_  :				Tuple      =      json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()		)

							self.assertEqual(pre_tokenizer_state["add_prefix_space"]			,			lowerCAmelCase_		)

							self.assertEqual(post_processor_state["add_prefix_space"]			,			lowerCAmelCase_		)
							self.assertEqual(post_processor_state["trim_offsets"]			,			lowerCAmelCase_		)





	def   _SCREAMING_SNAKE_CASE   (   self      :				int		)		->    int:
				# Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space` and
				# `trim_offsets`
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
										UpperCAmelCase_  :				Any      =      "hello"  # `hello` is a token in the vocabulary of `pretrained_name`
										UpperCAmelCase_  :				Dict      =      f"""{text_of_1_token} {text_of_1_token}"""

										UpperCAmelCase_  :				List[str]      =      self.rust_tokenizer_class.from_pretrained(
										    lowerCAmelCase_			,			use_fast=lowerCAmelCase_			,			add_prefix_space=lowerCAmelCase_			,			trim_offsets=lowerCAmelCase_		)
										UpperCAmelCase_  :				Tuple      =      tokenizer_r(lowerCAmelCase_			,			return_offsets_mapping=lowerCAmelCase_			,			add_special_tokens=lowerCAmelCase_		)
										self.assertEqual(encoding.offset_mapping[0]			,			(0, len(lowerCAmelCase_		))		)
										self.assertEqual(
										    encoding.offset_mapping[1]			,			(len(lowerCAmelCase_		) + 1, len(lowerCAmelCase_		) + 1 + len(lowerCAmelCase_		))			,			)

										UpperCAmelCase_  :				Union[str, Any]      =      self.rust_tokenizer_class.from_pretrained(
										    lowerCAmelCase_			,			use_fast=lowerCAmelCase_			,			add_prefix_space=lowerCAmelCase_			,			trim_offsets=lowerCAmelCase_		)
										UpperCAmelCase_  :				List[str]      =      tokenizer_r(lowerCAmelCase_			,			return_offsets_mapping=lowerCAmelCase_			,			add_special_tokens=lowerCAmelCase_		)
										self.assertEqual(encoding.offset_mapping[0]			,			(0, len(lowerCAmelCase_		))		)
										self.assertEqual(
										    encoding.offset_mapping[1]			,			(len(lowerCAmelCase_		) + 1, len(lowerCAmelCase_		) + 1 + len(lowerCAmelCase_		))			,			)

										UpperCAmelCase_  :				Any      =      self.rust_tokenizer_class.from_pretrained(
										    lowerCAmelCase_			,			use_fast=lowerCAmelCase_			,			add_prefix_space=lowerCAmelCase_			,			trim_offsets=lowerCAmelCase_		)
										UpperCAmelCase_  :				List[str]      =      tokenizer_r(lowerCAmelCase_			,			return_offsets_mapping=lowerCAmelCase_			,			add_special_tokens=lowerCAmelCase_		)
										self.assertEqual(encoding.offset_mapping[0]			,			(0, len(lowerCAmelCase_		))		)
										self.assertEqual(
										    encoding.offset_mapping[1]			,			(len(lowerCAmelCase_		), len(lowerCAmelCase_		) + 1 + len(lowerCAmelCase_		))			,			)

										UpperCAmelCase_  :				Dict      =      self.rust_tokenizer_class.from_pretrained(
										    lowerCAmelCase_			,			use_fast=lowerCAmelCase_			,			add_prefix_space=lowerCAmelCase_			,			trim_offsets=lowerCAmelCase_		)
										UpperCAmelCase_  :				str      =      tokenizer_r(lowerCAmelCase_			,			return_offsets_mapping=lowerCAmelCase_			,			add_special_tokens=lowerCAmelCase_		)
										self.assertEqual(encoding.offset_mapping[0]			,			(0, len(lowerCAmelCase_		))		)
										self.assertEqual(
										    encoding.offset_mapping[1]			,			(len(lowerCAmelCase_		), len(lowerCAmelCase_		) + 1 + len(lowerCAmelCase_		))			,			)

										UpperCAmelCase_  :				Optional[int]      =      f""" {text}"""

										# tokenizer_r = self.rust_tokenizer_class.from_pretrained(
										#     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
										# )
										# encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
										# self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
										# self.assertEqual(
										#     encoding.offset_mapping[1],
										#     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
										# )

										UpperCAmelCase_  :				str      =      self.rust_tokenizer_class.from_pretrained(
										    lowerCAmelCase_			,			use_fast=lowerCAmelCase_			,			add_prefix_space=lowerCAmelCase_			,			trim_offsets=lowerCAmelCase_		)
										UpperCAmelCase_  :				List[str]      =      tokenizer_r(lowerCAmelCase_			,			return_offsets_mapping=lowerCAmelCase_			,			add_special_tokens=lowerCAmelCase_		)
										self.assertEqual(encoding.offset_mapping[0]			,			(1, 1 + len(lowerCAmelCase_		))		)
										self.assertEqual(
										    encoding.offset_mapping[1]			,			(1 + len(lowerCAmelCase_		) + 1, 1 + len(lowerCAmelCase_		) + 1 + len(lowerCAmelCase_		))			,			)

										UpperCAmelCase_  :				Optional[Any]      =      self.rust_tokenizer_class.from_pretrained(
										    lowerCAmelCase_			,			use_fast=lowerCAmelCase_			,			add_prefix_space=lowerCAmelCase_			,			trim_offsets=lowerCAmelCase_		)
										UpperCAmelCase_  :				Optional[Any]      =      tokenizer_r(lowerCAmelCase_			,			return_offsets_mapping=lowerCAmelCase_			,			add_special_tokens=lowerCAmelCase_		)
										self.assertEqual(encoding.offset_mapping[0]			,			(0, 1 + len(lowerCAmelCase_		))		)
										self.assertEqual(
										    encoding.offset_mapping[1]			,			(1 + len(lowerCAmelCase_		), 1 + len(lowerCAmelCase_		) + 1 + len(lowerCAmelCase_		))			,			)

										UpperCAmelCase_  :				Tuple      =      self.rust_tokenizer_class.from_pretrained(
										    lowerCAmelCase_			,			use_fast=lowerCAmelCase_			,			add_prefix_space=lowerCAmelCase_			,			trim_offsets=lowerCAmelCase_		)
										UpperCAmelCase_  :				Tuple      =      tokenizer_r(lowerCAmelCase_			,			return_offsets_mapping=lowerCAmelCase_			,			add_special_tokens=lowerCAmelCase_		)
										self.assertEqual(encoding.offset_mapping[0]			,			(0, 1 + len(lowerCAmelCase_		))		)
										self.assertEqual(
										    encoding.offset_mapping[1]			,			(1 + len(lowerCAmelCase_		), 1 + len(lowerCAmelCase_		) + 1 + len(lowerCAmelCase_		))			,			)











"""simple docstring"""
import doctest
import logging
import os
import unittest
from pathlib import Path
from typing import List, Union

import transformers
from transformers.testing_utils import require_tf, require_torch, slow


lowerCamelCase_       			=   logging.getLogger()







@unittest.skip('''Temporarily disable the doc tests.'''  )
@require_torch
@require_tf
@slow
class     UpperCamelCase_ (unittest.TestCase  ):





	def   _SCREAMING_SNAKE_CASE   (   self      :				Union[str, Any]			,			lowerCAmelCase_      :				Path			,			lowerCAmelCase_      :				Union[str, None] = None			,			lowerCAmelCase_      :				Union[List[str], None] = None			,			lowerCAmelCase_      :				Union[str, List[str], None] = None			,			lowerCAmelCase_      :				bool = True			,			)		->    Tuple:
				UpperCAmelCase_  :				Optional[Any]      =      [file for file in os.listdir(lowerCAmelCase_		) if os.path.isfile(os.path.join(lowerCAmelCase_			,			lowerCAmelCase_		)		)]

				if identifier is not None:
							UpperCAmelCase_  :				Optional[int]      =      [file for file in files if identifier in file]

				if n_identifier is not None:
							if isinstance(lowerCAmelCase_			,			lowerCAmelCase_		):
										for n_ in n_identifier:
													UpperCAmelCase_  :				List[Any]      =      [file for file in files if n_ not in file]
							else:
										UpperCAmelCase_  :				Optional[Any]      =      [file for file in files if n_identifier not in file]

				UpperCAmelCase_  :				List[Any]      =      ignore_files or []
				ignore_files.append("__init__.py"		)
				UpperCAmelCase_  :				Optional[Any]      =      [file for file in files if file not in ignore_files]

				for file in files:
							# Open all files
							print("Testing"			,			lowerCAmelCase_		)

							if only_modules:
										UpperCAmelCase_  :				str      =      file.split("."		)[0]
										try:
													UpperCAmelCase_  :				Tuple      =      getattr(lowerCAmelCase_			,			lowerCAmelCase_		)
													UpperCAmelCase_  :				int      =      doctest.DocTestSuite(lowerCAmelCase_		)
													UpperCAmelCase_  :				str      =      unittest.TextTestRunner().run(lowerCAmelCase_		)
													self.assertIs(len(result.failures		)			,			0		)
										except AttributeError:
													logger.info(f"""{module_identifier} is not a module."""		)
							else:
										UpperCAmelCase_  :				List[Any]      =      doctest.testfile(str(".." / directory / file		)			,			optionflags=doctest.ELLIPSIS		)
										self.assertIs(result.failed			,			0		)





	def   _SCREAMING_SNAKE_CASE   (   self      :				Optional[int]		)		->    List[Any]:
				UpperCAmelCase_  :				Optional[int]      =      Path("src/transformers"		)
				UpperCAmelCase_  :				List[Any]      =      "modeling"
				UpperCAmelCase_  :				Tuple      =      [
				    "modeling_ctrl.py",
				    "modeling_tf_ctrl.py",
				]
				self.analyze_directory(lowerCAmelCase_			,			identifier=lowerCAmelCase_			,			ignore_files=lowerCAmelCase_		)





	def   _SCREAMING_SNAKE_CASE   (   self      :				Union[str, Any]		)		->    List[Any]:
				UpperCAmelCase_  :				List[str]      =      Path("src/transformers"		)
				UpperCAmelCase_  :				int      =      "tokenization"
				self.analyze_directory(lowerCAmelCase_			,			identifier=lowerCAmelCase_		)





	def   _SCREAMING_SNAKE_CASE   (   self      :				Optional[Any]		)		->    Optional[Any]:
				UpperCAmelCase_  :				int      =      Path("src/transformers"		)
				UpperCAmelCase_  :				Any      =      "configuration"
				self.analyze_directory(lowerCAmelCase_			,			identifier=lowerCAmelCase_		)





	def   _SCREAMING_SNAKE_CASE   (   self      :				Any		)		->    Any:
				UpperCAmelCase_  :				int      =      Path("src/transformers"		)
				UpperCAmelCase_  :				List[Any]      =      ["configuration", "modeling", "tokenization"]
				self.analyze_directory(lowerCAmelCase_			,			n_identifier=lowerCAmelCase_		)





	def   _SCREAMING_SNAKE_CASE   (   self      :				List[str]		)		->    Dict:
				UpperCAmelCase_  :				List[str]      =      Path("docs/source"		)
				UpperCAmelCase_  :				int      =      ["favicon.ico"]
				self.analyze_directory(lowerCAmelCase_			,			ignore_files=lowerCAmelCase_			,			only_modules=lowerCAmelCase_		)



