





import json
import os
import unittest

from transformers.models.xlm.tokenization_xlm import VOCAB_FILES_NAMES, XLMTokenizer
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin





class      lowercase__    (							__A     ,					unittest.TestCase  ):
  __UpperCamelCase	     =  XLMTokenizer
  __UpperCamelCase	     =  False







  def 	UpperCAmelCase__		(	self						):
       super().setUp()

       # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
       lowerCAmelCase_					:       List[Any]    =		[
           """l""",
           """o""",
           """w""",
           """e""",
           """r""",
           """s""",
           """t""",
           """i""",
           """d""",
           """n""",
           """w</w>""",
           """r</w>""",
           """t</w>""",
           """lo""",
           """low""",
           """er</w>""",
           """low</w>""",
           """lowest</w>""",
           """newer</w>""",
           """wider</w>""",
           """<unk>""",
       ]
       lowerCAmelCase_					:       Any    =		dict(zip(_lowercase    ,						range(len(_lowercase						)						)						)						)
       lowerCAmelCase_					:       List[Any]    =		["""l o 123""", """lo w 1456""", """e r</w> 1789""", """"""]

       lowerCAmelCase_					:       Optional[int]    =		os.path.join(self.tmpdirname    ,						VOCAB_FILES_NAMES["""vocab_file"""]						)
       lowerCAmelCase_					:       List[str]    =		os.path.join(self.tmpdirname    ,						VOCAB_FILES_NAMES["""merges_file"""]						)
       with open(self.vocab_file    ,						"""w"""						) as fp:
            fp.write(json.dumps(_lowercase						)						)
       with open(self.merges_file    ,						"""w"""						) as fp:
            fp.write("""\n""".join(_lowercase						)						)







  def 	UpperCAmelCase__		(	self    ,						_lowercase						):
       lowerCAmelCase_					:       List[str]    =		"""lower newer"""
       lowerCAmelCase_					:       Union[str, Any]    =		"""lower newer"""
       return input_text, output_text







  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       Optional[Any]    =		XLMTokenizer(self.vocab_file    ,						self.merges_file						)

       lowerCAmelCase_					:       int    =		"""lower"""
       lowerCAmelCase_					:       Union[str, Any]    =		["""low""", """er</w>"""]
       lowerCAmelCase_					:       List[str]    =		tokenizer.tokenize(_lowercase						)
       self.assertListEqual(_lowercase    ,						_lowercase						)

       lowerCAmelCase_					:       Dict    =		tokens + ["""<unk>"""]
       lowerCAmelCase_					:       Union[str, Any]    =		[14, 15, 20]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowercase						)    ,						_lowercase						)







  @slow
  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       Dict    =		XLMTokenizer.from_pretrained("""xlm-mlm-en-2048"""						)

       lowerCAmelCase_					:       Dict    =		tokenizer.encode("""sequence builders"""    ,						add_special_tokens=_lowercase						)
       lowerCAmelCase_					:       Optional[int]    =		tokenizer.encode("""multi-sequence build"""    ,						add_special_tokens=_lowercase						)

       lowerCAmelCase_					:       Union[str, Any]    =		tokenizer.build_inputs_with_special_tokens(_lowercase						)
       lowerCAmelCase_					:       Optional[int]    =		tokenizer.build_inputs_with_special_tokens(_lowercase    ,						_lowercase						)

       assert encoded_sentence == [0] + text + [1]
       assert encoded_pair == [0] + text + [1] + text_a + [1]







import json
import os
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from requests.exceptions import HTTPError

from transformers.utils import (
    CONFIG_NAME,
    FLAX_WEIGHTS_NAME,
    TF2_WEIGHTS_NAME,
    TRANSFORMERS_CACHE,
    WEIGHTS_NAME,
    cached_file,
    get_file_from_repo,
    has_file,
)


UpperCAmelCase_					:   List[str] 							=						"""hf-internal-testing/tiny-random-bert"""
UpperCAmelCase_					:   List[Any] 							=						os.path.join(TRANSFORMERS_CACHE, """models--hf-internal-testing--tiny-random-bert""")
UpperCAmelCase_					:   Tuple 							=						"""9b8c223d42b2188cb49d29af482996f9d0f3e5a6"""





class      lowercase__    (							unittest.TestCase  ):







  def 	UpperCAmelCase__		(	self						):
       lowerCAmelCase_					:       Union[str, Any]    =		cached_file(_lowercase    ,						_lowercase						)
       # Should have downloaded the file in here
       self.assertTrue(os.path.isdir(_lowercase						)						)
       # Cache should contain at least those three subfolders:
       for subfolder in ["blobs", "refs", "snapshots"]:
            self.assertTrue(os.path.isdir(os.path.join(_lowercase    ,						_lowercase						)						)						)
       with open(os.path.join(_lowercase    ,						"""refs"""    ,						"""main"""						)						) as f:
            lowerCAmelCase_					:       Any    =		f.read()
       self.assertEqual(_lowercase    ,						os.path.join(_lowercase    ,						"""snapshots"""    ,						_lowercase    ,						_lowercase						)						)
       self.assertTrue(os.path.isfile(_lowercase						)						)

       # File is cached at the same place the second time.
       lowerCAmelCase_					:       Dict    =		cached_file(_lowercase    ,						_lowercase						)
       self.assertEqual(_lowercase    ,						_lowercase						)

       # Using a specific revision to test the full commit hash.
       lowerCAmelCase_					:       Optional[int]    =		cached_file(_lowercase    ,						_lowercase    ,						revision="""9b8c223"""						)
       self.assertEqual(_lowercase    ,						os.path.join(_lowercase    ,						"""snapshots"""    ,						_lowercase    ,						_lowercase						)						)







  def 	UpperCAmelCase__		(	self						):
       with self.assertRaisesRegex(_lowercase    ,						"""is not a valid model identifier"""						):
            lowerCAmelCase_					:       Dict    =		cached_file("""tiny-random-bert"""    ,						_lowercase						)

       with self.assertRaisesRegex(_lowercase    ,						"""is not a valid git identifier"""						):
            lowerCAmelCase_					:       Tuple    =		cached_file(_lowercase    ,						_lowercase    ,						revision="""aaaa"""						)

       with self.assertRaisesRegex(_lowercase    ,						"""does not appear to have a file named"""						):
            lowerCAmelCase_					:       List[Any]    =		cached_file(_lowercase    ,						"""conf"""						)







  def 	UpperCAmelCase__		(	self						):
       with self.assertRaisesRegex(_lowercase    ,						"""does not appear to have a file named"""						):
            lowerCAmelCase_					:       Any    =		cached_file(_lowercase    ,						"""conf"""						)

       with open(os.path.join(_lowercase    ,						"""refs"""    ,						"""main"""						)						) as f:
            lowerCAmelCase_					:       str    =		f.read()
       self.assertTrue(os.path.isfile(os.path.join(_lowercase    ,						""".no_exist"""    ,						_lowercase    ,						"""conf"""						)						)						)

       lowerCAmelCase_					:       Union[str, Any]    =		cached_file(_lowercase    ,						"""conf"""    ,						_raise_exceptions_for_missing_entries=_lowercase						)
       self.assertIsNone(_lowercase						)

       lowerCAmelCase_					:       Any    =		cached_file(_lowercase    ,						"""conf"""    ,						local_files_only=_lowercase    ,						_raise_exceptions_for_missing_entries=_lowercase						)
       self.assertIsNone(_lowercase						)

       lowerCAmelCase_					:       Any    =		mock.Mock()
       lowerCAmelCase_					:       Optional[Any]    =		500
       lowerCAmelCase_					:       Any    =		{}
       lowerCAmelCase_					:       List[str]    =		HTTPError
       lowerCAmelCase_					:       List[str]    =		{}

       # Under the mock environment we get a 500 error when trying to reach the tokenizer.
       with mock.patch("""requests.Session.request"""    ,						return_value=_lowercase						) as mock_head:
            lowerCAmelCase_					:       Union[str, Any]    =		cached_file(_lowercase    ,						"""conf"""    ,						_raise_exceptions_for_connection_errors=_lowercase						)
            self.assertIsNone(_lowercase						)
            # This check we did call the fake head request
            mock_head.assert_called()







  def 	UpperCAmelCase__		(	self						):
       self.assertTrue(has_file("""hf-internal-testing/tiny-bert-pt-only"""    ,						_lowercase						)						)
       self.assertFalse(has_file("""hf-internal-testing/tiny-bert-pt-only"""    ,						_lowercase						)						)
       self.assertFalse(has_file("""hf-internal-testing/tiny-bert-pt-only"""    ,						_lowercase						)						)







  def 	UpperCAmelCase__		(	self						):
       # `get_file_from_repo` returns None if the file does not exist
       self.assertIsNone(get_file_from_repo("""bert-base-cased"""    ,						"""ahah.txt"""						)						)

       # The function raises if the repository does not exist.
       with self.assertRaisesRegex(_lowercase    ,						"""is not a valid model identifier"""						):
            get_file_from_repo("""bert-base-case"""    ,						_lowercase						)

       # The function raises if the revision does not exist.
       with self.assertRaisesRegex(_lowercase    ,						"""is not a valid git identifier"""						):
            get_file_from_repo("""bert-base-cased"""    ,						_lowercase    ,						revision="""ahaha"""						)

       lowerCAmelCase_					:       Union[str, Any]    =		get_file_from_repo("""bert-base-cased"""    ,						_lowercase						)
       # The name is the cached name which is not very easy to test, so instead we load the content.
       lowerCAmelCase_					:       Any    =		json.loads(open(_lowercase    ,						"""r"""						).read()						)
       self.assertEqual(config["""hidden_size"""]    ,						768						)







  def 	UpperCAmelCase__		(	self						):
       with tempfile.TemporaryDirectory() as tmp_dir:
            lowerCAmelCase_					:       Optional[Any]    =		Path(_lowercase						) / """a.txt"""
            filename.touch()
            self.assertEqual(get_file_from_repo(_lowercase    ,						"""a.txt"""						)    ,						str(_lowercase						)						)

            self.assertIsNone(get_file_from_repo(_lowercase    ,						"""b.txt"""						)						)
