






'''simple docstring'''






import os
import unittest

from transformers.models.phobert.tokenization_phobert import VOCAB_FILES_NAMES, PhobertTokenizer

from ...test_tokenization_common import TokenizerTesterMixin




class a_    (    UpperCamelCase_		,      unittest.TestCase							):
   _snake_case        =							PhobertTokenizer
   _snake_case        =							False







   def     SCREAMING_SNAKE_CASE__					(self)			->      List[Any]:
      """simple docstring"""






      super().setUp()

      # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
      __snake_case				:      Optional[int]     							= ['T@@', 'i', 'I', 'R@@', 'r', 'e@@']
      __snake_case				:      Optional[int]     							= dict(zip(__a					,   range(len(__a))))
      __snake_case				:      Any     							= ['#version: 0.2', 'l à</w>']
      __snake_case				:      Tuple     							= {'unk_token': '<unk>'}

      __snake_case				:      Union[str, Any]     							= os.path.join(self.tmpdirname					,   VOCAB_FILES_NAMES['vocab_file'])
      __snake_case				:      List[str]     							= os.path.join(self.tmpdirname					,   VOCAB_FILES_NAMES['merges_file'])

      with open(self.vocab_file					,   'w'					,   encoding='utf-8') as fp:
         for token in vocab_tokens:
            fp.write(F"""{token} {vocab_tokens[token]}\n""")
      with open(self.merges_file					,   'w'					,   encoding='utf-8') as fp:
         fp.write('\n'.join(__a))







   def     SCREAMING_SNAKE_CASE__					(self					,   **__a)			->      Dict:
      """simple docstring"""






      kwargs.update(self.special_tokens_map)
      return PhobertTokenizer.from_pretrained(self.tmpdirname					,   **__a)







   def     SCREAMING_SNAKE_CASE__					(self					,   __a)			->      Any:
      """simple docstring"""






      __snake_case				:      List[str]     							= 'Tôi là VinAI Research'
      __snake_case				:      Optional[int]     							= 'T<unk> i <unk> <unk> <unk> <unk> <unk> <unk> I Re<unk> e<unk> <unk> <unk> <unk>'
      return input_text, output_text







   def     SCREAMING_SNAKE_CASE__					(self)			->      Tuple:
      """simple docstring"""






      __snake_case				:      str     							= PhobertTokenizer(self.vocab_file					,   self.merges_file					,   **self.special_tokens_map)
      __snake_case				:      int     							= 'Tôi là VinAI Research'
      __snake_case				:      Optional[Any]     							= 'T@@ ô@@ i l@@ à V@@ i@@ n@@ A@@ I R@@ e@@ s@@ e@@ a@@ r@@ c@@ h'.split()
      __snake_case				:      str     							= tokenizer.tokenize(__a)
      print(__a)
      self.assertListEqual(__a					,   __a)

      __snake_case				:      List[Any]     							= tokens + [tokenizer.unk_token]

      __snake_case				:      List[str]     							= [4, 3, 5, 3, 3, 3, 3, 3, 3, 6, 7, 9, 3, 9, 3, 3, 3, 3, 3]
      self.assertListEqual(tokenizer.convert_tokens_to_ids(__a)					,   __a)







'''simple docstring'''






import contextlib
import importlib
import io
import unittest

import transformers

# Try to import everything from transformers to ensure every object can be loaded.
from transformers import *  # noqa F406
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, require_flax, require_tf, require_torch
from transformers.utils import ContextManagers, find_labels, is_flax_available, is_tf_available, is_torch_available


if is_torch_available():
      from transformers import BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification

if is_tf_available():
      from transformers import TFBertForPreTraining, TFBertForQuestionAnswering, TFBertForSequenceClassification

if is_flax_available():
      from transformers import FlaxBertForPreTraining, FlaxBertForQuestionAnswering, FlaxBertForSequenceClassification


__A    	=   DUMMY_UNKNOWN_IDENTIFIER
# An actual model hosted on huggingface.co

__A    	=   '''main'''
# Default branch name
__A    	=   '''f2c752cfc5c0ab6f4bdec59acea69eefbee381c2'''
# One particular commit (not the top of `main`)
__A    	=   '''aaaaaaa'''
# This commit does not exist, so we should 404.

__A    	=   '''d9e9f15bc825e4b2c9249e9578f884bbcb5e3684'''
# Sha-1 of config.json on the top of `main`, for checking purposes
__A    	=   '''4b243c475af8d0a7754e87d7d096c92e5199ec2fe168a2ee7998e3b8e9bcb1d3'''




@contextlib.contextmanager
def        _SCREAMING_SNAKE_CASE  (				)    -> List[Any]:







   """simple docstring"""
   print('Welcome!'  )
   yield
   print('Bye!'  )




@contextlib.contextmanager
def        _SCREAMING_SNAKE_CASE  (				)    -> Optional[Any]:







   """simple docstring"""
   print('Bonjour!'  )
   yield
   print('Au revoir!'  )




class a_    (    unittest.TestCase							):







   def     SCREAMING_SNAKE_CASE__					(self)			->      int:
      """simple docstring"""






      assert transformers.__spec__ is not None
      assert importlib.util.find_spec('transformers') is not None




class a_    (    unittest.TestCase							):







   @unittest.mock.patch('sys.stdout'					,   new_callable=io.StringIO)
   def     SCREAMING_SNAKE_CASE__					(self					,   __a)			->      int:
      """simple docstring"""






      with ContextManagers([]):
         print('Transformers are awesome!')
      # The print statement adds a new line at the end of the output
      self.assertEqual(mock_stdout.getvalue()					,   'Transformers are awesome!\n')







   @unittest.mock.patch('sys.stdout'					,   new_callable=io.StringIO)
   def     SCREAMING_SNAKE_CASE__					(self					,   __a)			->      List[str]:
      """simple docstring"""






      with ContextManagers([context_en()]):
         print('Transformers are awesome!')
      # The output should be wrapped with an English welcome and goodbye
      self.assertEqual(mock_stdout.getvalue()					,   'Welcome!\nTransformers are awesome!\nBye!\n')







   @unittest.mock.patch('sys.stdout'					,   new_callable=io.StringIO)
   def     SCREAMING_SNAKE_CASE__					(self					,   __a)			->      Tuple:
      """simple docstring"""






      with ContextManagers([context_fr(), context_en()]):
         print('Transformers are awesome!')
      # The output should be wrapped with an English and French welcome and goodbye
      self.assertEqual(mock_stdout.getvalue()					,   'Bonjour!\nWelcome!\nTransformers are awesome!\nBye!\nAu revoir!\n')







   @require_torch
   def     SCREAMING_SNAKE_CASE__					(self)			->      Union[str, Any]:
      """simple docstring"""






      self.assertEqual(find_labels(__a)					,   ['labels'])
      self.assertEqual(find_labels(__a)					,   ['labels', 'next_sentence_label'])
      self.assertEqual(find_labels(__a)					,   ['start_positions', 'end_positions'])




      class a_    (    UpperCamelCase_							):
         pass


      self.assertEqual(find_labels(__a)					,   ['labels'])







   @require_tf
   def     SCREAMING_SNAKE_CASE__					(self)			->      List[Any]:
      """simple docstring"""






      self.assertEqual(find_labels(__a)					,   ['labels'])
      self.assertEqual(find_labels(__a)					,   ['labels', 'next_sentence_label'])
      self.assertEqual(find_labels(__a)					,   ['start_positions', 'end_positions'])




      class a_    (    UpperCamelCase_							):
         pass


      self.assertEqual(find_labels(__a)					,   ['labels'])







   @require_flax
   def     SCREAMING_SNAKE_CASE__					(self)			->      int:
      """simple docstring"""






      self.assertEqual(find_labels(__a)					,   [])
      self.assertEqual(find_labels(__a)					,   [])
      self.assertEqual(find_labels(__a)					,   [])




      class a_    (    UpperCamelCase_							):
         pass


      self.assertEqual(find_labels(__a)					,   [])