






"""simple docstring"""



import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / '''utils'''))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
						from test_module.custom_tokenization_fast import CustomTokenizerFast






class 							__SCREAMING_SNAKE_CASE						(				unittest.TestCase		):


							def 						_UpperCamelCase       (			self      :						List[str]							):
													'''simple docstring'''





													A__						:  Optional[Any]		  =  mock.Mock()
													A__						:  int		  =  500
													A__						:  Any		  =  {}
													A__						:  Optional[int]		  =  HTTPError
													A__						:  str		  =  {}

													# Download this model to make sure it's in the cache.
													A__						:  int		  =  BertTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""							)

													# Under the mock environment we get a 500 error when trying to reach the tokenizer.
													with mock.patch("""requests.Session.request"""			,    return_value=_lowercase							) as mock_head:
																			A__						:  List[Any]		  =  BertTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""							)
																			# This check we did call the fake head request
																			mock_head.assert_called()


							@require_tokenizers
							def 						_UpperCamelCase       (			self      :						List[str]							):
													'''simple docstring'''





													A__						:  Dict		  =  mock.Mock()
													A__						:  Optional[int]		  =  500
													A__						:  int		  =  {}
													A__						:  str		  =  HTTPError
													A__						:  List[str]		  =  {}

													# Download this model to make sure it's in the cache.
													A__						:  Optional[Any]		  =  GPTaTokenizerFast.from_pretrained("""gpt2"""							)

													# Under the mock environment we get a 500 error when trying to reach the tokenizer.
													with mock.patch("""requests.Session.request"""			,    return_value=_lowercase							) as mock_head:
																			A__						:  Union[str, Any]		  =  GPTaTokenizerFast.from_pretrained("""gpt2"""							)
																			# This check we did call the fake head request
																			mock_head.assert_called()


							def 						_UpperCamelCase       (			self      :						str							):
													'''simple docstring'''





													try:
																			A__						:  Any		  =  tempfile.mktemp()
																			with open(_lowercase			,    """wb"""							) as f:
																									http_get("""https://huggingface.co/albert-base-v1/resolve/main/spiece.model"""			,    _lowercase							)

																			A__						:  str		  =  AlbertTokenizer.from_pretrained(_lowercase							)
													finally:
																			os.remove(_lowercase							)

													# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
													# the current folder and have the right name.
													if os.path.isfile("""tokenizer.json"""							):
																			# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
																			return
													try:
																			with open("""tokenizer.json"""			,    """wb"""							) as f:
																									http_get("""https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"""			,    _lowercase							)
																			A__						:  List[Any]		  =  AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-gpt2"""							)
																			# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
																			self.assertEqual(tokenizer.vocab_size			,    1000							)
																			# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

													finally:
																			os.remove("""tokenizer.json"""							)


							def 						_UpperCamelCase       (			self      :						Dict							):
													'''simple docstring'''





													A__						:  Tuple		  =  AlbertTokenizer.from_pretrained("""https://huggingface.co/albert-base-v1/resolve/main/spiece.model"""							)






@is_staging_test
class 							__SCREAMING_SNAKE_CASE						(				unittest.TestCase		):
							snake_case_     =							['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', 'bla', 'blou']


							@classmethod
							def 						_UpperCamelCase       (			cls      :						Union[str, Any]							):
													'''simple docstring'''





													A__						:  Dict		  =  TOKEN
													HfFolder.save_token(_lowercase							)


							@classmethod
							def 						_UpperCamelCase       (			cls      :						List[Any]							):
													'''simple docstring'''





													try:
																			delete_repo(token=cls._token			,    repo_id="""test-tokenizer"""							)
													except HTTPError:
																			pass

													try:
																			delete_repo(token=cls._token			,    repo_id="""valid_org/test-tokenizer-org"""							)
													except HTTPError:
																			pass

													try:
																			delete_repo(token=cls._token			,    repo_id="""test-dynamic-tokenizer"""							)
													except HTTPError:
																			pass


							def 						_UpperCamelCase       (			self      :						Any							):
													'''simple docstring'''





													with tempfile.TemporaryDirectory() as tmp_dir:
																			A__						:  Tuple		  =  os.path.join(_lowercase			,    """vocab.txt"""							)
																			with open(_lowercase			,    """w"""			,    encoding="""utf-8"""							) as vocab_writer:
																									vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]							)							)
																			A__						:  Tuple		  =  BertTokenizer(_lowercase							)

													tokenizer.push_to_hub("""test-tokenizer"""			,    use_auth_token=self._token							)
													A__						:  int		  =  BertTokenizer.from_pretrained(F'{USER}/test-tokenizer'							)
													self.assertDictEqual(new_tokenizer.vocab			,    tokenizer.vocab							)

													# Reset repo
													delete_repo(token=self._token			,    repo_id="""test-tokenizer"""							)

													# Push to hub via save_pretrained
													with tempfile.TemporaryDirectory() as tmp_dir:
																			tokenizer.save_pretrained(_lowercase			,    repo_id="""test-tokenizer"""			,    push_to_hub=_lowercase			,    use_auth_token=self._token							)

													A__						:  int		  =  BertTokenizer.from_pretrained(F'{USER}/test-tokenizer'							)
													self.assertDictEqual(new_tokenizer.vocab			,    tokenizer.vocab							)


							def 						_UpperCamelCase       (			self      :						List[Any]							):
													'''simple docstring'''





													with tempfile.TemporaryDirectory() as tmp_dir:
																			A__						:  int		  =  os.path.join(_lowercase			,    """vocab.txt"""							)
																			with open(_lowercase			,    """w"""			,    encoding="""utf-8"""							) as vocab_writer:
																									vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]							)							)
																			A__						:  int		  =  BertTokenizer(_lowercase							)

													tokenizer.push_to_hub("""valid_org/test-tokenizer-org"""			,    use_auth_token=self._token							)
													A__						:  Any		  =  BertTokenizer.from_pretrained("""valid_org/test-tokenizer-org"""							)
													self.assertDictEqual(new_tokenizer.vocab			,    tokenizer.vocab							)

													# Reset repo
													delete_repo(token=self._token			,    repo_id="""valid_org/test-tokenizer-org"""							)

													# Push to hub via save_pretrained
													with tempfile.TemporaryDirectory() as tmp_dir:
																			tokenizer.save_pretrained(
																			    _lowercase			,    repo_id="""valid_org/test-tokenizer-org"""			,    push_to_hub=_lowercase			,    use_auth_token=self._token							)

													A__						:  Optional[Any]		  =  BertTokenizer.from_pretrained("""valid_org/test-tokenizer-org"""							)
													self.assertDictEqual(new_tokenizer.vocab			,    tokenizer.vocab							)


							@require_tokenizers
							def 						_UpperCamelCase       (			self      :						int							):
													'''simple docstring'''





													CustomTokenizer.register_for_auto_class()
													with tempfile.TemporaryDirectory() as tmp_dir:
																			A__						:  Dict		  =  os.path.join(_lowercase			,    """vocab.txt"""							)
																			with open(_lowercase			,    """w"""			,    encoding="""utf-8"""							) as vocab_writer:
																									vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]							)							)
																			A__						:  Optional[int]		  =  CustomTokenizer(_lowercase							)

													# No fast custom tokenizer
													tokenizer.push_to_hub("""test-dynamic-tokenizer"""			,    use_auth_token=self._token							)

													A__						:  List[str]		  =  AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'			,    trust_remote_code=_lowercase							)
													# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
													self.assertEqual(tokenizer.__class__.__name__			,    """CustomTokenizer"""							)

													# Fast and slow custom tokenizer
													CustomTokenizerFast.register_for_auto_class()
													with tempfile.TemporaryDirectory() as tmp_dir:
																			A__						:  Union[str, Any]		  =  os.path.join(_lowercase			,    """vocab.txt"""							)
																			with open(_lowercase			,    """w"""			,    encoding="""utf-8"""							) as vocab_writer:
																									vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]							)							)

																			A__						:  List[Any]		  =  BertTokenizerFast.from_pretrained(_lowercase							)
																			bert_tokenizer.save_pretrained(_lowercase							)
																			A__						:  Optional[Any]		  =  CustomTokenizerFast.from_pretrained(_lowercase							)

													tokenizer.push_to_hub("""test-dynamic-tokenizer"""			,    use_auth_token=self._token							)

													A__						:  List[Any]		  =  AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'			,    trust_remote_code=_lowercase							)
													# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
													self.assertEqual(tokenizer.__class__.__name__			,    """CustomTokenizerFast"""							)
													A__						:  Union[str, Any]		  =  AutoTokenizer.from_pretrained(
													    F'{USER}/test-dynamic-tokenizer'			,    use_fast=_lowercase			,    trust_remote_code=_lowercase							)
													# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
													self.assertEqual(tokenizer.__class__.__name__			,    """CustomTokenizer"""							)






class 							__SCREAMING_SNAKE_CASE						(				unittest.TestCase		):


							def 						_UpperCamelCase       (			self      :						Union[str, Any]							):
													'''simple docstring'''





													A__						:  Tuple		  =  Trie()
													trie.add("""Hello 友達"""							)
													self.assertEqual(trie.data			,    {"""H""": {"""e""": {"""l""": {"""l""": {"""o""": {""" """: {"""友""": {"""達""": {"""""": 1}}}}}}}}}							)
													trie.add("""Hello"""							)
													trie.data
													self.assertEqual(trie.data			,    {"""H""": {"""e""": {"""l""": {"""l""": {"""o""": {"""""": 1, """ """: {"""友""": {"""達""": {"""""": 1}}}}}}}}}							)


							def 						_UpperCamelCase       (			self      :						str							):
													'''simple docstring'''





													A__						:  Optional[int]		  =  Trie()
													self.assertEqual(trie.split("""[CLS] This is a extra_id_100"""							)			,    ["""[CLS] This is a extra_id_100"""]							)
													trie.add("""[CLS]"""							)
													trie.add("""extra_id_1"""							)
													trie.add("""extra_id_100"""							)
													self.assertEqual(trie.split("""[CLS] This is a extra_id_100"""							)			,    ["""[CLS]""", """ This is a """, """extra_id_100"""]							)


							def 						_UpperCamelCase       (			self      :						Dict							):
													'''simple docstring'''





													A__						:  Union[str, Any]		  =  Trie()
													trie.add("""A"""							)
													self.assertEqual(trie.split("""ABC"""							)			,    ["""A""", """BC"""]							)
													self.assertEqual(trie.split("""BCA"""							)			,    ["""BC""", """A"""]							)


							def 						_UpperCamelCase       (			self      :						List[str]							):
													'''simple docstring'''





													A__						:  Tuple		  =  Trie()
													trie.add("""TOKEN]"""							)
													trie.add("""[SPECIAL_TOKEN]"""							)
													self.assertEqual(trie.split("""This is something [SPECIAL_TOKEN]"""							)			,    ["""This is something """, """[SPECIAL_TOKEN]"""]							)


							def 						_UpperCamelCase       (			self      :						Dict							):
													'''simple docstring'''





													A__						:  Union[str, Any]		  =  Trie()
													trie.add("""A"""							)
													trie.add("""P"""							)
													trie.add("""[SPECIAL_TOKEN]"""							)
													self.assertEqual(trie.split("""This is something [SPECIAL_TOKEN]"""							)			,    ["""This is something """, """[SPECIAL_TOKEN]"""]							)


							def 						_UpperCamelCase       (			self      :						List[str]							):
													'''simple docstring'''





													A__						:  Union[str, Any]		  =  Trie()
													trie.add("""AB"""							)
													trie.add("""B"""							)
													trie.add("""C"""							)
													self.assertEqual(trie.split("""ABC"""							)			,    ["""AB""", """C"""]							)


							def 						_UpperCamelCase       (			self      :						List[Any]							):
													'''simple docstring'''





													A__						:  int		  =  Trie()
													trie.add("""ABC"""							)
													trie.add("""B"""							)
													trie.add("""CD"""							)
													self.assertEqual(trie.split("""ABCD"""							)			,    ["""ABC""", """D"""]							)


							def 						_UpperCamelCase       (			self      :						Union[str, Any]							):
													'''simple docstring'''





													A__						:  Optional[int]		  =  Trie()
													A__						:  Any		  =  trie.cut_text("""ABC"""			,    [0, 0, 2, 1, 2, 3]							)
													self.assertEqual(_lowercase			,    ["""AB""", """C"""]							)














"""simple docstring"""



import os

try:
						from .build_directory_md import good_file_paths
except ImportError:
						from build_directory_md import good_file_paths  # type: ignore

A_    =							list(good_file_paths())
assert filepaths, "good_file_paths() failed!"

A_    =							[file for file in filepaths if file != file.lower()]
if upper_files:
						print(F'{len(upper_files)} files contain uppercase characters:')
						print('''\n'''.join(upper_files) + '''\n''')

A_    =							[file for file in filepaths if ''' ''' in file]
if space_files:
						print(F'{len(space_files)} files contain space characters:')
						print('''\n'''.join(space_files) + '''\n''')

A_    =							[file for file in filepaths if '''-''' in file]
if hyphen_files:
						print(F'{len(hyphen_files)} files contain hyphen characters:')
						print('''\n'''.join(hyphen_files) + '''\n''')

A_    =							[file for file in filepaths if os.sep not in file]
if nodir_files:
						print(F'{len(nodir_files)} files are not in a directory:')
						print('''\n'''.join(nodir_files) + '''\n''')

A_    =							len(upper_files + space_files + hyphen_files + nodir_files)
if bad_files:
						import sys

						sys.exit(bad_files)






