import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / '''utils'''))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
				from test_module.custom_tokenization_fast import CustomTokenizerFast


class 			lowerCamelCase_  (     unittest.TestCase				):






				'''simple docstring'''






				def     UpperCamelCase__     (			self) ->			int:
								__UpperCamelCase  :List[str]    						=	mock.Mock()
								__UpperCamelCase  :Any    						=	500
								__UpperCamelCase  :int    						=	{}
								__UpperCamelCase  :Union[str, Any]    						=	HTTPError
								__UpperCamelCase  :int    						=	{}

								# Download this model to make sure it's in the cache.
								__UpperCamelCase  :Union[str, Any]    						=	BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert''')

								# Under the mock environment we get a 500 error when trying to reach the tokenizer.
								with mock.patch('''requests.Session.request'''				,				return_value=lowerCamelCase_) as mock_head:
												__UpperCamelCase  :List[str]    						=	BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert''')
												# This check we did call the fake head request
												mock_head.assert_called()







				@require_tokenizers
				def     UpperCamelCase__     (			self) ->			Any:
								__UpperCamelCase  :List[str]    						=	mock.Mock()
								__UpperCamelCase  :Any    						=	500
								__UpperCamelCase  :Any    						=	{}
								__UpperCamelCase  :Union[str, Any]    						=	HTTPError
								__UpperCamelCase  :Optional[Any]    						=	{}

								# Download this model to make sure it's in the cache.
								__UpperCamelCase  :Optional[Any]    						=	GPTaTokenizerFast.from_pretrained('''gpt2''')

								# Under the mock environment we get a 500 error when trying to reach the tokenizer.
								with mock.patch('''requests.Session.request'''				,				return_value=lowerCamelCase_) as mock_head:
												__UpperCamelCase  :Optional[Any]    						=	GPTaTokenizerFast.from_pretrained('''gpt2''')
												# This check we did call the fake head request
												mock_head.assert_called()







				def     UpperCamelCase__     (			self) ->			Optional[Any]:




								try:
												__UpperCamelCase  :Optional[int]    						=	tempfile.mktemp()
												with open(lowerCamelCase_				,				'''wb''') as f:
																http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''				,				lowerCamelCase_)

												__UpperCamelCase  :List[str]    						=	AlbertTokenizer.from_pretrained(lowerCamelCase_)
								finally:
												os.remove(lowerCamelCase_)

								# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
								# the current folder and have the right name.
								if os.path.isfile('''tokenizer.json'''):
												# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
												return
								try:
												with open('''tokenizer.json'''				,				'''wb''') as f:
																http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'''				,				lowerCamelCase_)
												__UpperCamelCase  :str    						=	AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2''')
												# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
												self.assertEqual(tokenizer.vocab_size				,				1_000)
												# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

								finally:
												os.remove('''tokenizer.json''')







				def     UpperCamelCase__     (			self) ->			Union[str, Any]:
								__UpperCamelCase  :Optional[Any]    						=	AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model''')


@is_staging_test
class 			lowerCamelCase_  (     unittest.TestCase				):






				'''simple docstring'''






				a__			:		int      			=   ["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """bla""", """blou"""]







				@classmethod
				def     UpperCamelCase__     (			cls) ->			List[str]:
								__UpperCamelCase  :str    						=	TOKEN
								HfFolder.save_token(lowerCamelCase_)







				@classmethod
				def     UpperCamelCase__     (			cls) ->			Any:




								try:
												delete_repo(token=cls._token				,				repo_id='''test-tokenizer''')
								except HTTPError:
												pass

								try:
												delete_repo(token=cls._token				,				repo_id='''valid_org/test-tokenizer-org''')
								except HTTPError:
												pass

								try:
												delete_repo(token=cls._token				,				repo_id='''test-dynamic-tokenizer''')
								except HTTPError:
												pass







				def     UpperCamelCase__     (			self) ->			Tuple:




								with tempfile.TemporaryDirectory() as tmp_dir:
												__UpperCamelCase  :Union[str, Any]    						=	os.path.join(lowerCamelCase_				,				'''vocab.txt''')
												with open(lowerCamelCase_				,				'''w'''				,				encoding='''utf-8''') as vocab_writer:
																vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]))
												__UpperCamelCase  :Tuple    						=	BertTokenizer(lowerCamelCase_)

								tokenizer.push_to_hub('''test-tokenizer'''				,				use_auth_token=self._token)
								__UpperCamelCase  :Optional[Any]    						=	BertTokenizer.from_pretrained(f"""{USER}/test-tokenizer""")
								self.assertDictEqual(new_tokenizer.vocab				,				tokenizer.vocab)

								# Reset repo
								delete_repo(token=self._token				,				repo_id='''test-tokenizer''')

								# Push to hub via save_pretrained
								with tempfile.TemporaryDirectory() as tmp_dir:
												tokenizer.save_pretrained(lowerCamelCase_				,				repo_id='''test-tokenizer'''				,				push_to_hub=lowerCamelCase_				,				use_auth_token=self._token)

								__UpperCamelCase  :Optional[int]    						=	BertTokenizer.from_pretrained(f"""{USER}/test-tokenizer""")
								self.assertDictEqual(new_tokenizer.vocab				,				tokenizer.vocab)







				def     UpperCamelCase__     (			self) ->			str:




								with tempfile.TemporaryDirectory() as tmp_dir:
												__UpperCamelCase  :int    						=	os.path.join(lowerCamelCase_				,				'''vocab.txt''')
												with open(lowerCamelCase_				,				'''w'''				,				encoding='''utf-8''') as vocab_writer:
																vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]))
												__UpperCamelCase  :Union[str, Any]    						=	BertTokenizer(lowerCamelCase_)

								tokenizer.push_to_hub('''valid_org/test-tokenizer-org'''				,				use_auth_token=self._token)
								__UpperCamelCase  :Dict    						=	BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org''')
								self.assertDictEqual(new_tokenizer.vocab				,				tokenizer.vocab)

								# Reset repo
								delete_repo(token=self._token				,				repo_id='''valid_org/test-tokenizer-org''')

								# Push to hub via save_pretrained
								with tempfile.TemporaryDirectory() as tmp_dir:
												tokenizer.save_pretrained(
												    lowerCamelCase_				,				repo_id='''valid_org/test-tokenizer-org'''				,				push_to_hub=lowerCamelCase_				,				use_auth_token=self._token)

								__UpperCamelCase  :Union[str, Any]    						=	BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org''')
								self.assertDictEqual(new_tokenizer.vocab				,				tokenizer.vocab)







				@require_tokenizers
				def     UpperCamelCase__     (			self) ->			int:
								CustomTokenizer.register_for_auto_class()
								with tempfile.TemporaryDirectory() as tmp_dir:
												__UpperCamelCase  :List[Any]    						=	os.path.join(lowerCamelCase_				,				'''vocab.txt''')
												with open(lowerCamelCase_				,				'''w'''				,				encoding='''utf-8''') as vocab_writer:
																vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]))
												__UpperCamelCase  :Optional[Any]    						=	CustomTokenizer(lowerCamelCase_)

								# No fast custom tokenizer
								tokenizer.push_to_hub('''test-dynamic-tokenizer'''				,				use_auth_token=self._token)

								__UpperCamelCase  :Optional[Any]    						=	AutoTokenizer.from_pretrained(f"""{USER}/test-dynamic-tokenizer"""				,				trust_remote_code=lowerCamelCase_)
								# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
								self.assertEqual(tokenizer.__class__.__name__				,				'''CustomTokenizer''')

								# Fast and slow custom tokenizer
								CustomTokenizerFast.register_for_auto_class()
								with tempfile.TemporaryDirectory() as tmp_dir:
												__UpperCamelCase  :Dict    						=	os.path.join(lowerCamelCase_				,				'''vocab.txt''')
												with open(lowerCamelCase_				,				'''w'''				,				encoding='''utf-8''') as vocab_writer:
																vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]))

												__UpperCamelCase  :Dict    						=	BertTokenizerFast.from_pretrained(lowerCamelCase_)
												bert_tokenizer.save_pretrained(lowerCamelCase_)
												__UpperCamelCase  :List[Any]    						=	CustomTokenizerFast.from_pretrained(lowerCamelCase_)

								tokenizer.push_to_hub('''test-dynamic-tokenizer'''				,				use_auth_token=self._token)

								__UpperCamelCase  :List[Any]    						=	AutoTokenizer.from_pretrained(f"""{USER}/test-dynamic-tokenizer"""				,				trust_remote_code=lowerCamelCase_)
								# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
								self.assertEqual(tokenizer.__class__.__name__				,				'''CustomTokenizerFast''')
								__UpperCamelCase  :List[Any]    						=	AutoTokenizer.from_pretrained(
								    f"""{USER}/test-dynamic-tokenizer"""				,				use_fast=lowerCamelCase_				,				trust_remote_code=lowerCamelCase_)
								# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
								self.assertEqual(tokenizer.__class__.__name__				,				'''CustomTokenizer''')


class 			lowerCamelCase_  (     unittest.TestCase				):






				'''simple docstring'''






				def     UpperCamelCase__     (			self) ->			List[str]:
								__UpperCamelCase  :Union[str, Any]    						=	Trie()
								trie.add('''Hello 友達''')
								self.assertEqual(trie.data				,				{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}})
								trie.add('''Hello''')
								trie.data
								self.assertEqual(trie.data				,				{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}})







				def     UpperCamelCase__     (			self) ->			Optional[Any]:
								__UpperCamelCase  :Union[str, Any]    						=	Trie()
								self.assertEqual(trie.split('''[CLS] This is a extra_id_100''')				,				['''[CLS] This is a extra_id_100'''])
								trie.add('''[CLS]''')
								trie.add('''extra_id_1''')
								trie.add('''extra_id_100''')
								self.assertEqual(trie.split('''[CLS] This is a extra_id_100''')				,				['''[CLS]''', ''' This is a ''', '''extra_id_100'''])







				def     UpperCamelCase__     (			self) ->			int:
								__UpperCamelCase  :List[Any]    						=	Trie()
								trie.add('''A''')
								self.assertEqual(trie.split('''ABC''')				,				['''A''', '''BC'''])
								self.assertEqual(trie.split('''BCA''')				,				['''BC''', '''A'''])







				def     UpperCamelCase__     (			self) ->			int:
								__UpperCamelCase  :Optional[int]    						=	Trie()
								trie.add('''TOKEN]''')
								trie.add('''[SPECIAL_TOKEN]''')
								self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]''')				,				['''This is something ''', '''[SPECIAL_TOKEN]'''])







				def     UpperCamelCase__     (			self) ->			Optional[int]:
								__UpperCamelCase  :Optional[Any]    						=	Trie()
								trie.add('''A''')
								trie.add('''P''')
								trie.add('''[SPECIAL_TOKEN]''')
								self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]''')				,				['''This is something ''', '''[SPECIAL_TOKEN]'''])







				def     UpperCamelCase__     (			self) ->			Tuple:
								__UpperCamelCase  :List[Any]    						=	Trie()
								trie.add('''AB''')
								trie.add('''B''')
								trie.add('''C''')
								self.assertEqual(trie.split('''ABC''')				,				['''AB''', '''C'''])







				def     UpperCamelCase__     (			self) ->			List[Any]:
								__UpperCamelCase  :List[Any]    						=	Trie()
								trie.add('''ABC''')
								trie.add('''B''')
								trie.add('''CD''')
								self.assertEqual(trie.split('''ABCD''')				,				['''ABC''', '''D'''])







				def     UpperCamelCase__     (			self) ->			int:
								__UpperCamelCase  :int    						=	Trie()
								__UpperCamelCase  :Optional[int]    						=	trie.cut_text('''ABC'''				,				[0, 0, 2, 1, 2, 3])
								self.assertEqual(lowerCamelCase_				,				['''AB''', '''C'''])






def lowerCamelCase						(     SCREAMING_SNAKE_CASE			):
    '''simple docstring'''







    __UpperCamelCase  :Optional[int]    						=	generate_pascal_triangle(SCREAMING_SNAKE_CASE			)
    for row_idx in range(SCREAMING_SNAKE_CASE			):
        # Print left spaces
        for _ in range(num_rows - row_idx - 1			):
            print(end=''' '''			)
        # Print row values
        for col_idx in range(row_idx + 1			):
            if col_idx != row_idx:
                print(triangle[row_idx][col_idx]						,    end=''' '''			)
            else:
                print(triangle[row_idx][col_idx]						,    end=''''''			)
        print()



def lowerCamelCase						(     SCREAMING_SNAKE_CASE			):
    '''simple docstring'''







    if not isinstance(SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE			):
        raise TypeError('''The input value of \'num_rows\' should be \'int\''''			)

    if num_rows == 0:
        return []
    elif num_rows < 0:
        raise ValueError(
            '''The input value of \'num_rows\' should be greater than or equal to 0'''			)

    __UpperCamelCase  :list[list[int]]    						=	[]
    for current_row_idx in range(SCREAMING_SNAKE_CASE			):
        __UpperCamelCase  :List[Any]    						=	populate_current_row(SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE			)
        triangle.append(SCREAMING_SNAKE_CASE			)
    return triangle



def lowerCamelCase						(     SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE			):
    '''simple docstring'''







    __UpperCamelCase  :int    						=	[-1] * (current_row_idx + 1)
    # first and last elements of current row are equal to 1
    __UpperCamelCase						, __UpperCamelCase  :int    						=	1, 1
    for current_col_idx in range(1						,    SCREAMING_SNAKE_CASE			):
        calculate_current_element(
            SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE			)
    return current_row



def lowerCamelCase						(     SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE						,    ):
    '''simple docstring'''







    __UpperCamelCase  :Dict    						=	triangle[current_row_idx - 1][current_col_idx - 1]
    __UpperCamelCase  :List[Any]    						=	triangle[current_row_idx - 1][current_col_idx]
    __UpperCamelCase  :List[Any]    						=	above_to_left_elt + above_to_right_elt



def lowerCamelCase						(     SCREAMING_SNAKE_CASE			):
    '''simple docstring'''







    if not isinstance(SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE			):
        raise TypeError('''The input value of \'num_rows\' should be \'int\''''			)

    if num_rows == 0:
        return []
    elif num_rows < 0:
        raise ValueError(
            '''The input value of \'num_rows\' should be greater than or equal to 0'''			)

    __UpperCamelCase  :list[list[int]]    						=	[[1]]

    for row_index in range(1						,    SCREAMING_SNAKE_CASE			):
        __UpperCamelCase  :Optional[Any]    						=	[0] + result[-1] + [0]
        __UpperCamelCase  :Any    						=	row_index + 1
        # Calculate the number of distinct elements in a row
        __UpperCamelCase  :Optional[Any]    						=	sum(divmod(SCREAMING_SNAKE_CASE						,    2			)			)
        __UpperCamelCase  :Union[str, Any]    						=	[
            temp_row[i - 1] + temp_row[i] for i in range(1						,    distinct_elements + 1			)
        ]
        __UpperCamelCase  :List[Any]    						=	row_first_half[: (row_index + 1) // 2]
        row_second_half.reverse()
        __UpperCamelCase  :List[str]    						=	row_first_half + row_second_half
        result.append(SCREAMING_SNAKE_CASE			)

    return result



def lowerCamelCase						(     ):
    '''simple docstring'''







    from collections.abc import Callable
    from timeit import timeit

    def benchmark_a_function(SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE			) -> None:
        __UpperCamelCase  :List[str]    						=	f"""{func.__name__}({value})"""
        __UpperCamelCase  :Optional[int]    						=	timeit(f"""__main__.{call}"""						,    setup='''import __main__'''			)
        # print(f"{call:38} = {func(value)} -- {timing:.4f} seconds")
        print(f"""{call:38} -- {timing:.4f} seconds"""			)

    for value in range(15			):  # (1, 7, 14):
        for func in (generate_pascal_triangle, generate_pascal_triangle_optimized):
            benchmark_a_function(SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE			)
        print()


if __name__ == "__main__":
    import doctest

    doctest.testmod()
    benchmark()





