




import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bart import BartTokenizer


UpperCAmelCase_						=logging.get_logger(__name__)


UpperCAmelCase_						={"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

# See all BART models at https://huggingface.co/models?filter=bart
UpperCAmelCase_						={
    """vocab_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/vocab.json""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/vocab.json""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/merges.txt""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/merges.txt""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/tokenizer.json""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/tokenizer.json""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/tokenizer.json""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/tokenizer.json""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/tokenizer.json""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/tokenizer.json""",
    },
}

UpperCAmelCase_						={
    """facebook/bart-base""": 1024,
    """facebook/bart-large""": 1024,
    """facebook/bart-large-mnli""": 1024,
    """facebook/bart-large-cnn""": 1024,
    """facebook/bart-large-xsum""": 1024,
    """yjernite/bart_eli5""": 1024,
}


class 							__UpperCamelCase			( __UpperCAmelCase  ):





				'''simple docstring'''



				__a							: Tuple										=VOCAB_FILES_NAMES
				__a							: Optional[int]										=PRETRAINED_VOCAB_FILES_MAP
				__a							: Any										=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__a							: Optional[int]										=["""input_ids""", """attention_mask"""]
				__a							: Tuple										=BartTokenizer



				def __init__( self       , UpperCAmelCase_=None       , UpperCAmelCase_=None       , UpperCAmelCase_=None       , UpperCAmelCase_="replace"       , UpperCAmelCase_="<s>"       , UpperCAmelCase_="</s>"       , UpperCAmelCase_="</s>"       , UpperCAmelCase_="<s>"       , UpperCAmelCase_="<unk>"       , UpperCAmelCase_="<pad>"       , UpperCAmelCase_="<mask>"       , UpperCAmelCase_=False       , UpperCAmelCase_=True       , **UpperCAmelCase_       , ):
											super().__init__(
											    UpperCAmelCase_       , UpperCAmelCase_       , tokenizer_file=UpperCAmelCase_       , errors=UpperCAmelCase_       , bos_token=UpperCAmelCase_       , eos_token=UpperCAmelCase_       , sep_token=UpperCAmelCase_       , cls_token=UpperCAmelCase_       , unk_token=UpperCAmelCase_       , pad_token=UpperCAmelCase_       , mask_token=UpperCAmelCase_       , add_prefix_space=UpperCAmelCase_       , trim_offsets=UpperCAmelCase_       , **UpperCAmelCase_       , )

											lowerCAmelCase   =		json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()						)
											if pre_tok_state.get('''add_prefix_space'''       , UpperCAmelCase_						) != add_prefix_space:
																		lowerCAmelCase   =		getattr(UpperCAmelCase_       , pre_tok_state.pop('''type'''						)						)
																		lowerCAmelCase   =		add_prefix_space
																		lowerCAmelCase   =		pre_tok_class(**UpperCAmelCase_						)

											lowerCAmelCase   =		add_prefix_space

											# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
											lowerCAmelCase   =		'''post_processor'''
											lowerCAmelCase   =		getattr(self.backend_tokenizer       , UpperCAmelCase_       , UpperCAmelCase_						)
											if tokenizer_component_instance:
																		lowerCAmelCase   =		json.loads(tokenizer_component_instance.__getstate__()						)

																		# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
																		if "sep" in state:
																									lowerCAmelCase   =		tuple(state['''sep''']						)
																		if "cls" in state:
																									lowerCAmelCase   =		tuple(state['''cls''']						)

																		lowerCAmelCase   =		False

																		if state.get('''add_prefix_space'''       , UpperCAmelCase_						) != add_prefix_space:
																									lowerCAmelCase   =		add_prefix_space
																									lowerCAmelCase   =		True

																		if state.get('''trim_offsets'''       , UpperCAmelCase_						) != trim_offsets:
																									lowerCAmelCase   =		trim_offsets
																									lowerCAmelCase   =		True

																		if changes_to_apply:
																									lowerCAmelCase   =		getattr(UpperCAmelCase_       , state.pop('''type'''						)						)
																									lowerCAmelCase   =		component_class(**UpperCAmelCase_						)
																									setattr(self.backend_tokenizer       , UpperCAmelCase_       , UpperCAmelCase_						)



				@property
				def       __snake_case		( self						):
											if self._mask_token is None:
																		if self.verbose:
																									logger.error('''Using mask_token, but it is not set yet.'''						)
																		return None
											return str(self._mask_token						)



				@mask_token.setter
				def       __snake_case		( self       , UpperCAmelCase_						):
											lowerCAmelCase   =		AddedToken(UpperCAmelCase_       , lstrip=UpperCAmelCase_       , rstrip=UpperCAmelCase_						) if isinstance(UpperCAmelCase_       , UpperCAmelCase_						) else value
											lowerCAmelCase   =		value



				def       __snake_case		( self       , *UpperCAmelCase_       , **UpperCAmelCase_						):
											lowerCAmelCase   =		kwargs.get('''is_split_into_words'''       , UpperCAmelCase_						)

											if is_split_into_words and not self.add_prefix_space:
																		raise ValueError(
																		    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
																		    '''to use it with pretokenized inputs.'''						)

											return super()._batch_encode_plus(*UpperCAmelCase_       , **UpperCAmelCase_						)



				def       __snake_case		( self       , *UpperCAmelCase_       , **UpperCAmelCase_						):
											lowerCAmelCase   =		kwargs.get('''is_split_into_words'''       , UpperCAmelCase_						)

											if is_split_into_words and not self.add_prefix_space:
																		raise ValueError(
																		    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
																		    '''to use it with pretokenized inputs.'''						)

											return super()._encode_plus(*UpperCAmelCase_       , **UpperCAmelCase_						)



				def       __snake_case		( self       , UpperCAmelCase_       , UpperCAmelCase_ = None						):
											lowerCAmelCase   =		self._tokenizer.model.save(UpperCAmelCase_       , name=UpperCAmelCase_						)
											return tuple(UpperCAmelCase_						)



				def       __snake_case		( self       , UpperCAmelCase_       , UpperCAmelCase_=None						):
											lowerCAmelCase   =		[self.bos_token_id] + token_ids_a + [self.eos_token_id]
											if token_ids_a is None:
																		return output

											return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]






				def       __snake_case		( self       , UpperCAmelCase_       , UpperCAmelCase_ = None						):
											lowerCAmelCase   =		[self.sep_token_id]
											lowerCAmelCase   =		[self.cls_token_id]

											if token_ids_a is None:
																		return len(cls + token_ids_a + sep						) * [0]
											return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]









import json
import os
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from requests.exceptions import HTTPError

from transformers.utils import (
    CONFIG_NAME,
    FLAX_WEIGHTS_NAME,
    TF2_WEIGHTS_NAME,
    TRANSFORMERS_CACHE,
    WEIGHTS_NAME,
    cached_file,
    get_file_from_repo,
    has_file,
)


UpperCAmelCase_						="""hf-internal-testing/tiny-random-bert"""
UpperCAmelCase_						=os.path.join(TRANSFORMERS_CACHE, """models--hf-internal-testing--tiny-random-bert""")
UpperCAmelCase_						="""9b8c223d42b2188cb49d29af482996f9d0f3e5a6"""


class 							__UpperCamelCase			( unittest.TestCase  ):





				'''simple docstring'''



				def       __snake_case		( self						):
											lowerCAmelCase   =		cached_file(UpperCAmelCase_       , UpperCAmelCase_						)
											# Should have downloaded the file in here
											self.assertTrue(os.path.isdir(UpperCAmelCase_						)						)
											# Cache should contain at least those three subfolders:
											for subfolder in ["blobs", "refs", "snapshots"]:
																		self.assertTrue(os.path.isdir(os.path.join(UpperCAmelCase_       , UpperCAmelCase_						)						)						)
											with open(os.path.join(UpperCAmelCase_       , '''refs'''       , '''main'''						)						) as f:
																		lowerCAmelCase   =		f.read()
											self.assertEqual(UpperCAmelCase_       , os.path.join(UpperCAmelCase_       , '''snapshots'''       , UpperCAmelCase_       , UpperCAmelCase_						)						)
											self.assertTrue(os.path.isfile(UpperCAmelCase_						)						)

											# File is cached at the same place the second time.
											lowerCAmelCase   =		cached_file(UpperCAmelCase_       , UpperCAmelCase_						)
											self.assertEqual(UpperCAmelCase_       , UpperCAmelCase_						)

											# Using a specific revision to test the full commit hash.
											lowerCAmelCase   =		cached_file(UpperCAmelCase_       , UpperCAmelCase_       , revision='''9b8c223'''						)
											self.assertEqual(UpperCAmelCase_       , os.path.join(UpperCAmelCase_       , '''snapshots'''       , UpperCAmelCase_       , UpperCAmelCase_						)						)



				def       __snake_case		( self						):
											with self.assertRaisesRegex(UpperCAmelCase_       , '''is not a valid model identifier'''						):
																		lowerCAmelCase   =		cached_file('''tiny-random-bert'''       , UpperCAmelCase_						)

											with self.assertRaisesRegex(UpperCAmelCase_       , '''is not a valid git identifier'''						):
																		lowerCAmelCase   =		cached_file(UpperCAmelCase_       , UpperCAmelCase_       , revision='''aaaa'''						)

											with self.assertRaisesRegex(UpperCAmelCase_       , '''does not appear to have a file named'''						):
																		lowerCAmelCase   =		cached_file(UpperCAmelCase_       , '''conf'''						)



				def       __snake_case		( self						):
											with self.assertRaisesRegex(UpperCAmelCase_       , '''does not appear to have a file named'''						):
																		lowerCAmelCase   =		cached_file(UpperCAmelCase_       , '''conf'''						)

											with open(os.path.join(UpperCAmelCase_       , '''refs'''       , '''main'''						)						) as f:
																		lowerCAmelCase   =		f.read()
											self.assertTrue(os.path.isfile(os.path.join(UpperCAmelCase_       , '''.no_exist'''       , UpperCAmelCase_       , '''conf'''						)						)						)

											lowerCAmelCase   =		cached_file(UpperCAmelCase_       , '''conf'''       , _raise_exceptions_for_missing_entries=UpperCAmelCase_						)
											self.assertIsNone(UpperCAmelCase_						)

											lowerCAmelCase   =		cached_file(UpperCAmelCase_       , '''conf'''       , local_files_only=UpperCAmelCase_       , _raise_exceptions_for_missing_entries=UpperCAmelCase_						)
											self.assertIsNone(UpperCAmelCase_						)

											lowerCAmelCase   =		mock.Mock()
											lowerCAmelCase   =		5_00
											lowerCAmelCase   =		{}
											lowerCAmelCase   =		HTTPError
											lowerCAmelCase   =		{}

											# Under the mock environment we get a 500 error when trying to reach the tokenizer.
											with mock.patch('''requests.Session.request'''       , return_value=UpperCAmelCase_						) as mock_head:
																		lowerCAmelCase   =		cached_file(UpperCAmelCase_       , '''conf'''       , _raise_exceptions_for_connection_errors=UpperCAmelCase_						)
																		self.assertIsNone(UpperCAmelCase_						)
																		# This check we did call the fake head request
																		mock_head.assert_called()



				def       __snake_case		( self						):
											self.assertTrue(has_file('''hf-internal-testing/tiny-bert-pt-only'''       , UpperCAmelCase_						)						)
											self.assertFalse(has_file('''hf-internal-testing/tiny-bert-pt-only'''       , UpperCAmelCase_						)						)
											self.assertFalse(has_file('''hf-internal-testing/tiny-bert-pt-only'''       , UpperCAmelCase_						)						)



				def       __snake_case		( self						):
											# `get_file_from_repo` returns None if the file does not exist
											self.assertIsNone(get_file_from_repo('''bert-base-cased'''       , '''ahah.txt'''						)						)

											# The function raises if the repository does not exist.
											with self.assertRaisesRegex(UpperCAmelCase_       , '''is not a valid model identifier'''						):
																		get_file_from_repo('''bert-base-case'''       , UpperCAmelCase_						)

											# The function raises if the revision does not exist.
											with self.assertRaisesRegex(UpperCAmelCase_       , '''is not a valid git identifier'''						):
																		get_file_from_repo('''bert-base-cased'''       , UpperCAmelCase_       , revision='''ahaha'''						)

											lowerCAmelCase   =		get_file_from_repo('''bert-base-cased'''       , UpperCAmelCase_						)
											# The name is the cached name which is not very easy to test, so instead we load the content.
											lowerCAmelCase   =		json.loads(open(UpperCAmelCase_       , '''r'''						).read()						)
											self.assertEqual(config['''hidden_size''']       , 7_68						)






				def       __snake_case		( self						):
											with tempfile.TemporaryDirectory() as tmp_dir:
																		lowerCAmelCase   =		Path(UpperCAmelCase_						) / '''a.txt'''
																		filename.touch()
																		self.assertEqual(get_file_from_repo(UpperCAmelCase_       , '''a.txt'''						)       , str(UpperCAmelCase_						)						)

																		self.assertIsNone(get_file_from_repo(UpperCAmelCase_       , '''b.txt'''						)						)



