






import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import AutoFeatureExtractor, WavaVecaFeatureExtractor
from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402


__UpperCAmelCase							:   str							=  get_tests_dir("fixtures")


class 					_snake_case						(			unittest.TestCase ):






 def    lowerCAmelCase_							(		self       )						->    int:
     # A mock response for an HTTP head request to emulate server down
     snake_case__      :Union[str, Any]         =   mock.Mock()
     snake_case__      :Union[str, Any]         =   500
     snake_case__      :str         =   {}
     snake_case__      :Dict         =   HTTPError
     snake_case__      :List[str]         =   {}

     # Download this model to make sure it's in the cache.
     snake_case__      :List[str]         =   WavaVecaFeatureExtractor.from_pretrained("hf-internal-testing/tiny-random-wav2vec2"       )
     # Under the mock environment we get a 500 error when trying to reach the model.
     with mock.patch("requests.Session.request"					,return_value=lowercase__       ) as mock_head:
         snake_case__      :Union[str, Any]         =   WavaVecaFeatureExtractor.from_pretrained("hf-internal-testing/tiny-random-wav2vec2"       )
         # This check we did call the fake head request
         mock_head.assert_called()






 def    lowerCAmelCase_							(		self       )						->    Tuple:
     # This test is for deprecated behavior and can be removed in v5
     snake_case__      :List[str]         =   WavaVecaFeatureExtractor.from_pretrained(
         "https://huggingface.co/hf-internal-testing/tiny-random-wav2vec2/resolve/main/preprocessor_config.json"       )




@is_staging_test
class 					_snake_case						(			unittest.TestCase ):






 @classmethod
 def    lowerCAmelCase_							(		cls       )						->    Any:
     snake_case__      :List[Any]         =   TOKEN
     HfFolder.save_token(lowercase__       )






 @classmethod
 def    lowerCAmelCase_							(		cls       )						->    Optional[Any]:
     try:
         delete_repo(token=cls._token					,repo_id="test-feature-extractor"       )
     except HTTPError:
         pass

     try:
         delete_repo(token=cls._token					,repo_id="valid_org/test-feature-extractor-org"       )
     except HTTPError:
         pass

     try:
         delete_repo(token=cls._token					,repo_id="test-dynamic-feature-extractor"       )
     except HTTPError:
         pass






 def    lowerCAmelCase_							(		self       )						->    List[Any]:
     snake_case__      :Tuple         =   WavaVecaFeatureExtractor.from_pretrained(lowercase__       )
     feature_extractor.push_to_hub("test-feature-extractor"					,use_auth_token=self._token       )

     snake_case__      :Optional[int]         =   WavaVecaFeatureExtractor.from_pretrained(f'{USER}/test-feature-extractor'       )
     for k, v in feature_extractor.__dict__.items():
         self.assertEqual(lowercase__					,getattr(lowercase__					,lowercase__       )       )

     # Reset repo
     delete_repo(token=self._token					,repo_id="test-feature-extractor"       )

     # Push to hub via save_pretrained
     with tempfile.TemporaryDirectory() as tmp_dir:
         feature_extractor.save_pretrained(
             lowercase__					,repo_id="test-feature-extractor"					,push_to_hub=lowercase__					,use_auth_token=self._token       )

     snake_case__      :List[str]         =   WavaVecaFeatureExtractor.from_pretrained(f'{USER}/test-feature-extractor'       )
     for k, v in feature_extractor.__dict__.items():
         self.assertEqual(lowercase__					,getattr(lowercase__					,lowercase__       )       )






 def    lowerCAmelCase_							(		self       )						->    Optional[int]:
     snake_case__      :Union[str, Any]         =   WavaVecaFeatureExtractor.from_pretrained(lowercase__       )
     feature_extractor.push_to_hub("valid_org/test-feature-extractor"					,use_auth_token=self._token       )

     snake_case__      :Any         =   WavaVecaFeatureExtractor.from_pretrained("valid_org/test-feature-extractor"       )
     for k, v in feature_extractor.__dict__.items():
         self.assertEqual(lowercase__					,getattr(lowercase__					,lowercase__       )       )

     # Reset repo
     delete_repo(token=self._token					,repo_id="valid_org/test-feature-extractor"       )

     # Push to hub via save_pretrained
     with tempfile.TemporaryDirectory() as tmp_dir:
         feature_extractor.save_pretrained(
             lowercase__					,repo_id="valid_org/test-feature-extractor-org"					,push_to_hub=lowercase__					,use_auth_token=self._token       )

     snake_case__      :Union[str, Any]         =   WavaVecaFeatureExtractor.from_pretrained("valid_org/test-feature-extractor-org"       )
     for k, v in feature_extractor.__dict__.items():
         self.assertEqual(lowercase__					,getattr(lowercase__					,lowercase__       )       )






 def    lowerCAmelCase_							(		self       )						->    List[Any]:
     CustomFeatureExtractor.register_for_auto_class()
     snake_case__      :List[Any]         =   CustomFeatureExtractor.from_pretrained(lowercase__       )

     feature_extractor.push_to_hub("test-dynamic-feature-extractor"					,use_auth_token=self._token       )

     # This has added the proper auto_map field to the config
     self.assertDictEqual(
         feature_extractor.auto_map					,{"AutoFeatureExtractor": "custom_feature_extraction.CustomFeatureExtractor"}					,)

     snake_case__      :str         =   AutoFeatureExtractor.from_pretrained(
         f'{USER}/test-dynamic-feature-extractor'					,trust_remote_code=lowercase__       )
     # Can't make an isinstance check because the new_feature_extractor is from the CustomFeatureExtractor class of a dynamic module
     self.assertEqual(new_feature_extractor.__class__.__name__					,"CustomFeatureExtractor"       )







from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_herbert import HerbertTokenizer


__UpperCAmelCase							:   str							=  logging.get_logger(__name__)

__UpperCAmelCase							:   List[Any]							=  {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

__UpperCAmelCase							:   List[Any]							=  {
    "vocab_file": {
        "allegro/herbert-base-cased": "https://huggingface.co/allegro/herbert-base-cased/resolve/main/vocab.json"
    },
    "merges_file": {
        "allegro/herbert-base-cased": "https://huggingface.co/allegro/herbert-base-cased/resolve/main/merges.txt"
    },
}

__UpperCAmelCase							:   str							=  {"allegro/herbert-base-cased": 5_1_4}
__UpperCAmelCase							:   List[str]							=  {}


class 					_snake_case						(			_A ):
	_A          =   VOCAB_FILES_NAMES
	_A          =   PRETRAINED_VOCAB_FILES_MAP
	_A          =   PRETRAINED_INIT_CONFIGURATION
	_A          =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_A          =   HerbertTokenizer






	def __init__(		self					,UpperCamelCase=None					,UpperCamelCase=None					,UpperCamelCase=None					,UpperCamelCase="<s>"					,UpperCamelCase="<unk>"					,UpperCamelCase="<pad>"					,UpperCamelCase="<mask>"					,UpperCamelCase="</s>"					,**UpperCamelCase					,)						->    Dict:
					super().__init__(
					    UpperCamelCase					,UpperCamelCase					,tokenizer_file=UpperCamelCase					,cls_token=UpperCamelCase					,unk_token=UpperCamelCase					,pad_token=UpperCamelCase					,mask_token=UpperCamelCase					,sep_token=UpperCamelCase					,**UpperCamelCase					,)






	def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase = None       )						->    List[int]:
					snake_case__      :Optional[int]         =   [self.cls_token_id]
					snake_case__      :Any         =   [self.sep_token_id]
					if token_ids_a is None:
									return cls + token_ids_a + sep

					return cls + token_ids_a + sep + token_ids_a + sep






	def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase = None					,UpperCamelCase = False       )						->    List[int]:
					if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=UpperCamelCase					,token_ids_a=UpperCamelCase					,already_has_special_tokens=UpperCamelCase       )

					if token_ids_a is None:
									return [1] + ([0] * len(UpperCamelCase       )) + [1]
					return [1] + ([0] * len(UpperCamelCase       )) + [1] + ([0] * len(UpperCamelCase       )) + [1]






	def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase = None       )						->    List[int]:
					snake_case__      :Any         =   [self.sep_token_id]
					snake_case__      :Union[str, Any]         =   [self.cls_token_id]

					if token_ids_a is None:
									return len(cls + token_ids_a + sep       ) * [0]
					return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]






	def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase = None       )						->    Tuple[str]:
					snake_case__      :List[str]         =   self._tokenizer.model.save(UpperCamelCase					,name=UpperCamelCase       )
					return tuple(UpperCamelCase       )