





'''simple docstring'''

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


a        =						logging.get_logger(__name__)

a        =						{"vocab_file": "vocab.txt"}

a        =						{
    "vocab_file": {
        "YituTech/conv-bert-base": "https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt",
        "YituTech/conv-bert-medium-small": (
            "https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt"
        ),
        "YituTech/conv-bert-small": "https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt",
    }
}

a        =						{
    "YituTech/conv-bert-base": 512,
    "YituTech/conv-bert-medium-small": 512,
    "YituTech/conv-bert-small": 512,
}


a        =						{
    "YituTech/conv-bert-base": {"do_lower_case": True},
    "YituTech/conv-bert-medium-small": {"do_lower_case": True},
    "YituTech/conv-bert-small": {"do_lower_case": True},
}







class __a (   _snake_case			):
   __UpperCamelCase   : List[Any]	      =	VOCAB_FILES_NAMES
   __UpperCamelCase   : str	      =	PRETRAINED_VOCAB_FILES_MAP
   __UpperCamelCase   : Dict	      =	PRETRAINED_INIT_CONFIGURATION
   __UpperCamelCase   : Optional[int]	      =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __UpperCamelCase   : List[Any]	      =	ConvBertTokenizer





   def __init__(					self				:			List[Any]				,lowerCamelCase				:			Dict=None				,lowerCamelCase				:			str=None				,lowerCamelCase				:			str=True				,lowerCamelCase				:			Any="[UNK]"				,lowerCamelCase				:			List[str]="[SEP]"				,lowerCamelCase				:			Union[str, Any]="[PAD]"				,lowerCamelCase				:			List[str]="[CLS]"				,lowerCamelCase				:			Union[str, Any]="[MASK]"				,lowerCamelCase				:			Optional[Any]=True				,lowerCamelCase				:			Dict=None				,**lowerCamelCase				:			Union[str, Any]				,):

      '''simple docstring'''

      super().__init__(
          lowerCamelCase				,tokenizer_file=lowerCamelCase				,do_lower_case=lowerCamelCase				,unk_token=lowerCamelCase				,sep_token=lowerCamelCase				,pad_token=lowerCamelCase				,cls_token=lowerCamelCase				,mask_token=lowerCamelCase				,tokenize_chinese_chars=lowerCamelCase				,strip_accents=lowerCamelCase				,**lowerCamelCase				,)

      __SCREAMING_SNAKE_CASE      							=       json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
      if (
          normalizer_state.get("""lowercase"""				,lowerCamelCase    ) != do_lower_case
          or normalizer_state.get("""strip_accents"""				,lowerCamelCase    ) != strip_accents
          or normalizer_state.get("""handle_chinese_chars"""				,lowerCamelCase    ) != tokenize_chinese_chars
      ):
         __SCREAMING_SNAKE_CASE      							=       getattr(lowerCamelCase				,normalizer_state.pop("""type"""    )    )
         __SCREAMING_SNAKE_CASE      							=       do_lower_case
         __SCREAMING_SNAKE_CASE      							=       strip_accents
         __SCREAMING_SNAKE_CASE      							=       tokenize_chinese_chars
         __SCREAMING_SNAKE_CASE      							=       normalizer_class(**lowerCamelCase    )

      __SCREAMING_SNAKE_CASE      							=       do_lower_case





   def      UpperCAmelCase__     (					self				:			Tuple				,lowerCamelCase				:			List[str]				,lowerCamelCase				:			List[Any]=None    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       [self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
         output += token_ids_a + [self.sep_token_id]

      return output





   def      UpperCAmelCase__     (					self				:			Union[str, Any]				,lowerCamelCase				:			List[int]				,lowerCamelCase				:			Optional[List[int]] = None    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       [self.sep_token_id]
      __SCREAMING_SNAKE_CASE      							=       [self.cls_token_id]
      if token_ids_a is None:
         return len(cls + token_ids_a + sep    ) * [0]
      return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]





   def      UpperCAmelCase__     (					self				:			Any				,lowerCamelCase				:			str				,lowerCamelCase				:			Optional[str] = None    ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE      							=       self._tokenizer.model.save(lowerCamelCase				,name=lowerCamelCase    )
      return tuple(lowerCamelCase    )












'''simple docstring'''

import time
from contextlib import contextmanager
from pathlib import Path

import pytest
import requests
from huggingface_hub.hf_api import HfApi, HfFolder


a        =						"__DUMMY_TRANSFORMERS_USER__"
a        =						"Dummy User"
a        =						"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"

a        =						"https://hub-ci.huggingface.co"
a        =						CI_HUB_ENDPOINT + "/datasets/{repo_id}/resolve/{revision}/{path}"
a        =						CI_HUB_ENDPOINT + "/{repo_id}/resolve/{revision}/{filename}"
a        =						Path("~/.huggingface/hub_ci_token").expanduser()

@pytest.fixture
def 						__magic_name__       (							__UpperCAmelCase      )						->					int:

   '''simple docstring'''







   monkeypatch.setattr(
       """huggingface_hub.file_download.HUGGINGFACE_CO_URL_TEMPLATE"""							,	__UpperCAmelCase      )

@pytest.fixture
def 						__magic_name__       (							__UpperCAmelCase      )						->					Optional[Any]:

   '''simple docstring'''







   monkeypatch.setattr("""datasets.config.HF_ENDPOINT"""							,	__UpperCAmelCase      )
   monkeypatch.setattr("""datasets.config.HUB_DATASETS_URL"""							,	__UpperCAmelCase      )

@pytest.fixture
def 						__magic_name__       (							__UpperCAmelCase      )						->					Optional[Any]:

   '''simple docstring'''







   monkeypatch.setattr("""huggingface_hub.hf_api.HfFolder.path_token"""							,	__UpperCAmelCase      )

@pytest.fixture
def 						__magic_name__       (							__UpperCAmelCase							,	__UpperCAmelCase      )						->					Dict:

   '''simple docstring'''







   HfFolder.save_token(__UpperCAmelCase      )
   yield
   HfFolder.delete_token()

@pytest.fixture(scope="""session"""      )
def 						__magic_name__       (							)						->					Optional[Any]:

   '''simple docstring'''







   return HfApi(endpoint=__UpperCAmelCase      )

@pytest.fixture(scope="""session"""      )
def 						__magic_name__       (							__UpperCAmelCase      )						->					Dict:

   '''simple docstring'''







   __SCREAMING_SNAKE_CASE      							=       HfFolder.get_token()
   HfFolder.save_token(__UpperCAmelCase      )
   yield CI_HUB_USER_TOKEN
   if previous_token is not None:
      HfFolder.save_token(__UpperCAmelCase      )

@pytest.fixture
def 						__magic_name__       (							__UpperCAmelCase      )						->					Dict:

   '''simple docstring'''







   def _cleanup_repo(__UpperCAmelCase      ):
      hf_api.delete_repo(__UpperCAmelCase							,	token=__UpperCAmelCase							,	repo_type="""dataset"""      )

   return _cleanup_repo

@pytest.fixture
def 						__magic_name__       (							__UpperCAmelCase      )						->					int:

   '''simple docstring'''







   @contextmanager
   def _temporary_repo(__UpperCAmelCase      ):
      try:
         yield repo_id
      finally:
         cleanup_repo(__UpperCAmelCase      )

   return _temporary_repo

@pytest.fixture(scope="""session"""      )
def 						__magic_name__       (							__UpperCAmelCase							,	__UpperCAmelCase							,	__UpperCAmelCase      )						->					Dict:

   '''simple docstring'''







   __SCREAMING_SNAKE_CASE      							=       f"""repo_txt_data-{int(time.time() * 1_0e3      )}"""
   __SCREAMING_SNAKE_CASE      							=       f"""{CI_HUB_USER}/{repo_name}"""
   hf_api.create_repo(__UpperCAmelCase							,	token=__UpperCAmelCase							,	repo_type="""dataset"""							,	private=__UpperCAmelCase      )
   hf_api.upload_file(
       token=__UpperCAmelCase							,	path_or_fileobj=str(__UpperCAmelCase      )							,	path_in_repo="""data/text_data.txt"""							,	repo_id=__UpperCAmelCase							,	repo_type="""dataset"""							,	)
   yield repo_id
   try:
      hf_api.delete_repo(__UpperCAmelCase							,	token=__UpperCAmelCase							,	repo_type="""dataset"""      )
   except (requests.exceptions.HTTPError, ValueError):  # catch http error and token invalid error
      pass

@pytest.fixture()
def 						__magic_name__       (							__UpperCAmelCase							,	__UpperCAmelCase							,	__UpperCAmelCase      )						->					Optional[Any]:

   '''simple docstring'''







   return hf_private_dataset_repo_txt_data_

@pytest.fixture(scope="""session"""      )
def 						__magic_name__       (							__UpperCAmelCase							,	__UpperCAmelCase							,	__UpperCAmelCase      )						->					Dict:

   '''simple docstring'''







   __SCREAMING_SNAKE_CASE      							=       f"""repo_zipped_txt_data-{int(time.time() * 1_0e3      )}"""
   __SCREAMING_SNAKE_CASE      							=       f"""{CI_HUB_USER}/{repo_name}"""
   hf_api.create_repo(__UpperCAmelCase							,	token=__UpperCAmelCase							,	repo_type="""dataset"""							,	private=__UpperCAmelCase      )
   hf_api.upload_file(
       token=__UpperCAmelCase							,	path_or_fileobj=str(__UpperCAmelCase      )							,	path_in_repo="""data.zip"""							,	repo_id=__UpperCAmelCase							,	repo_type="""dataset"""							,	)
   yield repo_id
   try:
      hf_api.delete_repo(__UpperCAmelCase							,	token=__UpperCAmelCase							,	repo_type="""dataset"""      )
   except (requests.exceptions.HTTPError, ValueError):  # catch http error and token invalid error
      pass

@pytest.fixture()
def 						__magic_name__       (							__UpperCAmelCase							,	__UpperCAmelCase							,	__UpperCAmelCase      )						->					Tuple:

   '''simple docstring'''







   return hf_private_dataset_repo_zipped_txt_data_

@pytest.fixture(scope="""session"""      )
def 						__magic_name__       (							__UpperCAmelCase							,	__UpperCAmelCase							,	__UpperCAmelCase      )						->					Union[str, Any]:

   '''simple docstring'''







   __SCREAMING_SNAKE_CASE      							=       f"""repo_zipped_img_data-{int(time.time() * 1_0e3      )}"""
   __SCREAMING_SNAKE_CASE      							=       f"""{CI_HUB_USER}/{repo_name}"""
   hf_api.create_repo(__UpperCAmelCase							,	token=__UpperCAmelCase							,	repo_type="""dataset"""							,	private=__UpperCAmelCase      )
   hf_api.upload_file(
       token=__UpperCAmelCase							,	path_or_fileobj=str(__UpperCAmelCase      )							,	path_in_repo="""data.zip"""							,	repo_id=__UpperCAmelCase							,	repo_type="""dataset"""							,	)
   yield repo_id
   try:
      hf_api.delete_repo(__UpperCAmelCase							,	token=__UpperCAmelCase							,	repo_type="""dataset"""      )
   except (requests.exceptions.HTTPError, ValueError):  # catch http error and token invalid error
      pass







@pytest.fixture()
def 						__magic_name__       (							__UpperCAmelCase							,	__UpperCAmelCase							,	__UpperCAmelCase      )						->					Dict:

   '''simple docstring'''







   return hf_private_dataset_repo_zipped_img_data_





