




"""simple docstring"""


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_squeezebert import SqueezeBertTokenizer


A_									=       logging.get_logger(__name__)

A_									=       {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

A_									=       {
    '''vocab_file''': {
        '''squeezebert/squeezebert-uncased''': (
            '''https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/vocab.txt'''
        ),
        '''squeezebert/squeezebert-mnli''': '''https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/vocab.txt''',
        '''squeezebert/squeezebert-mnli-headless''': (
            '''https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''squeezebert/squeezebert-uncased''': (
            '''https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/tokenizer.json'''
        ),
        '''squeezebert/squeezebert-mnli''': (
            '''https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/tokenizer.json'''
        ),
        '''squeezebert/squeezebert-mnli-headless''': (
            '''https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/tokenizer.json'''
        ),
    },
}

A_									=       {
    '''squeezebert/squeezebert-uncased''': 5_12,
    '''squeezebert/squeezebert-mnli''': 5_12,
    '''squeezebert/squeezebert-mnli-headless''': 5_12,
}


A_									=       {
    '''squeezebert/squeezebert-uncased''': {'''do_lower_case''': True},
    '''squeezebert/squeezebert-mnli''': {'''do_lower_case''': True},
    '''squeezebert/squeezebert-mnli-headless''': {'''do_lower_case''': True},
}






class       lowercase(  __SCREAMING_SNAKE_CASE				):







 '''simple docstring'''





 lowercase__				      =      VOCAB_FILES_NAMES
 lowercase__				      =      PRETRAINED_VOCAB_FILES_MAP
 lowercase__				      =      PRETRAINED_INIT_CONFIGURATION
 lowercase__				      =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowercase__				      =      SqueezeBertTokenizer


 def __init__(       self:	Optional[int],   a_:	Optional[Any]=None,   a_:	Optional[int]=None,   a_:	str=True,   a_:	Any="[UNK]",   a_:	str="[SEP]",   a_:	Optional[Any]="[PAD]",   a_:	List[str]="[CLS]",   a_:	List[str]="[MASK]",   a_:	Optional[Any]=True,   a_:	Optional[Any]=None,   **a_:	Any,   ):




   '''simple docstring'''



   super().__init__(
       _a,   tokenizer_file=_a,   do_lower_case=_a,   unk_token=_a,   sep_token=_a,   pad_token=_a,   cls_token=_a,   mask_token=_a,   tokenize_chinese_chars=_a,   strip_accents=_a,   **_a,   )

   _snake_case							:	Union[str, Any]			=						json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
   if (
       normalizer_state.get("""lowercase""",   _a	) != do_lower_case
       or normalizer_state.get("""strip_accents""",   _a	) != strip_accents
       or normalizer_state.get("""handle_chinese_chars""",   _a	) != tokenize_chinese_chars
   ):
     _snake_case							:	List[Any]			=						getattr(_a,   normalizer_state.pop("""type"""	)	)
     _snake_case							:	Union[str, Any]			=						do_lower_case
     _snake_case							:	List[str]			=						strip_accents
     _snake_case							:	Dict			=						tokenize_chinese_chars
     _snake_case							:	List[str]			=						normalizer_class(**_a	)

   _snake_case							:	Any			=						do_lower_case


 def 				UpperCamelCase_     (       self:	int,   a_:	Dict,   a_:	Dict=None	):




   '''simple docstring'''



   _snake_case							:	Any			=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

   if token_ids_a:
     output += token_ids_a + [self.sep_token_id]

   return output


 def 				UpperCamelCase_     (       self:	Any,   a_:	List[int],   a_:	Optional[List[int]] = None	):




   '''simple docstring'''



   _snake_case							:	Dict			=						[self.sep_token_id]
   _snake_case							:	Union[str, Any]			=						[self.cls_token_id]
   if token_ids_a is None:
     return len(cls + token_ids_a + sep	) * [0]
   return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]


 def 				UpperCamelCase_     (       self:	Optional[int],   a_:	str,   a_:	Optional[str] = None	):




   '''simple docstring'''



   _snake_case							:	Union[str, Any]			=						self._tokenizer.model.save(_a,   name=_a	)
   return tuple(_a	)








"""simple docstring"""


import os
import sys
import unittest


A_      			=       os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, '''utils'''))

import check_dummies  # noqa: E402
from check_dummies import create_dummy_files, create_dummy_object, find_backend, read_init  # noqa: E402


# Align TRANSFORMERS_PATH in check_dummies with the current path
A_      			=       os.path.join(git_repo_path, '''src''', '''diffusers''')






class       lowercase(  unittest.TestCase				):







 '''simple docstring'''





 def 				UpperCamelCase_     (       self:	Tuple	):




   '''simple docstring'''



   _snake_case							:	Optional[int]			=						find_backend("""    if not is_torch_available():"""	)
   self.assertEqual(a_,   """torch"""	)

   # backend_with_underscore = find_backend("    if not is_tensorflow_text_available():")
   # self.assertEqual(backend_with_underscore, "tensorflow_text")

   _snake_case							:	Any			=						find_backend("""    if not (is_torch_available() and is_transformers_available()):"""	)
   self.assertEqual(a_,   """torch_and_transformers"""	)

   # double_backend_with_underscore = find_backend(
   #    "    if not (is_sentencepiece_available() and is_tensorflow_text_available()):"
   # )
   # self.assertEqual(double_backend_with_underscore, "sentencepiece_and_tensorflow_text")

   _snake_case							:	Union[str, Any]			=						find_backend(
       """    if not (is_torch_available() and is_transformers_available() and is_onnx_available()):"""	)
   self.assertEqual(a_,   """torch_and_transformers_and_onnx"""	)


 def 				UpperCamelCase_     (       self:	List[Any]	):




   '''simple docstring'''



   _snake_case							:	Dict			=						read_init()
   # We don't assert on the exact list of keys to allow for smooth grow of backend-specific objects
   self.assertIn("""torch""",   a_	)
   self.assertIn("""torch_and_transformers""",   a_	)
   self.assertIn("""flax_and_transformers""",   a_	)
   self.assertIn("""torch_and_transformers_and_onnx""",   a_	)

   # Likewise, we can't assert on the exact content of a key
   self.assertIn("""UNet2DModel""",   objects["""torch"""]	)
   self.assertIn("""FlaxUNet2DConditionModel""",   objects["""flax"""]	)
   self.assertIn("""StableDiffusionPipeline""",   objects["""torch_and_transformers"""]	)
   self.assertIn("""FlaxStableDiffusionPipeline""",   objects["""flax_and_transformers"""]	)
   self.assertIn("""LMSDiscreteScheduler""",   objects["""torch_and_scipy"""]	)
   self.assertIn("""OnnxStableDiffusionPipeline""",   objects["""torch_and_transformers_and_onnx"""]	)


 def 				UpperCamelCase_     (       self:	List[str]	):




   '''simple docstring'''



   _snake_case							:	Optional[Any]			=						create_dummy_object("""CONSTANT""",   """'torch'"""	)
   self.assertEqual(a_,   """\nCONSTANT = None\n"""	)

   _snake_case							:	Optional[int]			=						create_dummy_object("""function""",   """'torch'"""	)
   self.assertEqual(
       a_,   """\ndef function(*args, **kwargs):\n    requires_backends(function, 'torch')\n"""	)

   _snake_case							:	List[Any]			=						"""
class FakeClass(metaclass=DummyObject):
    _backends = 'torch'

    def __init__(self, *args, **kwargs):
        requires_backends(self, 'torch')

    @classmethod
    def from_config(cls, *args, **kwargs):
        requires_backends(cls, 'torch')

    @classmethod
    def from_pretrained(cls, *args, **kwargs):
        requires_backends(cls, 'torch')
"""
   _snake_case							:	Union[str, Any]			=						create_dummy_object("""FakeClass""",   """'torch'"""	)
   self.assertEqual(a_,   a_	)


 def 				UpperCamelCase_     (       self:	Union[str, Any]	):




   '''simple docstring'''



   _snake_case							:	Union[str, Any]			=						"""# This file is autogenerated by the command `make fix-copies`, do not edit.
from ..utils import DummyObject, requires_backends


CONSTANT = None


def function(*args, **kwargs):
    requires_backends(function, [\"torch\"])


class FakeClass(metaclass=DummyObject):
    _backends = [\"torch\"]

    def __init__(self, *args, **kwargs):
        requires_backends(self, [\"torch\"])

    @classmethod
    def from_config(cls, *args, **kwargs):
        requires_backends(cls, [\"torch\"])

    @classmethod
    def from_pretrained(cls, *args, **kwargs):
        requires_backends(cls, [\"torch\"])
"""
   _snake_case							:	List[Any]			=						create_dummy_files({"""torch""": ["""CONSTANT""", """function""", """FakeClass"""]}	)
   self.assertEqual(dummy_files["""torch"""],   a_	)


