
"""simple docstring"""





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


UpperCamelCase_					:						List[Any]				= logging.get_logger(__name__)

UpperCamelCase_					:						List[str]				= {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCamelCase_					:						Optional[int]				= {
    '''vocab_file''': {
        '''junnyu/roformer_chinese_small''': '''https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt''',
        '''junnyu/roformer_chinese_base''': '''https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt''',
        '''junnyu/roformer_chinese_char_small''': (
            '''https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_chinese_char_base''': (
            '''https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_small_discriminator''': (
            '''https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_small_generator''': (
            '''https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'''
        ),
    }
}

UpperCamelCase_					:						Optional[int]				= {
    '''junnyu/roformer_chinese_small''': 1536,
    '''junnyu/roformer_chinese_base''': 1536,
    '''junnyu/roformer_chinese_char_small''': 512,
    '''junnyu/roformer_chinese_char_base''': 512,
    '''junnyu/roformer_small_discriminator''': 128,
    '''junnyu/roformer_small_generator''': 128,
}


UpperCamelCase_					:						Optional[Any]				= {
    '''junnyu/roformer_chinese_small''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_base''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_char_small''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_char_base''': {'''do_lower_case''': True},
    '''junnyu/roformer_small_discriminator''': {'''do_lower_case''': True},
    '''junnyu/roformer_small_generator''': {'''do_lower_case''': True},
}



class __lowerCAmelCase				(					_lowercase      ):




   """simple docstring"""







   snake_case	      =				VOCAB_FILES_NAMES
   snake_case	      =				PRETRAINED_VOCAB_FILES_MAP
   snake_case	      =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   snake_case	      =				PRETRAINED_INIT_CONFIGURATION
   snake_case	      =				RoFormerTokenizer




   def __init__(				self						:					List[Any]       ,   _snake_case						:					List[Any]=None       ,   _snake_case						:					str=None       ,   _snake_case						:					str=True       ,   _snake_case						:					Optional[int]="[UNK]"       ,   _snake_case						:					List[Any]="[SEP]"       ,   _snake_case						:					List[Any]="[PAD]"       ,   _snake_case						:					Tuple="[CLS]"       ,   _snake_case						:					List[Any]="[MASK]"       ,   _snake_case						:					Optional[Any]=True       ,   _snake_case						:					List[Any]=None       ,   **_snake_case						:					Any       ,   )      -> str:







        """simple docstring"""
        super().__init__(
            _snake_case       ,   tokenizer_file=_snake_case       ,   do_lower_case=_snake_case       ,   unk_token=_snake_case       ,   sep_token=_snake_case       ,   pad_token=_snake_case       ,   cls_token=_snake_case       ,   mask_token=_snake_case       ,   tokenize_chinese_chars=_snake_case       ,   strip_accents=_snake_case       ,   **_snake_case       ,   )

        A_				 =						json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
        if (
            pre_tok_state.get("lowercase"       ,   _snake_case	) != do_lower_case
            or pre_tok_state.get("strip_accents"       ,   _snake_case	) != strip_accents
        ):
             A_				 =						getattr(_snake_case       ,   pre_tok_state.pop("type"	)	)
             A_				 =						do_lower_case
             A_				 =						strip_accents
             A_				 =						pre_tok_class(**_snake_case	)

        A_				 =						do_lower_case




   def __getstate__(				self						:					Optional[int]	)      -> str:







        """simple docstring"""
        A_				 =						self.__dict__.copy()
        A_				 =						BertPreTokenizer()
        return state




   def __setstate__(				self						:					Optional[int]       ,   _snake_case						:					Optional[Any]	)      -> Union[str, Any]:







        """simple docstring"""
        A_				 =						d
        A_				 =						self.__dict__["_tokenizer"].get_vocab()
        A_				 =						PreTokenizer.custom(JiebaPreTokenizer(_snake_case	)	)




   def      lowerCamelCase__     (				self						:					Dict       ,   _snake_case						:					Any       ,   _snake_case						:					Optional[Any]=None	)      -> int:







        """simple docstring"""
        A_				 =						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
             output += token_ids_a + [self.sep_token_id]

        return output




   def      lowerCamelCase__     (				self						:					int       ,   _snake_case						:					List[int]       ,   _snake_case						:					Optional[List[int]] = None	)      -> List[int]:







        """simple docstring"""
        A_				 =						[self.sep_token_id]
        A_				 =						[self.cls_token_id]
        if token_ids_a is None:
             return len(cls + token_ids_a + sep	) * [0]
        return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]




   def      lowerCamelCase__     (				self						:					List[str]       ,   _snake_case						:					str       ,   _snake_case						:					Optional[str] = None	)      -> Tuple[str]:







        """simple docstring"""
        A_				 =						self._tokenizer.model.save(_snake_case       ,   name=_snake_case	)
        return tuple(_snake_case	)






   def      lowerCamelCase__     (				self						:					str       ,   _snake_case						:					Dict       ,   _snake_case						:					Dict=None       ,   _snake_case						:					int=None       ,   _snake_case						:					Any=False       ,   **_snake_case						:					int       ,   )      -> str:







        """simple docstring"""
        A_				 =						BertPreTokenizer()
        return super().save_pretrained(_snake_case       ,   _snake_case       ,   _snake_case       ,   _snake_case       ,   **_snake_case	)






"""simple docstring"""





import unittest

from transformers import load_tool

from .test_tools_common import ToolTesterMixin


UpperCamelCase_					:						int				= '''
Hugging Face was founded in 2016 by French entrepreneurs Clément Delangue, Julien Chaumond, and Thomas Wolf originally as a company that developed a chatbot app targeted at teenagers.[2] After open-sourcing the model behind the chatbot, the company pivoted to focus on being a platform for machine learning.

In March 2021, Hugging Face raised $40 million in a Series B funding round.[3]

On April 28, 2021, the company launched the BigScience Research Workshop in collaboration with several other research groups to release an open large language model.[4] In 2022, the workshop concluded with the announcement of BLOOM, a multilingual large language model with 176 billion parameters.[5]
'''



class __lowerCAmelCase				(					unittest.TestCase     ,				_lowercase      ):




			"""simple docstring"""







			def      lowerCamelCase__     (				self						:					str	)      -> Dict:







								"""simple docstring"""
								A_				 =						load_tool("text-question-answering"	)
								self.tool.setup()
								A_				 =						load_tool("text-question-answering"       ,   remote=_snake_case	)




			def      lowerCamelCase__     (				self						:					Union[str, Any]	)      -> Any:







								"""simple docstring"""
								A_				 =						self.tool(_snake_case       ,   "What did Hugging Face do in April 2021?"	)
								self.assertEqual(_snake_case       ,   "launched the BigScience Research Workshop"	)




			def      lowerCamelCase__     (				self						:					Union[str, Any]	)      -> int:







								"""simple docstring"""
								A_				 =						self.remote_tool(_snake_case       ,   "What did Hugging Face do in April 2021?"	)
								self.assertEqual(_snake_case       ,   "launched the BigScience Research Workshop"	)




			def      lowerCamelCase__     (				self						:					Optional[Any]	)      -> Optional[Any]:







								"""simple docstring"""
								A_				 =						self.tool(text=_snake_case       ,   question="What did Hugging Face do in April 2021?"	)
								self.assertEqual(_snake_case       ,   "launched the BigScience Research Workshop"	)






			def      lowerCamelCase__     (				self						:					str	)      -> Optional[Any]:







								"""simple docstring"""
								A_				 =						self.remote_tool(text=_snake_case       ,   question="What did Hugging Face do in April 2021?"	)
								self.assertEqual(_snake_case       ,   "launched the BigScience Research Workshop"	)




