'''simple docstring'''




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


_A								=   logging.get_logger(__name__)

_A								=   {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

_A								=   [
    'small',
    'small-base',
    'medium',
    'medium-base',
    'intermediate',
    'intermediate-base',
    'large',
    'large-base',
    'xlarge',
    'xlarge-base',
]

_A								=   {
    'vocab_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt',
        'funnel-transformer/small-base': 'https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt',
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt',
        'funnel-transformer/large-base': 'https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt',
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json',
        'funnel-transformer/small-base': (
            'https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json',
        'funnel-transformer/large-base': (
            'https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'
        ),
    },
}
_A								=   {F"""funnel-transformer/{name}""": 5_1_2 for name in _model_names}
_A								=   {F"""funnel-transformer/{name}""": {'do_lower_case': True} for name in _model_names}


class       UpperCAmelCase__  (       _snake_case			):






   """simple docstring"""
   A							:       Tuple     =    VOCAB_FILES_NAMES
   A							:       Optional[int]     =    PRETRAINED_VOCAB_FILES_MAP
   A							:       Optional[Any]     =    PRETRAINED_INIT_CONFIGURATION
   A							:       str     =    FunnelTokenizer
   A							:       Optional[int]     =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   A							:       int     =    2
   def __init__(self       ,				_a=None       ,				_a=None       ,				_a=True       ,				_a="<unk>"       ,				_a="<sep>"       ,				_a="<pad>"       ,				_a="<cls>"       ,				_a="<mask>"       ,				_a="<s>"       ,				_a="</s>"       ,				_a=True       ,				_a=True       ,				_a=None       ,				_a="##"       ,				**_a       ,				)    ->     List[Any]:
        super().__init__(
            _a       ,				tokenizer_file=_a       ,				do_lower_case=_a       ,				unk_token=_a       ,				sep_token=_a       ,				pad_token=_a       ,				cls_token=_a       ,				mask_token=_a       ,				bos_token=_a       ,				eos_token=_a       ,				clean_text=_a       ,				tokenize_chinese_chars=_a       ,				strip_accents=_a       ,				wordpieces_prefix=_a       ,				**_a       ,				)

        lowercase_					:       str     				=					json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
        if (
            normalizer_state.get('lowercase'       ,				_a				) != do_lower_case
            or normalizer_state.get('strip_accents'       ,				_a				) != strip_accents
            or normalizer_state.get('handle_chinese_chars'       ,				_a				) != tokenize_chinese_chars
        ):
             lowercase_					:       Dict     				=					getattr(_a       ,				normalizer_state.pop('type'				)				)
             lowercase_					:       int     				=					do_lower_case
             lowercase_					:       Any     				=					strip_accents
             lowercase_					:       Optional[Any]     				=					tokenize_chinese_chars
             lowercase_					:       Optional[int]     				=					normalizer_class(**_a				)

        lowercase_					:       Dict     				=					do_lower_case
   def 		_lowerCamelCase (self       ,				_a       ,				_a=None				)    ->     Optional[int]:
        lowercase_					:       Optional[Any]     				=					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
             output += token_ids_a + [self.sep_token_id]

        return output
   def 		_lowerCamelCase (self       ,				_a       ,				_a = None				)    ->     List[int]:
        lowercase_					:       Union[str, Any]     				=					[self.sep_token_id]
        lowercase_					:       int     				=					[self.cls_token_id]
        if token_ids_a is None:
             return len(cls				) * [self.cls_token_type_id] + len(token_ids_a + sep				) * [0]
        return len(cls				) * [self.cls_token_type_id] + len(token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]





   def 		_lowerCamelCase (self       ,				_a       ,				_a = None				)    ->     Tuple[str]:
        lowercase_					:       List[str]     				=					self._tokenizer.model.save(_a       ,				name=_a				)
        return tuple(_a				)





'''simple docstring'''




import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin


class       UpperCAmelCase__  (       _snake_case					,     unittest.TestCase			):






   """simple docstring"""
   A							:       str     =    TransfoXLTokenizer
   A							:       Optional[int]     =    False
   A							:       List[str]     =    False
   def 		_lowerCamelCase (self				)    ->     List[Any]:
        super().setUp()

        lowercase_					:       Any     				=					[
            '<unk>',
            '[CLS]',
            '[SEP]',
            'want',
            'unwanted',
            'wa',
            'un',
            'running',
            ',',
            'low',
            'l',
        ]
        lowercase_					:       Any     				=					os.path.join(self.tmpdirname       ,				VOCAB_FILES_NAMES['vocab_file']				)
        with open(self.vocab_file       ,				'w'       ,				encoding='utf-8'				) as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]				)				)
   def 		_lowerCamelCase (self       ,				**_a				)    ->     Union[str, Any]:
        lowercase_					:       str     				=					True
        return TransfoXLTokenizer.from_pretrained(self.tmpdirname       ,				**_a				)
   def 		_lowerCamelCase (self       ,				_a				)    ->     Union[str, Any]:
        lowercase_					:       int     				=					'<unk> UNwanted , running'
        lowercase_					:       int     				=					'<unk> unwanted, running'
        return input_text, output_text
   def 		_lowerCamelCase (self				)    ->     Tuple:
        lowercase_					:       Any     				=					TransfoXLTokenizer(vocab_file=self.vocab_file       ,				lower_case=_a				)

        lowercase_					:       Optional[Any]     				=					tokenizer.tokenize('<unk> UNwanted , running'				)
        self.assertListEqual(_a       ,				['<unk>', 'unwanted', ',', 'running']				)

        self.assertListEqual(tokenizer.convert_tokens_to_ids(_a				)       ,				[0, 4, 8, 7]				)
   def 		_lowerCamelCase (self				)    ->     Union[str, Any]:
        lowercase_					:       List[str]     				=					TransfoXLTokenizer(lower_case=_a				)

        self.assertListEqual(
            tokenizer.tokenize(' \tHeLLo ! how  \n Are yoU ?  '				)       ,				['hello', '!', 'how', 'are', 'you', '?']				)
   def 		_lowerCamelCase (self				)    ->     int:
        lowercase_					:       Dict     				=					TransfoXLTokenizer(lower_case=_a				)

        self.assertListEqual(
            tokenizer.tokenize(' \tHeLLo ! how  \n Are yoU ?  '				)       ,				['HeLLo', '!', 'how', 'Are', 'yoU', '?']				)
   def 		_lowerCamelCase (self				)    ->     Optional[Any]:
        lowercase_					:       Dict     				=					TransfoXLTokenizer(lower_case=_a				)
        lowercase_					:       int     				=					'Hello (bracket) and side-scrolled [and] Henry\'s $5,000 with 3.34 m. What\'s up!?'
        lowercase_					:       List[str]     				=					[
            'Hello',
            '(',
            'bracket',
            ')',
            'and',
            'side',
            '@-@',
            'scrolled',
            '[',
            'and',
            ']',
            'Henry',
            '\'s',
            '$',
            '5',
            '@,@',
            '000',
            'with',
            '3',
            '@.@',
            '34',
            'm',
            '.',
            'What',
            '\'s',
            'up',
            '!',
            '?',
        ]

        self.assertListEqual(tokenizer.tokenize(_a				)       ,				_a				)

        self.assertEqual(tokenizer.convert_tokens_to_string(_a				)       ,				_a				)





   def 		_lowerCamelCase (self				)    ->     Any:
        lowercase_					:       Optional[int]     				=					self.get_tokenizer()
        lowercase_					:       str     				=					len(_a				)

        tokenizer.add_tokens(['new1', 'new2']				)
        tokenizer.move_added_token('new1'       ,				1				)

        # Check that moved token is not copied (duplicate)
        self.assertEqual(len(_a				)       ,				original_len + 2				)
        # Check that token is moved to specified id
        self.assertEqual(tokenizer.encode('new1'				)       ,				[1]				)
        self.assertEqual(tokenizer.decode([1]				)       ,				'new1'				)




