


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


A   :					Union[str, Any]	=      logging.get_logger(__name__)

A   :					List[Any]	=      {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

A   :					List[Any]	=      [
    "small",
    "small-base",
    "medium",
    "medium-base",
    "intermediate",
    "intermediate-base",
    "large",
    "large-base",
    "xlarge",
    "xlarge-base",
]

A   :					Optional[int]	=      {
    "vocab_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt",
        "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt",
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt",
        "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt",
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json",
        "funnel-transformer/small-base": (
            "https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json",
        "funnel-transformer/large-base": (
            "https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"
        ),
    },
}
A   :					Union[str, Any]	=      {f'funnel-transformer/{name}': 512 for name in _model_names}
A   :					Union[str, Any]	=      {f'funnel-transformer/{name}': {"do_lower_case": True} for name in _model_names}







class     _lowercase				(							lowercase__):


       """simple docstring"""




       A__      		=    VOCAB_FILES_NAMES
       A__      		=    PRETRAINED_VOCAB_FILES_MAP
       A__      		=    PRETRAINED_INIT_CONFIGURATION
       A__      		=    FunnelTokenizer
       A__      		=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       A__      		=    2





       def __init__(     self							:    Optional[Any]      ,   __lowerCamelCase							:    int=None      ,   __lowerCamelCase							:    Optional[int]=None      ,   __lowerCamelCase							:    Union[str, Any]=True      ,   __lowerCamelCase							:    Optional[int]="<unk>"      ,   __lowerCamelCase							:    Optional[Any]="<sep>"      ,   __lowerCamelCase							:    Tuple="<pad>"      ,   __lowerCamelCase							:    str="<cls>"      ,   __lowerCamelCase							:    List[str]="<mask>"      ,   __lowerCamelCase							:    Dict="<s>"      ,   __lowerCamelCase							:    List[str]="</s>"      ,   __lowerCamelCase							:    Dict=True      ,   __lowerCamelCase							:    List[Any]=True      ,   __lowerCamelCase							:    Optional[Any]=None      ,   __lowerCamelCase							:    List[str]="##"      ,   **__lowerCamelCase							:    Optional[int]      ,   ):


             '''simple docstring'''



             super().__init__(
                 __lowerCamelCase      ,   tokenizer_file=__lowerCamelCase      ,   do_lower_case=__lowerCamelCase      ,   unk_token=__lowerCamelCase      ,   sep_token=__lowerCamelCase      ,   pad_token=__lowerCamelCase      ,   cls_token=__lowerCamelCase      ,   mask_token=__lowerCamelCase      ,   bos_token=__lowerCamelCase      ,   eos_token=__lowerCamelCase      ,   clean_text=__lowerCamelCase      ,   tokenize_chinese_chars=__lowerCamelCase      ,   strip_accents=__lowerCamelCase      ,   wordpieces_prefix=__lowerCamelCase      ,   **__lowerCamelCase      ,   )

             lowerCamelCase__	:       str					=						json.loads(self.backend_tokenizer.normalizer.__getstate__() )
             if (
                 normalizer_state.get("lowercase"      ,   __lowerCamelCase ) != do_lower_case
                 or normalizer_state.get("strip_accents"      ,   __lowerCamelCase ) != strip_accents
                 or normalizer_state.get("handle_chinese_chars"      ,   __lowerCamelCase ) != tokenize_chinese_chars
             ):
                   lowerCamelCase__	:       int					=						getattr(__lowerCamelCase      ,   normalizer_state.pop("type" ) )
                   lowerCamelCase__	:       Dict					=						do_lower_case
                   lowerCamelCase__	:       Union[str, Any]					=						strip_accents
                   lowerCamelCase__	:       Dict					=						tokenize_chinese_chars
                   lowerCamelCase__	:       Optional[int]					=						normalizer_class(**__lowerCamelCase )

             lowerCamelCase__	:       Optional[Any]					=						do_lower_case





       def 		lowerCAmelCase  (     self							:    Tuple      ,   __lowerCamelCase							:    Dict      ,   __lowerCamelCase							:    Union[str, Any]=None ):


             '''simple docstring'''



             lowerCamelCase__	:       Tuple					=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

             if token_ids_a:
                   output += token_ids_a + [self.sep_token_id]

             return output





       def 		lowerCAmelCase  (     self							:    Dict      ,   __lowerCamelCase							:    List[int]      ,   __lowerCamelCase							:    Optional[List[int]] = None ):


             '''simple docstring'''



             lowerCamelCase__	:       List[str]					=						[self.sep_token_id]
             lowerCamelCase__	:       Optional[int]					=						[self.cls_token_id]
             if token_ids_a is None:
                   return len(cls ) * [self.cls_token_type_id] + len(token_ids_a + sep ) * [0]
             return len(cls ) * [self.cls_token_type_id] + len(token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]





       def 		lowerCAmelCase  (     self							:    List[str]      ,   __lowerCamelCase							:    str      ,   __lowerCamelCase							:    Optional[str] = None ):


             '''simple docstring'''



             lowerCamelCase__	:       List[Any]					=						self._tokenizer.model.save(__lowerCamelCase      ,   name=__lowerCamelCase )
             return tuple(__lowerCamelCase )






from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow


if is_tf_available():
    import numpy as np
    import tensorflow as tf

    from transformers import TFCamembertModel







@require_tf
@require_sentencepiece
@require_tokenizers
class     _lowercase				(							unittest.TestCase):


       """simple docstring"""




       @slow
       def 		lowerCAmelCase  (     self							:    Any ):


             '''simple docstring'''



             lowerCamelCase__	:       Dict					=						TFCamembertModel.from_pretrained("jplu/tf-camembert-base" )

             lowerCamelCase__	:       str					=						tf.convert_to_tensor(
                 [[5, 121, 11, 660, 16, 730, 25543, 110, 83, 6]]      ,   dtype=tf.intaa      ,   )  # J'aime le camembert !"

             lowerCamelCase__	:       Any					=						model(__lowerCamelCase )["last_hidden_state"]
             lowerCamelCase__	:       List[str]					=						tf.TensorShape((1, 10, 768) )
             self.assertEqual(output.shape      ,   __lowerCamelCase )
             # compare the actual values for a slice.
             lowerCamelCase__	:       str					=						tf.convert_to_tensor(
                 [[[-0.0_2_5_4, 0.0_2_3_5, 0.1_0_2_7], [0.0_6_0_6, -0.1_8_1_1, -0.0_4_1_8], [-0.1_5_6_1, -0.1_1_2_7, 0.2_6_8_7]]]      ,   dtype=tf.floataa      ,   )
             # camembert = torch.hub.load('pytorch/fairseq', 'camembert.v0')
             # camembert.eval()
             # expected_slice = roberta.model.forward(input_ids)[0][:, :3, :3].detach()

             self.assertTrue(np.allclose(output[:, :3, :3].numpy()      ,   expected_slice.numpy()      ,   atol=1E-4 ) )


