



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


UpperCAmelCase   :							Tuple							 =logging.get_logger(__name__)

UpperCAmelCase   :							Optional[Any]							 ={"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

UpperCAmelCase   :							List[Any]							 ={
    """vocab_file""": {
        """junnyu/roformer_chinese_small""": """https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt""",
        """junnyu/roformer_chinese_base""": """https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt""",
        """junnyu/roformer_chinese_char_small""": (
            """https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_chinese_char_base""": (
            """https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_small_discriminator""": (
            """https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_small_generator""": (
            """https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt"""
        ),
    }
}

UpperCAmelCase   :							List[str]							 ={
    """junnyu/roformer_chinese_small""": 1536,
    """junnyu/roformer_chinese_base""": 1536,
    """junnyu/roformer_chinese_char_small""": 512,
    """junnyu/roformer_chinese_char_base""": 512,
    """junnyu/roformer_small_discriminator""": 128,
    """junnyu/roformer_small_generator""": 128,
}


UpperCAmelCase   :							Optional[Any]							 ={
    """junnyu/roformer_chinese_small""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_base""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_char_small""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_char_base""": {"""do_lower_case""": True},
    """junnyu/roformer_small_discriminator""": {"""do_lower_case""": True},
    """junnyu/roformer_small_generator""": {"""do_lower_case""": True},
}


class       _lowercase   (a_      ):
				'''simple docstring'''







				lowercase__   						=						VOCAB_FILES_NAMES
				lowercase__   						=						PRETRAINED_VOCAB_FILES_MAP
				lowercase__   						=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowercase__   						=						PRETRAINED_INIT_CONFIGURATION
				lowercase__   						=						RoFormerTokenizer

				def __init__( self     , snake_case__=None     , snake_case__=None     , snake_case__=True     , snake_case__="[UNK]"     , snake_case__="[SEP]"     , snake_case__="[PAD]"     , snake_case__="[CLS]"     , snake_case__="[MASK]"     , snake_case__=True     , snake_case__=None     , **snake_case__     , ):

									'''simple docstring'''


									super().__init__(
									    snake_case__     , tokenizer_file=snake_case__     , do_lower_case=snake_case__     , unk_token=snake_case__     , sep_token=snake_case__     , pad_token=snake_case__     , cls_token=snake_case__     , mask_token=snake_case__     , tokenize_chinese_chars=snake_case__     , strip_accents=snake_case__     , **snake_case__     , )

									UpperCamelCase_		  =   json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
									if (
									    pre_tok_state.get("lowercase"     , snake_case__    ) != do_lower_case
									    or pre_tok_state.get("strip_accents"     , snake_case__    ) != strip_accents
									):
														UpperCamelCase_		  =   getattr(snake_case__     , pre_tok_state.pop("type"    )    )
														UpperCamelCase_		  =   do_lower_case
														UpperCamelCase_		  =   strip_accents
														UpperCamelCase_		  =   pre_tok_class(**snake_case__    )

									UpperCamelCase_		  =   do_lower_case

				def __getstate__( self    ):

									'''simple docstring'''


									UpperCamelCase_		  =   self.__dict__.copy()
									UpperCamelCase_		  =   BertPreTokenizer()
									return state

				def __setstate__( self     , snake_case__    ):

									'''simple docstring'''


									UpperCamelCase_		  =   d
									UpperCamelCase_		  =   self.__dict__["_tokenizer"].get_vocab()
									UpperCamelCase_		  =   PreTokenizer.custom(JiebaPreTokenizer(snake_case__    )    )

				def       _lowerCamelCase					( self     , snake_case__     , snake_case__=None    ):

									'''simple docstring'''


									UpperCamelCase_		  =   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

									if token_ids_a:
														output += token_ids_a + [self.sep_token_id]

									return output

				def       _lowerCamelCase					( self     , snake_case__     , snake_case__ = None    ):

									'''simple docstring'''


									UpperCamelCase_		  =   [self.sep_token_id]
									UpperCamelCase_		  =   [self.cls_token_id]
									if token_ids_a is None:
														return len(cls + token_ids_a + sep    ) * [0]
									return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]

				def       _lowerCamelCase					( self     , snake_case__     , snake_case__ = None    ):

									'''simple docstring'''


									UpperCamelCase_		  =   self._tokenizer.model.save(snake_case__     , name=snake_case__    )
									return tuple(snake_case__    )

				def       _lowerCamelCase					( self     , snake_case__     , snake_case__=None     , snake_case__=None     , snake_case__=False     , **snake_case__     , ):

									'''simple docstring'''


									UpperCamelCase_		  =   BertPreTokenizer()
									return super().save_pretrained(snake_case__     , snake_case__     , snake_case__     , snake_case__     , **snake_case__    )





# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from ..models.auto import AutoModelForSeqaSeqLM, AutoTokenizer
from .base import PipelineTool


class       _lowercase   (a_      ):
				'''simple docstring'''







				lowercase__   						=						"""philschmid/bart-large-cnn-samsum"""
				lowercase__   						=						(
				    """This is a tool that summarizes an English text. It takes an input `text` containing the text to summarize, """
				    """and returns a summary of the text."""
				)
				lowercase__   						=						"""summarizer"""
				lowercase__   						=						AutoTokenizer
				lowercase__   						=						AutoModelForSeqaSeqLM

				lowercase__   						=						["""text"""]
				lowercase__   						=						["""text"""]

				def       _lowerCamelCase					( self     , snake_case__    ):

									'''simple docstring'''


									return self.pre_processor(snake_case__     , return_tensors="pt"     , truncation=snake_case__    )

				def       _lowerCamelCase					( self     , snake_case__    ):

									'''simple docstring'''


									return self.model.generate(**snake_case__    )[0]

				def       _lowerCamelCase					( self     , snake_case__    ):

									'''simple docstring'''


									return self.pre_processor.decode(snake_case__     , skip_special_tokens=snake_case__     , clean_up_tokenization_spaces=snake_case__    )
