



import enum
import warnings

from ..tokenization_utils import TruncationStrategy
from ..utils import add_end_docstrings, is_tf_available, is_torch_available, logging
from .base import PIPELINE_INIT_ARGS, Pipeline


if is_tf_available():
     import tensorflow as tf

     from ..models.auto.modeling_tf_auto import TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING

if is_torch_available():
     from ..models.auto.modeling_auto import MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING

a_						:     Dict	     =		logging.get_logger(__name__)





class    lowerCamelCase__  (						enum.Enum):
     """simple docstring"""



     _A       =			0
     _A       =			1





@add_end_docstrings(UpperCAmelCase_)
class    lowerCamelCase__  (						UpperCAmelCase_):
     """simple docstring"""



     _A       =			'generated'







     def __init__(self				,			*__a				,			**__a    ):






           '''simple docstring'''



           super().__init__(*lowercase_				,			**lowercase_    )

           self.check_model_type(
               TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
               if self.framework == "tf"
               else MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING    )







     def 						_a      (self				,			__a=None				,			__a=None				,			__a=None				,			__a=None				,			__a=None				,			__a=None				,			**__a				,			):






           '''simple docstring'''



           lowerCamelCase								=		{}
           if truncation is not None:
                 lowerCamelCase								=		truncation

           lowerCamelCase								=		generate_kwargs

           lowerCamelCase								=		{}
           if return_tensors is not None and return_type is None:
                 lowerCamelCase								=		ReturnType.TENSORS if return_tensors else ReturnType.TEXT
           if return_type is not None:
                 lowerCamelCase								=		return_type

           if clean_up_tokenization_spaces is not None:
                 lowerCamelCase								=		clean_up_tokenization_spaces

           if stop_sequence is not None:
                 lowerCamelCase								=		self.tokenizer.encode(lowercase_				,			add_special_tokens=lowercase_    )
                 if len(lowercase_    ) > 1:
                       warnings.warn(
                           "Stopping on a multiple token sequence is not yet supported on transformers. The first token of"
                           " the stop sequence will be used as the stop sequence string in the interim."    )
                 lowerCamelCase								=		stop_sequence_ids[0]

           return preprocess_params, forward_params, postprocess_params







     def 						_a      (self				,			__a				,			__a				,			__a    ):






           '''simple docstring'''



           return True







     def 						_a      (self				,			*__a				,			__a    ):






           '''simple docstring'''



           lowerCamelCase								=		self.model.config.prefix if self.model.config.prefix is not None else ""
           if isinstance(args[0]				,			lowercase_    ):
                 if self.tokenizer.pad_token_id is None:
                       raise ValueError("Please make sure that the tokenizer has a pad_token_id when using a batch input"    )
                 lowerCamelCase								=		([prefix + arg for arg in args[0]],)
                 lowerCamelCase								=		True

           elif isinstance(args[0]				,			lowercase_    ):
                 lowerCamelCase								=		(prefix + args[0],)
                 lowerCamelCase								=		False
           else:
                 raise ValueError(
                     F""" `args[0]`: {args[0]} have the wrong format. The should be either of type `str` or type `list`"""    )
           lowerCamelCase								=		self.tokenizer(*lowercase_				,			padding=lowercase_				,			truncation=lowercase_				,			return_tensors=self.framework    )
           # This is produced by tokenizers but is an invalid generate kwargs
           if "token_type_ids" in inputs:
                 del inputs["token_type_ids"]
           return inputs







     def __call__(self				,			*__a				,			**__a    ):






           '''simple docstring'''



           lowerCamelCase								=		super().__call__(*lowercase_				,			**lowercase_    )
           if (
               isinstance(args[0]				,			lowercase_    )
               and all(isinstance(lowercase_				,			lowercase_    ) for el in args[0]    )
               and all(len(lowercase_    ) == 1 for res in result    )
           ):
                 return [res[0] for res in result]
           return result







     def 						_a      (self				,			__a				,			__a=TruncationStrategy.DO_NOT_TRUNCATE				,			**__a    ):






           '''simple docstring'''



           lowerCamelCase								=		self._parse_and_tokenize(lowercase_				,			truncation=lowercase_				,			**lowercase_    )
           return inputs







     def 						_a      (self				,			__a				,			**__a    ):






           '''simple docstring'''



           if self.framework == "pt":
                 lowerCamelCase			,							lowerCamelCase								=		model_inputs["input_ids"].shape
           elif self.framework == "tf":
                 lowerCamelCase			,							lowerCamelCase								=		tf.shape(model_inputs["input_ids"]    ).numpy()

           lowerCamelCase								=		generate_kwargs.get("min_length"				,			self.model.config.min_length    )
           lowerCamelCase								=		generate_kwargs.get("max_length"				,			self.model.config.max_length    )
           self.check_inputs(lowercase_				,			generate_kwargs["min_length"]				,			generate_kwargs["max_length"]    )
           lowerCamelCase								=		self.model.generate(**lowercase_				,			**lowercase_    )
           lowerCamelCase								=		output_ids.shape[0]
           if self.framework == "pt":
                 lowerCamelCase								=		output_ids.reshape(lowercase_				,			out_b // in_b				,			*output_ids.shape[1:]    )
           elif self.framework == "tf":
                 lowerCamelCase								=		tf.reshape(lowercase_				,			(in_b, out_b // in_b, *output_ids.shape[1:])    )
           return {"output_ids": output_ids}







     def 						_a      (self				,			__a				,			__a=ReturnType.TEXT				,			__a=False    ):






           '''simple docstring'''



           lowerCamelCase								=		[]
           for output_ids in model_outputs["output_ids"][0]:
                 if return_type == ReturnType.TENSORS:
                       lowerCamelCase								=		{F"""{self.return_name}_token_ids""": output_ids}
                 elif return_type == ReturnType.TEXT:
                       lowerCamelCase								=		{
                           F"""{self.return_name}_text""": self.tokenizer.decode(
                               lowercase_				,			skip_special_tokens=lowercase_				,			clean_up_tokenization_spaces=lowercase_				,			)
                       }
                 records.append(lowercase_    )
           return records





@add_end_docstrings(UpperCAmelCase_)
class    lowerCamelCase__  (						UpperCAmelCase_):
     """simple docstring"""



     _A       =			'summary'







     def __call__(self				,			*__a				,			**__a    ):






           '''simple docstring'''



           return super().__call__(*lowercase_				,			**lowercase_    )







     def 						_a      (self				,			__a				,			__a				,			__a    ):






           '''simple docstring'''



           if max_length < min_length:
                 logger.warning(F"""Your min_length={min_length} must be inferior than your max_length={max_length}."""    )

           if input_length < max_length:
                 logger.warning(
                     F"""Your max_length is set to {max_length}, but your input_length is only {input_length}. Since this is """
                     "a summarization task, where outputs shorter than the input are typically wanted, you might "
                     F"""consider decreasing max_length manually, e.g. summarizer(\'...\', max_length={input_length//2})"""    )





@add_end_docstrings(UpperCAmelCase_)
class    lowerCamelCase__  (						UpperCAmelCase_):
     """simple docstring"""



     _A       =			'translation'







     def 						_a      (self				,			__a				,			__a				,			__a    ):






           '''simple docstring'''



           if input_length > 0.9 * max_length:
                 logger.warning(
                     F"""Your input_length: {input_length} is bigger than 0.9 * max_length: {max_length}. You might consider """
                     "increasing your max_length manually, e.g. translator('...', max_length=400)"    )
           return True







     def 						_a      (self				,			*__a				,			__a=TruncationStrategy.DO_NOT_TRUNCATE				,			__a=None				,			__a=None    ):






           '''simple docstring'''



           if getattr(self.tokenizer				,			"_build_translation_inputs"				,			lowercase_    ):
                 return self.tokenizer._build_translation_inputs(
                     *lowercase_				,			return_tensors=self.framework				,			truncation=lowercase_				,			src_lang=lowercase_				,			tgt_lang=lowercase_    )
           else:
                 return super()._parse_and_tokenize(*lowercase_				,			truncation=lowercase_    )







     def 						_a      (self				,			__a=None				,			__a=None				,			**__a    ):






           '''simple docstring'''



           lowerCamelCase			,							lowerCamelCase			,							lowerCamelCase								=		super()._sanitize_parameters(**lowercase_    )
           if src_lang is not None:
                 lowerCamelCase								=		src_lang
           if tgt_lang is not None:
                 lowerCamelCase								=		tgt_lang
           if src_lang is None and tgt_lang is None:
                 # Backward compatibility, direct arguments use is preferred.
                 lowerCamelCase								=		kwargs.get("task"				,			self.task    )
                 lowerCamelCase								=		task.split("_"    )
                 if task and len(lowercase_    ) == 4:
                       # translation, XX, to YY
                       lowerCamelCase								=		items[1]
                       lowerCamelCase								=		items[3]
           return preprocess_params, forward_params, postprocess_params







     def __call__(self				,			*__a				,			**__a    ):






           '''simple docstring'''



           return super().__call__(*lowercase_				,			**lowercase_    )











import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


a_						:     int	     =		logging.get_logger(__name__)

a_						:     List[str]	     =		{'vocab_file': 'spiece.model'}

a_						:     Optional[Any]	     =		{
    'vocab_file': {
        'TsinghuaAI/CPM-Generate': 'https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model',
    }
}





class    lowerCamelCase__  (						UpperCAmelCase_):
     """simple docstring"""



     def __init__(self				,			__a				,			__a=False				,			__a=True				,			__a=False				,			__a="<s>"				,			__a="</s>"				,			__a="<unk>"				,			__a="<sep>"				,			__a="<pad>"				,			__a="<cls>"				,			__a="<mask>"				,			__a=["<eop>", "<eod>"]				,			__a = None				,			**__a				,			):






           '''simple docstring'''



           lowerCamelCase								=		AddedToken(__a				,			lstrip=__a				,			rstrip=__a    ) if isinstance(__a				,			__a    ) else mask_token

           lowerCamelCase								=		{} if sp_model_kwargs is None else sp_model_kwargs

           super().__init__(
               do_lower_case=__a				,			remove_space=__a				,			keep_accents=__a				,			bos_token=__a				,			eos_token=__a				,			unk_token=__a				,			sep_token=__a				,			pad_token=__a				,			cls_token=__a				,			mask_token=__a				,			additional_special_tokens=__a				,			sp_model_kwargs=self.sp_model_kwargs				,			**__a				,			)

           lowerCamelCase								=		3

           lowerCamelCase								=		do_lower_case
           lowerCamelCase								=		remove_space
           lowerCamelCase								=		keep_accents
           lowerCamelCase								=		vocab_file

           lowerCamelCase								=		spm.SentencePieceProcessor(**self.sp_model_kwargs    )
           self.sp_model.Load(__a    )

           try:
                 import jieba
           except ModuleNotFoundError as error:
                 raise error.__class__(
                     "You need to install jieba to use CpmTokenizer or CpmTokenizerFast. "
                     "See https://pypi.org/project/jieba/ for installation."    )
           lowerCamelCase								=		jieba
           lowerCamelCase								=		str.maketrans(" \n"				,			"\u2582\u2583"    )







     @property
     # Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.vocab_size
     def 						_a      (self    ):






           '''simple docstring'''



           return len(self.sp_model    )







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		{self.convert_ids_to_tokens(__a    ): i for i in range(self.vocab_size    )}
           vocab.update(self.added_tokens_encoder    )
           return vocab







     def __getstate__(self    ):






           '''simple docstring'''



           lowerCamelCase								=		self.__dict__.copy()
           lowerCamelCase								=		None
           return state







     def __setstate__(self				,			__a    ):






           '''simple docstring'''



           lowerCamelCase								=		d

           # for backward compatibility
           if not hasattr(self				,			"sp_model_kwargs"    ):
                 lowerCamelCase								=		{}

           lowerCamelCase								=		spm.SentencePieceProcessor(**self.sp_model_kwargs    )
           self.sp_model.Load(self.vocab_file    )







     def 						_a      (self				,			__a    ):






           '''simple docstring'''



           if self.remove_space:
                 lowerCamelCase								=		" ".join(inputs.strip().split()    )
           else:
                 lowerCamelCase								=		inputs
           lowerCamelCase								=		outputs.replace("``"				,			"\""    ).replace("''"				,			"\""    )

           if not self.keep_accents:
                 lowerCamelCase								=		unicodedata.normalize("NFKD"				,			__a    )
                 lowerCamelCase								=		"".join([c for c in outputs if not unicodedata.combining(__a    )]    )
           if self.do_lower_case:
                 lowerCamelCase								=		outputs.lower()

           return outputs







     def 						_a      (self				,			__a    ):






           '''simple docstring'''



           lowerCamelCase								=		self.preprocess_text(__a    )
           lowerCamelCase								=		self.sp_model.encode(__a				,			out_type=__a    )
           lowerCamelCase								=		[]
           for piece in pieces:
                 if len(__a    ) > 1 and piece[-1] == str(","    ) and piece[-2].isdigit():
                       lowerCamelCase								=		self.sp_model.EncodeAsPieces(piece[:-1].replace(__a				,			""    )    )
                       if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                             if len(cur_pieces[0]    ) == 1:
                                   lowerCamelCase								=		cur_pieces[1:]
                             else:
                                   lowerCamelCase								=		cur_pieces[0][1:]
                       cur_pieces.append(piece[-1]    )
                       new_pieces.extend(__a    )
                 else:
                       new_pieces.append(__a    )

           return new_pieces







     def 						_a      (self				,			__a    ):






           '''simple docstring'''



           return self.sp_model.PieceToId(__a    )







     def 						_a      (self				,			__a    ):






           '''simple docstring'''



           return self.sp_model.IdToPiece(__a    )







     def 						_a      (self				,			__a    ):






           '''simple docstring'''



           lowerCamelCase								=		"".join(__a    ).replace(__a				,			" "    ).strip()
           return out_string







     def 						_a      (self				,			__a				,			__a = None    ):






           '''simple docstring'''



           lowerCamelCase								=		[self.sep_token_id]
           lowerCamelCase								=		[self.cls_token_id]
           if token_ids_a is None:
                 return token_ids_a + sep + cls
           return token_ids_a + sep + token_ids_a + sep + cls







     def 						_a      (self				,			__a				,			__a = None				,			__a = False    ):






           '''simple docstring'''



           if already_has_special_tokens:
                 return super().get_special_tokens_mask(
                     token_ids_a=__a				,			token_ids_a=__a				,			already_has_special_tokens=__a    )

           if token_ids_a is not None:
                 return ([0] * len(__a    )) + [1] + ([0] * len(__a    )) + [1, 1]
           return ([0] * len(__a    )) + [1, 1]







     def 						_a      (self				,			__a				,			__a = None    ):






           '''simple docstring'''



           lowerCamelCase								=		[self.sep_token_id]
           lowerCamelCase								=		[2]

           if token_ids_a is None:
                 return len(token_ids_a + sep    ) * [0] + cls_segment_id
           return len(token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1] + cls_segment_id







     def 						_a      (self				,			__a				,			__a = None    ):






           '''simple docstring'''



           if not os.path.isdir(__a    ):
                 logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""    )
                 return
           lowerCamelCase								=		os.path.join(
               __a				,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]    )

           if os.path.abspath(self.vocab_file    ) != os.path.abspath(__a    ) and os.path.isfile(self.vocab_file    ):
                 copyfile(self.vocab_file				,			__a    )
           elif not os.path.isfile(self.vocab_file    ):
                 with open(__a				,			"wb"    ) as fi:
                       lowerCamelCase								=		self.sp_model.serialized_model_proto()
                       fi.write(__a    )

           return (out_vocab_file,)







     def 						_a      (self				,			*__a				,			**__a    ):






           '''simple docstring'''



           lowerCamelCase								=		super()._decode(*__a				,			**__a    )
           lowerCamelCase								=		text.replace(" "				,			""    ).replace("\u2582"				,			" "    ).replace("\u2583"				,			"\n"    )
           return text