



import unittest

from transformers import (
    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    Pipeline,
    ZeroShotClassificationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow

from .test_pipelines_common import ANY


# These 2 model types require different inputs than those of the usual text models.
a_						:     Optional[int]	     =		{'LayoutLMv2Config', 'LayoutLMv3Config'}





@is_pipeline_test
class    lowerCamelCase__  (						unittest.TestCase):
     """simple docstring"""



     _A       =			MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
     _A       =			TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING

     if model_mapping is not None:
          _A       =			{config: model for config, model in model_mapping.items() if config.__name__ not in _TO_SKIP}
     if tf_model_mapping is not None:
          _A       =			{
              config: model for config, model in tf_model_mapping.items() if config.__name__ not in _TO_SKIP
          }







     def 						_a      (self				,			__a				,			__a				,			__a    ):






           '''simple docstring'''



           lowerCamelCase								=		ZeroShotClassificationPipeline(
               model=__a				,			tokenizer=__a				,			candidate_labels=["polics", "health"]    )
           return classifier, ["Who are you voting for in 2020?", "My stomach hurts."]







     def 						_a      (self				,			__a				,			__a    ):






           '''simple docstring'''



           lowerCamelCase								=		classifier("Who are you voting for in 2020?"				,			candidate_labels="politics"    )
           self.assertEqual(__a				,			{"sequence": ANY(__a    ), "labels": [ANY(__a    )], "scores": [ANY(__a    )]}    )

           # No kwarg
           lowerCamelCase								=		classifier("Who are you voting for in 2020?"				,			["politics"]    )
           self.assertEqual(__a				,			{"sequence": ANY(__a    ), "labels": [ANY(__a    )], "scores": [ANY(__a    )]}    )

           lowerCamelCase								=		classifier("Who are you voting for in 2020?"				,			candidate_labels=["politics"]    )
           self.assertEqual(__a				,			{"sequence": ANY(__a    ), "labels": [ANY(__a    )], "scores": [ANY(__a    )]}    )

           lowerCamelCase								=		classifier("Who are you voting for in 2020?"				,			candidate_labels="politics, public health"    )
           self.assertEqual(
               __a				,			{"sequence": ANY(__a    ), "labels": [ANY(__a    ), ANY(__a    )], "scores": [ANY(__a    ), ANY(__a    )]}    )
           self.assertAlmostEqual(sum(nested_simplify(outputs["scores"]    )    )				,			1.0    )

           lowerCamelCase								=		classifier("Who are you voting for in 2020?"				,			candidate_labels=["politics", "public health"]    )
           self.assertEqual(
               __a				,			{"sequence": ANY(__a    ), "labels": [ANY(__a    ), ANY(__a    )], "scores": [ANY(__a    ), ANY(__a    )]}    )
           self.assertAlmostEqual(sum(nested_simplify(outputs["scores"]    )    )				,			1.0    )

           lowerCamelCase								=		classifier(
               "Who are you voting for in 2020?"				,			candidate_labels="politics"				,			hypothesis_template="This text is about {}"    )
           self.assertEqual(__a				,			{"sequence": ANY(__a    ), "labels": [ANY(__a    )], "scores": [ANY(__a    )]}    )

           # https://github.com/huggingface/transformers/issues/13846
           lowerCamelCase								=		classifier(["I am happy"]				,			["positive", "negative"]    )
           self.assertEqual(
               __a				,			[
                   {"sequence": ANY(__a    ), "labels": [ANY(__a    ), ANY(__a    )], "scores": [ANY(__a    ), ANY(__a    )]}
                   for i in range(1    )
               ]				,			)
           lowerCamelCase								=		classifier(["I am happy", "I am sad"]				,			["positive", "negative"]    )
           self.assertEqual(
               __a				,			[
                   {"sequence": ANY(__a    ), "labels": [ANY(__a    ), ANY(__a    )], "scores": [ANY(__a    ), ANY(__a    )]}
                   for i in range(2    )
               ]				,			)

           with self.assertRaises(__a    ):
                 classifier(""				,			candidate_labels="politics"    )

           with self.assertRaises(__a    ):
                 classifier(__a				,			candidate_labels="politics"    )

           with self.assertRaises(__a    ):
                 classifier("Who are you voting for in 2020?"				,			candidate_labels=""    )

           with self.assertRaises(__a    ):
                 classifier("Who are you voting for in 2020?"				,			candidate_labels=__a    )

           with self.assertRaises(__a    ):
                 classifier(
                     "Who are you voting for in 2020?"				,			candidate_labels="politics"				,			hypothesis_template="Not formatting template"				,			)

           with self.assertRaises(__a    ):
                 classifier(
                     "Who are you voting for in 2020?"				,			candidate_labels="politics"				,			hypothesis_template=__a				,			)

           self.run_entailment_id(__a    )







     def 						_a      (self				,			__a    ):






           '''simple docstring'''



           lowerCamelCase								=		zero_shot_classifier.model.config
           lowerCamelCase								=		config.labelaid
           lowerCamelCase								=		zero_shot_classifier.entailment_id

           lowerCamelCase								=		{"LABEL_0": 0, "LABEL_1": 1, "LABEL_2": 2}
           self.assertEqual(zero_shot_classifier.entailment_id				,			-1    )

           lowerCamelCase								=		{"entailment": 0, "neutral": 1, "contradiction": 2}
           self.assertEqual(zero_shot_classifier.entailment_id				,			0    )

           lowerCamelCase								=		{"ENTAIL": 0, "NON-ENTAIL": 1}
           self.assertEqual(zero_shot_classifier.entailment_id				,			0    )

           lowerCamelCase								=		{"ENTAIL": 2, "NEUTRAL": 1, "CONTR": 0}
           self.assertEqual(zero_shot_classifier.entailment_id				,			2    )

           lowerCamelCase								=		original_labelaid
           self.assertEqual(__a				,			zero_shot_classifier.entailment_id    )







     @require_torch
     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		pipeline(
               "zero-shot-classification"				,			model="sshleifer/tiny-distilbert-base-cased-distilled-squad"				,			framework="pt"				,			)
           # There was a regression in 4.10 for this
           # Adding a test so we don't make the mistake again.
           # https://github.com/huggingface/transformers/issues/13381#issuecomment-912343499
           zero_shot_classifier(
               "Who are you voting for in 2020?" * 1_00				,			candidate_labels=["politics", "public health", "science"]    )







     @require_torch
     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		pipeline(
               "zero-shot-classification"				,			model="sshleifer/tiny-distilbert-base-cased-distilled-squad"				,			framework="pt"				,			)
           lowerCamelCase								=		zero_shot_classifier(
               "Who are you voting for in 2020?"				,			candidate_labels=["politics", "public health", "science"]    )

           self.assertEqual(
               nested_simplify(__a    )				,			{
                   "sequence": "Who are you voting for in 2020?",
                   "labels": ["science", "public health", "politics"],
                   "scores": [0.333, 0.333, 0.333],
               }				,			)







     @require_tf
     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		pipeline(
               "zero-shot-classification"				,			model="sshleifer/tiny-distilbert-base-cased-distilled-squad"				,			framework="tf"				,			)
           lowerCamelCase								=		zero_shot_classifier(
               "Who are you voting for in 2020?"				,			candidate_labels=["politics", "public health", "science"]    )

           self.assertEqual(
               nested_simplify(__a    )				,			{
                   "sequence": "Who are you voting for in 2020?",
                   "labels": ["science", "public health", "politics"],
                   "scores": [0.333, 0.333, 0.333],
               }				,			)







     @slow
     @require_torch
     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		pipeline("zero-shot-classification"				,			model="roberta-large-mnli"				,			framework="pt"    )
           lowerCamelCase								=		zero_shot_classifier(
               "Who are you voting for in 2020?"				,			candidate_labels=["politics", "public health", "science"]    )

           self.assertEqual(
               nested_simplify(__a    )				,			{
                   "sequence": "Who are you voting for in 2020?",
                   "labels": ["politics", "public health", "science"],
                   "scores": [0.976, 0.015, 0.009],
               }				,			)
           lowerCamelCase								=		zero_shot_classifier(
               "The dominant sequence transduction models are based on complex recurrent or convolutional neural networks"
               " in an encoder-decoder configuration. The best performing models also connect the encoder and decoder"
               " through an attention mechanism. We propose a new simple network architecture, the Transformer, based"
               " solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two"
               " machine translation tasks show these models to be superior in quality while being more parallelizable"
               " and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014"
               " English-to-German translation task, improving over the existing best results, including ensembles by"
               " over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new"
               " single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small"
               " fraction of the training costs of the best models from the literature. We show that the Transformer"
               " generalizes well to other tasks by applying it successfully to English constituency parsing both with"
               " large and limited training data."				,			candidate_labels=["machine learning", "statistics", "translation", "vision"]				,			multi_label=__a				,			)
           self.assertEqual(
               nested_simplify(__a    )				,			{
                   "sequence": (
                       "The dominant sequence transduction models are based on complex recurrent or convolutional neural"
                       " networks in an encoder-decoder configuration. The best performing models also connect the"
                       " encoder and decoder through an attention mechanism. We propose a new simple network"
                       " architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence"
                       " and convolutions entirely. Experiments on two machine translation tasks show these models to be"
                       " superior in quality while being more parallelizable and requiring significantly less time to"
                       " train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task,"
                       " improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014"
                       " English-to-French translation task, our model establishes a new single-model state-of-the-art"
                       " BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training"
                       " costs of the best models from the literature. We show that the Transformer generalizes well to"
                       " other tasks by applying it successfully to English constituency parsing both with large and"
                       " limited training data."
                   ),
                   "labels": ["translation", "machine learning", "vision", "statistics"],
                   "scores": [0.817, 0.713, 0.018, 0.018],
               }				,			)







     @slow
     @require_tf
     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		pipeline("zero-shot-classification"				,			model="roberta-large-mnli"				,			framework="tf"    )
           lowerCamelCase								=		zero_shot_classifier(
               "Who are you voting for in 2020?"				,			candidate_labels=["politics", "public health", "science"]    )

           self.assertEqual(
               nested_simplify(__a    )				,			{
                   "sequence": "Who are you voting for in 2020?",
                   "labels": ["politics", "public health", "science"],
                   "scores": [0.976, 0.015, 0.009],
               }				,			)
           lowerCamelCase								=		zero_shot_classifier(
               "The dominant sequence transduction models are based on complex recurrent or convolutional neural networks"
               " in an encoder-decoder configuration. The best performing models also connect the encoder and decoder"
               " through an attention mechanism. We propose a new simple network architecture, the Transformer, based"
               " solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two"
               " machine translation tasks show these models to be superior in quality while being more parallelizable"
               " and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014"
               " English-to-German translation task, improving over the existing best results, including ensembles by"
               " over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new"
               " single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small"
               " fraction of the training costs of the best models from the literature. We show that the Transformer"
               " generalizes well to other tasks by applying it successfully to English constituency parsing both with"
               " large and limited training data."				,			candidate_labels=["machine learning", "statistics", "translation", "vision"]				,			multi_label=__a				,			)
           self.assertEqual(
               nested_simplify(__a    )				,			{
                   "sequence": (
                       "The dominant sequence transduction models are based on complex recurrent or convolutional neural"
                       " networks in an encoder-decoder configuration. The best performing models also connect the"
                       " encoder and decoder through an attention mechanism. We propose a new simple network"
                       " architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence"
                       " and convolutions entirely. Experiments on two machine translation tasks show these models to be"
                       " superior in quality while being more parallelizable and requiring significantly less time to"
                       " train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task,"
                       " improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014"
                       " English-to-French translation task, our model establishes a new single-model state-of-the-art"
                       " BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training"
                       " costs of the best models from the literature. We show that the Transformer generalizes well to"
                       " other tasks by applying it successfully to English constituency parsing both with large and"
                       " limited training data."
                   ),
                   "labels": ["translation", "machine learning", "vision", "statistics"],
                   "scores": [0.817, 0.713, 0.018, 0.018],
               }				,			)




import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


a_						:     Optional[int]	     =		logging.get_logger(__name__)

a_						:     Any	     =		{'vocab_file': 'spiece.model'}

a_						:     List[Any]	     =		{
    'vocab_file': {
        'bert_for_seq_generation': (
            'https://huggingface.co/google/bert_for_seq_generation_L-24_bbc_encoder/resolve/main/spiece.model'
        ),
    }
}

a_						:     List[Any]	     =		{'bert_for_seq_generation': 5_1_2}





class    lowerCamelCase__  (						UpperCAmelCase_):
     """simple docstring"""



     _A       =			VOCAB_FILES_NAMES
     _A       =			PRETRAINED_VOCAB_FILES_MAP
     _A       =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _A       =			[]
     _A       =			['input_ids', 'attention_mask']







     def __init__(self				,			__a				,			__a="<s>"				,			__a="</s>"				,			__a="<unk>"				,			__a="<pad>"				,			__a="<::::>"				,			__a = None				,			**__a				,			):






           '''simple docstring'''



           lowerCamelCase								=		{} if sp_model_kwargs is None else sp_model_kwargs

           # Add extra_ids to the special token list
           super().__init__(
               bos_token=__a				,			eos_token=__a				,			unk_token=__a				,			pad_token=__a				,			sep_token=__a				,			sp_model_kwargs=self.sp_model_kwargs				,			**__a				,			)

           lowerCamelCase								=		vocab_file

           lowerCamelCase								=		spm.SentencePieceProcessor(**self.sp_model_kwargs    )
           self.sp_model.Load(__a    )







     @property
     def 						_a      (self    ):






           '''simple docstring'''



           return self.sp_model.get_piece_size()







     def 						_a      (self    ):






           '''simple docstring'''



           lowerCamelCase								=		{self.convert_ids_to_tokens(__a    ): i for i in range(self.vocab_size    )}
           vocab.update(self.added_tokens_encoder    )
           return vocab







     def __getstate__(self    ):






           '''simple docstring'''



           lowerCamelCase								=		self.__dict__.copy()
           lowerCamelCase								=		None
           return state







     def __setstate__(self				,			__a    ):






           '''simple docstring'''



           lowerCamelCase								=		d

           # for backward compatibility
           if not hasattr(self				,			"sp_model_kwargs"    ):
                 lowerCamelCase								=		{}

           lowerCamelCase								=		spm.SentencePieceProcessor(**self.sp_model_kwargs    )
           self.sp_model.Load(self.vocab_file    )







     def 						_a      (self				,			__a    ):






           '''simple docstring'''



           return self.sp_model.encode(__a				,			out_type=__a    )







     def 						_a      (self				,			__a    ):






           '''simple docstring'''



           return self.sp_model.piece_to_id(__a    )







     def 						_a      (self				,			__a    ):






           '''simple docstring'''



           lowerCamelCase								=		self.sp_model.IdToPiece(__a    )
           return token







     def 						_a      (self				,			__a    ):






           '''simple docstring'''



           lowerCamelCase								=		[]
           lowerCamelCase								=		""
           for token in tokens:
                 # make sure that special tokens are not decoded using sentencepiece model
                 if token in self.all_special_tokens:
                       out_string += self.sp_model.decode(__a    ) + token
                       lowerCamelCase								=		[]
                 else:
                       current_sub_tokens.append(__a    )
           out_string += self.sp_model.decode(__a    )
           return out_string.strip()







     def 						_a      (self				,			__a				,			__a = None    ):






           '''simple docstring'''



           if not os.path.isdir(__a    ):
                 logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""    )
                 return
           lowerCamelCase								=		os.path.join(
               __a				,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]    )

           if os.path.abspath(self.vocab_file    ) != os.path.abspath(__a    ) and os.path.isfile(self.vocab_file    ):
                 copyfile(self.vocab_file				,			__a    )
           elif not os.path.isfile(self.vocab_file    ):
                 with open(__a				,			"wb"    ) as fi:
                       lowerCamelCase								=		self.sp_model.serialized_model_proto()
                       fi.write(__a    )

           return (out_vocab_file,)