






"""simple docstring"""



import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
						from .tokenization_big_bird import BigBirdTokenizer
else:
						A_    =							None

A_    =							logging.get_logger(__name__)
A_    =							{'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

A_    =							{
    '''vocab_file''': {
        '''google/bigbird-roberta-base''': '''https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model''',
        '''google/bigbird-roberta-large''': (
            '''https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model'''
        ),
        '''google/bigbird-base-trivia-itc''': (
            '''https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model'''
        ),
    },
    '''tokenizer_file''': {
        '''google/bigbird-roberta-base''': (
            '''https://huggingface.co/google/bigbird-roberta-base/resolve/main/tokenizer.json'''
        ),
        '''google/bigbird-roberta-large''': (
            '''https://huggingface.co/google/bigbird-roberta-large/resolve/main/tokenizer.json'''
        ),
        '''google/bigbird-base-trivia-itc''': (
            '''https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/tokenizer.json'''
        ),
    },
}

A_    =							{
    '''google/bigbird-roberta-base''': 4096,
    '''google/bigbird-roberta-large''': 4096,
    '''google/bigbird-base-trivia-itc''': 4096,
}


A_    =							'''▁'''






class 							__SCREAMING_SNAKE_CASE						(				UpperCamelCase		):
							snake_case_     =							VOCAB_FILES_NAMES
							snake_case_     =							PRETRAINED_VOCAB_FILES_MAP
							snake_case_     =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							snake_case_     =							BigBirdTokenizer
							snake_case_     =							['input_ids', 'attention_mask']
							snake_case_     =							[]


							def __init__(			self      :						Optional[int]			,    snake_case      :						Optional[int]=None			,    snake_case      :						int=None			,    snake_case      :						Optional[Any]="<unk>"			,    snake_case      :						Dict="<s>"			,    snake_case      :						Dict="</s>"			,    snake_case      :						Union[str, Any]="<pad>"			,    snake_case      :						Any="[SEP]"			,    snake_case      :						Union[str, Any]="[MASK]"			,    snake_case      :						Tuple="[CLS]"			,    **snake_case      :						str			,    ):
													'''simple docstring'''





													A__						:  List[str]		  =  AddedToken(snake_case			,    lstrip=snake_case			,    rstrip=snake_case							) if isinstance(snake_case			,    snake_case							) else bos_token
													A__						:  List[Any]		  =  AddedToken(snake_case			,    lstrip=snake_case			,    rstrip=snake_case							) if isinstance(snake_case			,    snake_case							) else eos_token
													A__						:  Optional[Any]		  =  AddedToken(snake_case			,    lstrip=snake_case			,    rstrip=snake_case							) if isinstance(snake_case			,    snake_case							) else unk_token
													A__						:  str		  =  AddedToken(snake_case			,    lstrip=snake_case			,    rstrip=snake_case							) if isinstance(snake_case			,    snake_case							) else pad_token
													A__						:  Any		  =  AddedToken(snake_case			,    lstrip=snake_case			,    rstrip=snake_case							) if isinstance(snake_case			,    snake_case							) else cls_token
													A__						:  str		  =  AddedToken(snake_case			,    lstrip=snake_case			,    rstrip=snake_case							) if isinstance(snake_case			,    snake_case							) else sep_token

													# Mask token behave like a normal word, i.e. include the space before it
													A__						:  str		  =  AddedToken(snake_case			,    lstrip=snake_case			,    rstrip=snake_case							) if isinstance(snake_case			,    snake_case							) else mask_token

													super().__init__(
													    snake_case			,    tokenizer_file=snake_case			,    bos_token=snake_case			,    eos_token=snake_case			,    unk_token=snake_case			,    sep_token=snake_case			,    pad_token=snake_case			,    cls_token=snake_case			,    mask_token=snake_case			,    **snake_case			,    )

													A__						:  Tuple		  =  vocab_file
													A__						:  Any		  =  False if not self.vocab_file else True


							def 						_UpperCamelCase       (			self      :						Tuple			,    snake_case      :						List[int]			,    snake_case      :						Optional[List[int]] = None							):
													'''simple docstring'''





													A__						:  Dict		  =  [self.sep_token_id]
													A__						:  str		  =  [self.cls_token_id]
													if token_ids_a is None:
																			return cls + token_ids_a + sep
													return cls + token_ids_a + sep + token_ids_a + sep


							def 						_UpperCamelCase       (			self      :						Dict			,    snake_case      :						List[int]			,    snake_case      :						Optional[List[int]] = None			,    snake_case      :						bool = False							):
													'''simple docstring'''





													if already_has_special_tokens:
																			if token_ids_a is not None:
																									raise ValueError(
																									    """You should not supply a second sequence if the provided sequence of """
																									    """ids is already formatted with special tokens for the model."""							)
																			return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

													if token_ids_a is None:
																			return [1] + ([0] * len(snake_case							)) + [1]
													return [1] + ([0] * len(snake_case							)) + [1] + ([0] * len(snake_case							)) + [1]


							def 						_UpperCamelCase       (			self      :						Optional[Any]			,    snake_case      :						List[int]			,    snake_case      :						Optional[List[int]] = None							):
													'''simple docstring'''





													A__						:  Union[str, Any]		  =  [self.sep_token_id]
													A__						:  Any		  =  [self.cls_token_id]

													if token_ids_a is None:
																			return len(cls + token_ids_a + sep							) * [0]
													return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]


							def 						_UpperCamelCase       (			self      :						Dict			,    snake_case      :						str			,    snake_case      :						Optional[str] = None							):
													'''simple docstring'''





													if not self.can_save_slow_tokenizer:
																			raise ValueError(
																			    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
																			    """tokenizer."""							)

													if not os.path.isdir(snake_case							):
																			logger.error(F'Vocabulary path ({save_directory}) should be a directory'							)
																			return
													A__						:  List[Any]		  =  os.path.join(
													    snake_case			,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)

													if os.path.abspath(self.vocab_file							) != os.path.abspath(snake_case							):
																			copyfile(self.vocab_file			,    snake_case							)

													return (out_vocab_file,)














"""simple docstring"""



import unittest

from transformers import (
    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TextClassificationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow

from .test_pipelines_common import ANY


# These 2 model types require different inputs than those of the usual text models.
A_    =							{'''LayoutLMv2Config''', '''LayoutLMv3Config'''}






@is_pipeline_test
class 							__SCREAMING_SNAKE_CASE						(				unittest.TestCase		):
							snake_case_     =							MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
							snake_case_     =							TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING

							if model_mapping is not None:
														snake_case_     =							{config: model for config, model in model_mapping.items() if config.__name__ not in _TO_SKIP}
							if tf_model_mapping is not None:
														snake_case_     =							{
														    config: model for config, model in tf_model_mapping.items() if config.__name__ not in _TO_SKIP
														}


							@require_torch
							def 						_UpperCamelCase       (			self      :						Tuple							):
													'''simple docstring'''





													A__						:  Tuple		  =  pipeline(
													    task="""text-classification"""			,    model="""hf-internal-testing/tiny-random-distilbert"""			,    framework="""pt"""							)

													A__						:  Dict		  =  text_classifier("""This is great !"""							)
													self.assertEqual(nested_simplify(snake_case							)			,    [{"""label""": """LABEL_0""", """score""": 0.504}]							)

													A__						:  Optional[int]		  =  text_classifier("""This is great !"""			,    top_k=2							)
													self.assertEqual(
													    nested_simplify(snake_case							)			,    [{"""label""": """LABEL_0""", """score""": 0.504}, {"""label""": """LABEL_1""", """score""": 0.496}]							)

													A__						:  Any		  =  text_classifier(["""This is great !""", """This is bad"""]			,    top_k=2							)
													self.assertEqual(
													    nested_simplify(snake_case							)			,    [
													        [{"""label""": """LABEL_0""", """score""": 0.504}, {"""label""": """LABEL_1""", """score""": 0.496}],
													        [{"""label""": """LABEL_0""", """score""": 0.504}, {"""label""": """LABEL_1""", """score""": 0.496}],
													    ]			,    )

													A__						:  Optional[Any]		  =  text_classifier("""This is great !"""			,    top_k=1							)
													self.assertEqual(nested_simplify(snake_case							)			,    [{"""label""": """LABEL_0""", """score""": 0.504}]							)

													# Legacy behavior
													A__						:  str		  =  text_classifier("""This is great !"""			,    return_all_scores=snake_case							)
													self.assertEqual(nested_simplify(snake_case							)			,    [{"""label""": """LABEL_0""", """score""": 0.504}]							)

													A__						:  int		  =  text_classifier("""This is great !"""			,    return_all_scores=snake_case							)
													self.assertEqual(
													    nested_simplify(snake_case							)			,    [[{"""label""": """LABEL_0""", """score""": 0.504}, {"""label""": """LABEL_1""", """score""": 0.496}]]							)

													A__						:  str		  =  text_classifier(["""This is great !""", """Something else"""]			,    return_all_scores=snake_case							)
													self.assertEqual(
													    nested_simplify(snake_case							)			,    [
													        [{"""label""": """LABEL_0""", """score""": 0.504}, {"""label""": """LABEL_1""", """score""": 0.496}],
													        [{"""label""": """LABEL_0""", """score""": 0.504}, {"""label""": """LABEL_1""", """score""": 0.496}],
													    ]			,    )

													A__						:  Optional[int]		  =  text_classifier(["""This is great !""", """Something else"""]			,    return_all_scores=snake_case							)
													self.assertEqual(
													    nested_simplify(snake_case							)			,    [
													        {"""label""": """LABEL_0""", """score""": 0.504},
													        {"""label""": """LABEL_0""", """score""": 0.504},
													    ]			,    )


							@require_torch
							def 						_UpperCamelCase       (			self      :						int							):
													'''simple docstring'''





													import torch

													A__						:  Tuple		  =  pipeline(
													    task="""text-classification"""			,    model="""hf-internal-testing/tiny-random-distilbert"""			,    framework="""pt"""			,    device=torch.device("""cpu"""							)			,    )

													A__						:  List[str]		  =  text_classifier("""This is great !"""							)
													self.assertEqual(nested_simplify(snake_case							)			,    [{"""label""": """LABEL_0""", """score""": 0.504}]							)


							@require_tf
							def 						_UpperCamelCase       (			self      :						List[str]							):
													'''simple docstring'''





													A__						:  str		  =  pipeline(
													    task="""text-classification"""			,    model="""hf-internal-testing/tiny-random-distilbert"""			,    framework="""tf"""							)

													A__						:  Optional[Any]		  =  text_classifier("""This is great !"""							)
													self.assertEqual(nested_simplify(snake_case							)			,    [{"""label""": """LABEL_0""", """score""": 0.504}]							)


							@slow
							@require_torch
							def 						_UpperCamelCase       (			self      :						List[Any]							):
													'''simple docstring'''





													A__						:  Optional[int]		  =  pipeline("""text-classification"""							)

													A__						:  Optional[int]		  =  text_classifier("""This is great !"""							)
													self.assertEqual(nested_simplify(snake_case							)			,    [{"""label""": """POSITIVE""", """score""": 1.0}]							)
													A__						:  Optional[Any]		  =  text_classifier("""This is bad !"""							)
													self.assertEqual(nested_simplify(snake_case							)			,    [{"""label""": """NEGATIVE""", """score""": 1.0}]							)
													A__						:  Dict		  =  text_classifier("""Birds are a type of animal"""							)
													self.assertEqual(nested_simplify(snake_case							)			,    [{"""label""": """POSITIVE""", """score""": 0.988}]							)


							@slow
							@require_tf
							def 						_UpperCamelCase       (			self      :						Any							):
													'''simple docstring'''





													A__						:  Union[str, Any]		  =  pipeline("""text-classification"""			,    framework="""tf"""							)

													A__						:  Dict		  =  text_classifier("""This is great !"""							)
													self.assertEqual(nested_simplify(snake_case							)			,    [{"""label""": """POSITIVE""", """score""": 1.0}]							)
													A__						:  Optional[int]		  =  text_classifier("""This is bad !"""							)
													self.assertEqual(nested_simplify(snake_case							)			,    [{"""label""": """NEGATIVE""", """score""": 1.0}]							)
													A__						:  Tuple		  =  text_classifier("""Birds are a type of animal"""							)
													self.assertEqual(nested_simplify(snake_case							)			,    [{"""label""": """POSITIVE""", """score""": 0.988}]							)


							def 						_UpperCamelCase       (			self      :						Optional[Any]			,    snake_case      :						Any			,    snake_case      :						List[Any]			,    snake_case      :						Any							):
													'''simple docstring'''





													A__						:  Union[str, Any]		  =  TextClassificationPipeline(model=snake_case			,    tokenizer=snake_case							)
													return text_classifier, ["HuggingFace is in", "This is another test"]


							def 						_UpperCamelCase       (			self      :						int			,    snake_case      :						Union[str, Any]			,    snake_case      :						Dict							):
													'''simple docstring'''





													A__						:  List[str]		  =  text_classifier.model
													# Small inputs because BartTokenizer tiny has maximum position embeddings = 22
													A__						:  int		  =  """HuggingFace is in"""
													A__						:  List[Any]		  =  text_classifier(snake_case							)

													self.assertEqual(nested_simplify(snake_case							)			,    [{"""label""": ANY(snake_case							), """score""": ANY(snake_case							)}]							)
													self.assertTrue(outputs[0]["""label"""] in model.config.idalabel.values()							)

													A__						:  Any		  =  ["""HuggingFace is in """, """Paris is in France"""]
													A__						:  Any		  =  text_classifier(snake_case							)
													self.assertEqual(
													    nested_simplify(snake_case							)			,    [{"""label""": ANY(snake_case							), """score""": ANY(snake_case							)}, {"""label""": ANY(snake_case							), """score""": ANY(snake_case							)}]			,    )
													self.assertTrue(outputs[0]["""label"""] in model.config.idalabel.values()							)
													self.assertTrue(outputs[1]["""label"""] in model.config.idalabel.values()							)

													# Forcing to get all results with `top_k=None`
													# This is NOT the legacy format
													A__						:  List[Any]		  =  text_classifier(snake_case			,    top_k=snake_case							)
													A__						:  Tuple		  =  len(model.config.idalabel.values()							)
													self.assertEqual(
													    nested_simplify(snake_case							)			,    [[{"""label""": ANY(snake_case							), """score""": ANY(snake_case							)}] * N, [{"""label""": ANY(snake_case							), """score""": ANY(snake_case							)}] * N]			,    )

													A__						:  Optional[Any]		  =  {"""text""": """HuggingFace is in """, """text_pair""": """Paris is in France"""}
													A__						:  Union[str, Any]		  =  text_classifier(snake_case							)
													self.assertEqual(
													    nested_simplify(snake_case							)			,    {"""label""": ANY(snake_case							), """score""": ANY(snake_case							)}			,    )
													self.assertTrue(outputs["""label"""] in model.config.idalabel.values()							)

													# This might be used a text pair, but tokenizer + pipe interaction
													# makes it hard to understand that it's not using the pair properly
													# https://github.com/huggingface/transformers/issues/17305
													# We disabled this usage instead as it was outputting wrong outputs.
													A__						:  Tuple		  =  [["""HuggingFace is in """, """Paris is in France"""]]
													with self.assertRaises(snake_case							):
																			text_classifier(snake_case							)

													# This used to be valid for doing text pairs
													# We're keeping it working because of backward compatibility
													A__						:  List[Any]		  =  text_classifier([[["""HuggingFace is in """, """Paris is in France"""]]]							)
													self.assertEqual(
													    nested_simplify(snake_case							)			,    [{"""label""": ANY(snake_case							), """score""": ANY(snake_case							)}]			,    )
													self.assertTrue(outputs[0]["""label"""] in model.config.idalabel.values()							)






