






"""simple docstring"""
import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class      SCREAMING_SNAKE_CASE  (				_SCREAMING_SNAKE_CASE       ,       unittest.TestCase						):





	"""simple docstring"""




	_A       :				Tuple    =				GPTaTokenizer
	_A       :				Any    =				GPTaTokenizerFast
	_A       :				Tuple    =				True
	_A       :				Optional[int]    =				{"""add_prefix_space""": True}
	_A       :				str    =				False





	def    lowerCamelCase(self							):
		super().setUp()

		# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
		A_		:						Dict	    =   [
		    """l""",
		    """o""",
		    """w""",
		    """e""",
		    """r""",
		    """s""",
		    """t""",
		    """i""",
		    """d""",
		    """n""",
		    """\u0120""",
		    """\u0120l""",
		    """\u0120n""",
		    """\u0120lo""",
		    """\u0120low""",
		    """er""",
		    """\u0120lowest""",
		    """\u0120newer""",
		    """\u0120wider""",
		    """<unk>""",
		    """<|endoftext|>""",
		]
		A_		:						Dict	    =   dict(zip(lowerCAmelCase_     ,     range(len(lowerCAmelCase_							)							)							)							)
		A_		:						Union[str, Any]	    =   ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
		A_		:						Optional[int]	    =   {"""unk_token""": """<unk>"""}

		A_		:						List[str]	    =   os.path.join(self.tmpdirname     ,     VOCAB_FILES_NAMES["""vocab_file"""]							)
		A_		:						List[Any]	    =   os.path.join(self.tmpdirname     ,     VOCAB_FILES_NAMES["""merges_file"""]							)
		with open(self.vocab_file     ,     """w"""     ,     encoding="""utf-8"""							) as fp:
			fp.write(json.dumps(lowerCAmelCase_							) + """\n"""							)
		with open(self.merges_file     ,     """w"""     ,     encoding="""utf-8"""							) as fp:
			fp.write("""\n""".join(lowerCAmelCase_							)							)





	def    lowerCamelCase(self     ,     **lowerCAmelCase_							):
		kwargs.update(self.special_tokens_map							)
		return GPTaTokenizer.from_pretrained(self.tmpdirname     ,     **lowerCAmelCase_							)





	def    lowerCamelCase(self     ,     **lowerCAmelCase_							):
		kwargs.update(self.special_tokens_map							)
		return GPTaTokenizerFast.from_pretrained(self.tmpdirname     ,     **lowerCAmelCase_							)





	def    lowerCamelCase(self     ,     lowerCAmelCase_							):
		A_		:						Union[str, Any]	    =   """lower newer"""
		A_		:						List[Any]	    =   """lower newer"""
		return input_text, output_text





	def    lowerCamelCase(self							):
		A_		:						int	    =   GPTaTokenizer(self.vocab_file     ,     self.merges_file     ,     **self.special_tokens_map							)
		A_		:						Tuple	    =   """lower newer"""
		A_		:						Any	    =   ["""\u0120low""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
		A_		:						Any	    =   tokenizer.tokenize(lowerCAmelCase_     ,     add_prefix_space=lowerCAmelCase_							)
		self.assertListEqual(lowerCAmelCase_     ,     lowerCAmelCase_							)

		A_		:						Dict	    =   tokens + [tokenizer.unk_token]
		A_		:						int	    =   [14, 15, 10, 9, 3, 2, 15, 19]
		self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_							)     ,     lowerCAmelCase_							)





	def    lowerCamelCase(self							):
		if not self.test_rust_tokenizer:
			return

		A_		:						List[str]	    =   self.get_tokenizer()
		A_		:						Optional[int]	    =   self.get_rust_tokenizer(add_prefix_space=lowerCAmelCase_							)

		A_		:						List[Any]	    =   """lower newer"""

		# Testing tokenization
		A_		:						str	    =   tokenizer.tokenize(lowerCAmelCase_     ,     add_prefix_space=lowerCAmelCase_							)
		A_		:						Optional[Any]	    =   rust_tokenizer.tokenize(lowerCAmelCase_							)
		self.assertListEqual(lowerCAmelCase_     ,     lowerCAmelCase_							)

		# Testing conversion to ids without special tokens
		A_		:						Optional[Any]	    =   tokenizer.encode(lowerCAmelCase_     ,     add_special_tokens=lowerCAmelCase_     ,     add_prefix_space=lowerCAmelCase_							)
		A_		:						Union[str, Any]	    =   rust_tokenizer.encode(lowerCAmelCase_     ,     add_special_tokens=lowerCAmelCase_							)
		self.assertListEqual(lowerCAmelCase_     ,     lowerCAmelCase_							)

		# Testing conversion to ids with special tokens
		A_		:						Dict	    =   self.get_rust_tokenizer(add_prefix_space=lowerCAmelCase_							)
		A_		:						List[str]	    =   tokenizer.encode(lowerCAmelCase_     ,     add_prefix_space=lowerCAmelCase_							)
		A_		:						Dict	    =   rust_tokenizer.encode(lowerCAmelCase_							)
		self.assertListEqual(lowerCAmelCase_     ,     lowerCAmelCase_							)

		# Testing the unknown token
		A_		:						Any	    =   tokens + [rust_tokenizer.unk_token]
		A_		:						List[str]	    =   [14, 15, 10, 9, 3, 2, 15, 19]
		self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(lowerCAmelCase_							)     ,     lowerCAmelCase_							)





	def    lowerCamelCase(self     ,     *lowerCAmelCase_     ,     **lowerCAmelCase_							):
		# It's very difficult to mix/test pretokenization with byte-level
		# And get both GPT2 and Roberta to work at the same time (mostly an issue of adding a space before the string)
		pass





	def    lowerCamelCase(self     ,     lowerCAmelCase_=15							):
		for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
			with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
				A_		:						Tuple	    =   self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_     ,     **lowerCAmelCase_							)

				# Simple input
				A_		:						Tuple	    =   """This is a simple input"""
				A_		:						int	    =   ["""This is a simple input 1""", """This is a simple input 2"""]
				A_		:						Optional[int]	    =   ("""This is a simple input""", """This is a pair""")
				A_		:						Optional[int]	    =   [
				    ("""This is a simple input 1""", """This is a simple input 2"""),
				    ("""This is a simple pair 1""", """This is a simple pair 2"""),
				]

				# Simple input tests
				self.assertRaises(lowerCAmelCase_     ,     tokenizer_r.encode     ,     lowerCAmelCase_     ,     max_length=lowerCAmelCase_     ,     padding="""max_length"""							)

				# Simple input
				self.assertRaises(lowerCAmelCase_     ,     tokenizer_r.encode_plus     ,     lowerCAmelCase_     ,     max_length=lowerCAmelCase_     ,     padding="""max_length"""							)

				# Simple input
				self.assertRaises(
				    lowerCAmelCase_     ,     tokenizer_r.batch_encode_plus     ,     lowerCAmelCase_     ,     max_length=lowerCAmelCase_     ,     padding="""max_length"""     ,     )

				# Pair input
				self.assertRaises(lowerCAmelCase_     ,     tokenizer_r.encode     ,     lowerCAmelCase_     ,     max_length=lowerCAmelCase_     ,     padding="""max_length"""							)

				# Pair input
				self.assertRaises(lowerCAmelCase_     ,     tokenizer_r.encode_plus     ,     lowerCAmelCase_     ,     max_length=lowerCAmelCase_     ,     padding="""max_length"""							)

				# Pair input
				self.assertRaises(
				    lowerCAmelCase_     ,     tokenizer_r.batch_encode_plus     ,     lowerCAmelCase_     ,     max_length=lowerCAmelCase_     ,     padding="""max_length"""     ,     )





	def    lowerCamelCase(self							):
		A_		:						List[Any]	    =   GPTaTokenizer.from_pretrained(self.tmpdirname     ,     pad_token="""<pad>"""							)

		# Simple input
		A_		:						str	    =   """This is a simple input"""
		A_		:						Optional[int]	    =   ["""This is a simple input looooooooong""", """This is a simple input"""]
		A_		:						Union[str, Any]	    =   ("""This is a simple input""", """This is a pair""")
		A_		:						List[Any]	    =   [
		    ("""This is a simple input loooooong""", """This is a simple input"""),
		    ("""This is a simple pair loooooong""", """This is a simple pair"""),
		]

		A_		:						Optional[Any]	    =   tokenizer.pad_token_id

		A_		:						Optional[Any]	    =   tokenizer(lowerCAmelCase_     ,     padding="""max_length"""     ,     max_length=30     ,     return_tensors="""np"""							)
		A_		:						Dict	    =   tokenizer(lowerCAmelCase_     ,     padding=lowerCAmelCase_     ,     truncate=lowerCAmelCase_     ,     return_tensors="""np"""							)
		A_		:						Tuple	    =   tokenizer(*lowerCAmelCase_     ,     padding="""max_length"""     ,     max_length=60     ,     return_tensors="""np"""							)
		A_		:						List[Any]	    =   tokenizer(lowerCAmelCase_     ,     padding=lowerCAmelCase_     ,     truncate=lowerCAmelCase_     ,     return_tensors="""np"""							)

		# s
		# test single string max_length padding
		self.assertEqual(out_s["""input_ids"""].shape[-1]     ,     30							)
		self.assertTrue(pad_token_id in out_s["""input_ids"""]							)
		self.assertTrue(0 in out_s["""attention_mask"""]							)

		# s2
		# test automatic padding
		self.assertEqual(out_sa["""input_ids"""].shape[-1]     ,     33							)
		# long slice doesn't have padding
		self.assertFalse(pad_token_id in out_sa["""input_ids"""][0]							)
		self.assertFalse(0 in out_sa["""attention_mask"""][0]							)
		# short slice does have padding
		self.assertTrue(pad_token_id in out_sa["""input_ids"""][1]							)
		self.assertTrue(0 in out_sa["""attention_mask"""][1]							)

		# p
		# test single pair max_length padding
		self.assertEqual(out_p["""input_ids"""].shape[-1]     ,     60							)
		self.assertTrue(pad_token_id in out_p["""input_ids"""]							)
		self.assertTrue(0 in out_p["""attention_mask"""]							)

		# p2
		# test automatic padding pair
		self.assertEqual(out_pa["""input_ids"""].shape[-1]     ,     52							)
		# long slice pair doesn't have padding
		self.assertFalse(pad_token_id in out_pa["""input_ids"""][0]							)
		self.assertFalse(0 in out_pa["""attention_mask"""][0]							)
		# short slice pair does have padding
		self.assertTrue(pad_token_id in out_pa["""input_ids"""][1]							)
		self.assertTrue(0 in out_pa["""attention_mask"""][1]							)





	def    lowerCamelCase(self							):
		A_		:						int	    =   """$$$"""
		A_		:						Dict	    =   GPTaTokenizer.from_pretrained(self.tmpdirname     ,     bos_token=lowerCAmelCase_     ,     add_bos_token=lowerCAmelCase_							)

		A_		:						str	    =   """This is a simple input"""
		A_		:						str	    =   ["""This is a simple input 1""", """This is a simple input 2"""]

		A_		:						List[Any]	    =   tokenizer.bos_token_id

		A_		:						Union[str, Any]	    =   tokenizer(lowerCAmelCase_							)
		A_		:						Optional[int]	    =   tokenizer(lowerCAmelCase_							)

		self.assertEqual(out_s.input_ids[0]     ,     lowerCAmelCase_							)
		self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids							)							)

		A_		:						List[str]	    =   tokenizer.decode(out_s.input_ids							)
		A_		:						int	    =   tokenizer.batch_decode(out_sa.input_ids							)

		self.assertEqual(decode_s.split()[0]     ,     lowerCAmelCase_							)
		self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa							)							)





	def    lowerCamelCase(self							):
		pass





	def    lowerCamelCase(self							):
		# TODO: change to self.get_tokenizers() when the fast version is implemented
		A_		:						str	    =   [self.get_tokenizer(do_lower_case=lowerCAmelCase_     ,     add_bos_token=lowerCAmelCase_							)]
		for tokenizer in tokenizers:
			with self.subTest(f"""{tokenizer.__class__.__name__}"""							):
				A_		:						Optional[Any]	    =   """Encode this."""
				A_		:						List[str]	    =   """This one too please."""
				A_		:						Tuple	    =   tokenizer.encode(lowerCAmelCase_     ,     add_special_tokens=lowerCAmelCase_							)
				encoded_sequence += tokenizer.encode(lowerCAmelCase_     ,     add_special_tokens=lowerCAmelCase_							)
				A_		:						int	    =   tokenizer.encode_plus(
				    lowerCAmelCase_     ,     lowerCAmelCase_     ,     add_special_tokens=lowerCAmelCase_     ,     return_special_tokens_mask=lowerCAmelCase_     ,     )
				A_		:						List[Any]	    =   encoded_sequence_dict["""input_ids"""]
				A_		:						int	    =   encoded_sequence_dict["""special_tokens_mask"""]
				self.assertEqual(len(lowerCAmelCase_							)     ,     len(lowerCAmelCase_							)							)

				A_		:						Optional[Any]	    =   [
				    (x if not special_tokens_mask[i] else None) for i, x in enumerate(lowerCAmelCase_							)
				]
				A_		:						Any	    =   [x for x in filtered_sequence if x is not None]
				self.assertEqual(lowerCAmelCase_     ,     lowerCAmelCase_							)






@require_tokenizers
class      SCREAMING_SNAKE_CASE  (				unittest.TestCase						):





	"""simple docstring"""




	def    lowerCamelCase(self							):
		# More context:
		# https://huggingface.co/wjmcat/opt-350m-paddle/discussions/1
		# https://huggingface.slack.com/archives/C01N44FJDHT/p1653511495183519
		# https://github.com/huggingface/transformers/pull/17088#discussion_r871246439

		A_		:						Tuple	    =   AutoTokenizer.from_pretrained("""facebook/opt-350m"""     ,     from_slow=lowerCAmelCase_							)
		A_		:						List[Any]	    =   """A photo of a cat"""

		A_		:						Any	    =   tokenizer.encode(
		    lowerCAmelCase_     ,     )
		self.assertEqual(lowerCAmelCase_     ,     [2, 250, 1345, 9, 10, 4758]							)
		tokenizer.save_pretrained("""test_opt"""							)

		A_		:						List[Any]	    =   AutoTokenizer.from_pretrained("""./test_opt"""							)
		A_		:						Tuple	    =   tokenizer.encode(
		    lowerCAmelCase_     ,     )
		self.assertEqual(lowerCAmelCase_     ,     [2, 250, 1345, 9, 10, 4758]							)





	def    lowerCamelCase(self							):
		A_		:						Dict	    =   AutoTokenizer.from_pretrained("""facebook/opt-350m"""     ,     use_slow=lowerCAmelCase_							)
		A_		:						Optional[int]	    =   """A photo of a cat"""

		A_		:						List[Any]	    =   tokenizer.encode(
		    lowerCAmelCase_     ,     )
		# Same as above
		self.assertEqual(lowerCAmelCase_     ,     [2, 250, 1345, 9, 10, 4758]							)





	@unittest.skip("""This test is failing because of a bug in the fast tokenizer"""							)
	def    lowerCamelCase(self							):
		A_		:						Union[str, Any]	    =   AutoTokenizer.from_pretrained("""facebook/opt-350m"""     ,     from_slow=lowerCAmelCase_							)

		A_		:						Optional[Any]	    =   """bos"""
		A_		:						Tuple	    =   tokenizer.get_vocab()["""bos"""]

		A_		:						Union[str, Any]	    =   """A photo of a cat"""
		A_		:						List[str]	    =   tokenizer.encode(
		    lowerCAmelCase_     ,     )
		# We changed the bos token
		self.assertEqual(lowerCAmelCase_     ,     [31957, 250, 1345, 9, 10, 4758]							)
		tokenizer.save_pretrained("""./tok"""							)
		A_		:						List[Any]	    =   AutoTokenizer.from_pretrained("""./tok"""							)
		self.assertTrue(tokenizer.is_fast							)
		A_		:						Any	    =   tokenizer.encode(
		    lowerCAmelCase_     ,     )
		self.assertEqual(lowerCAmelCase_     ,     [31957, 250, 1345, 9, 10, 4758]							)









"""simple docstring"""
# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import torch

from ..models.auto import AutoModelForSequenceClassification, AutoTokenizer
from .base import PipelineTool



class      SCREAMING_SNAKE_CASE  (				_SCREAMING_SNAKE_CASE						):





	"""simple docstring"""




	_A       :				Optional[int]    =				"""facebook/bart-large-mnli"""
	_A       :				str    =				(
	    """This is a tool that classifies an English text using provided labels. It takes two inputs: `text`, which """
	    """should be the text to classify, and `labels`, which should be the list of labels to use for classification. """
	    """It returns the most likely label in the list of provided `labels` for the input text."""
	)
	_A       :				List[str]    =				"""text_classifier"""
	_A       :				Optional[int]    =				AutoTokenizer
	_A       :				Optional[Any]    =				AutoModelForSequenceClassification

	_A       :				List[str]    =				["""text""", ["""text"""]]
	_A       :				Dict    =				["""text"""]





	def    lowerCamelCase(self							):
		super().setup()
		A_		:						int	    =   self.model.config
		A_		:						List[str]	    =   -1
		for idx, label in config.idalabel.items():
			if label.lower().startswith("""entail"""							):
				A_		:						List[Any]	    =   int(lowerCAmelCase_							)
		if self.entailment_id == -1:
			raise ValueError("""Could not determine the entailment ID from the model config, please pass it at init."""							)





	def    lowerCamelCase(self     ,     lowerCAmelCase_     ,     lowerCAmelCase_							):
		A_		:						List[Any]	    =   labels
		return self.pre_processor(
		    [text] * len(lowerCAmelCase_							)     ,     [f"""This example is {label}""" for label in labels]     ,     return_tensors="""pt"""     ,     padding="""max_length"""     ,     )





	def    lowerCamelCase(self     ,     lowerCAmelCase_							):
		A_		:						str	    =   outputs.logits
		A_		:						Optional[int]	    =   torch.argmax(logits[:, 2]							).item()
		return self._labels[label_id]

