





import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class    _a     (				lowercase__					,							unittest.TestCase							):






							"""simple docstring"""

							snake_case_							 =						CodeGenTokenizer
							snake_case_							 =						CodeGenTokenizerFast
							snake_case_							 =						True
							snake_case_							 =						{"add_prefix_space": True}
							snake_case_							 =						False

							def      A_  (      self				:     List[Any]    )	->List[str]:
													super().setUp()

													# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
													SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					[
													    "l",
													    "o",
													    "w",
													    "e",
													    "r",
													    "s",
													    "t",
													    "i",
													    "d",
													    "n",
													    "\u0120",
													    "\u0120l",
													    "\u0120n",
													    "\u0120lo",
													    "\u0120low",
													    "er",
													    "\u0120lowest",
													    "\u0120newer",
													    "\u0120wider",
													    "<unk>",
													    "<|endoftext|>",
													]
													SCREAMING_SNAKE_CASE__     :				Union[str, Any]   =					dict(zip(a							,	range(len(a    )    )    )    )
													SCREAMING_SNAKE_CASE__     :				Any   =					["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
													SCREAMING_SNAKE_CASE__     :				List[str]   =					{"unk_token": "<unk>"}

													SCREAMING_SNAKE_CASE__     :				Optional[int]   =					os.path.join(self.tmpdirname							,	VOCAB_FILES_NAMES["vocab_file"]    )
													SCREAMING_SNAKE_CASE__     :				str   =					os.path.join(self.tmpdirname							,	VOCAB_FILES_NAMES["merges_file"]    )
													with open(self.vocab_file							,	"w"							,	encoding="utf-8"    ) as fp:
																			fp.write(json.dumps(a    ) + "\n"    )
													with open(self.merges_file							,	"w"							,	encoding="utf-8"    ) as fp:
																			fp.write("\n".join(a    )    )

							def      A_  (      self				:     int							,	**a				:     str    )	->Dict:
													kwargs.update(self.special_tokens_map    )
													return CodeGenTokenizer.from_pretrained(self.tmpdirname							,	**a    )

							def      A_  (      self				:     int							,	**a				:     List[str]    )	->Tuple:
													kwargs.update(self.special_tokens_map    )
													return CodeGenTokenizerFast.from_pretrained(self.tmpdirname							,	**a    )

							def      A_  (      self				:     Dict							,	a				:     Any    )	->Any:
													SCREAMING_SNAKE_CASE__     :				Tuple   =					"lower newer"
													SCREAMING_SNAKE_CASE__     :				Optional[int]   =					"lower newer"
													return input_text, output_text

							def      A_  (      self				:     Optional[int]    )	->int:
													SCREAMING_SNAKE_CASE__     :				Any   =					CodeGenTokenizer(self.vocab_file							,	self.merges_file							,	**self.special_tokens_map    )
													SCREAMING_SNAKE_CASE__     :				int   =					"lower newer"
													SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					["\u0120low", "er", "\u0120", "n", "e", "w", "er"]
													SCREAMING_SNAKE_CASE__     :				Tuple   =					tokenizer.tokenize(a							,	add_prefix_space=a    )
													self.assertListEqual(a							,	a    )

													SCREAMING_SNAKE_CASE__     :				List[str]   =					tokens + [tokenizer.unk_token]
													SCREAMING_SNAKE_CASE__     :				Union[str, Any]   =					[14, 15, 10, 9, 3, 2, 15, 19]
													self.assertListEqual(tokenizer.convert_tokens_to_ids(a    )							,	a    )

							def      A_  (      self				:     Union[str, Any]    )	->Any:
													if not self.test_rust_tokenizer:
																			return

													SCREAMING_SNAKE_CASE__     :				Tuple   =					self.get_tokenizer()
													SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					self.get_rust_tokenizer(add_prefix_space=a    )

													SCREAMING_SNAKE_CASE__     :				Any   =					"lower newer"

													# Testing tokenization
													SCREAMING_SNAKE_CASE__     :				Any   =					tokenizer.tokenize(a							,	add_prefix_space=a    )
													SCREAMING_SNAKE_CASE__     :				Dict   =					rust_tokenizer.tokenize(a    )
													self.assertListEqual(a							,	a    )

													# Testing conversion to ids without special tokens
													SCREAMING_SNAKE_CASE__     :				List[str]   =					tokenizer.encode(a							,	add_special_tokens=a							,	add_prefix_space=a    )
													SCREAMING_SNAKE_CASE__     :				List[str]   =					rust_tokenizer.encode(a							,	add_special_tokens=a    )
													self.assertListEqual(a							,	a    )

													# Testing conversion to ids with special tokens
													SCREAMING_SNAKE_CASE__     :				Optional[int]   =					self.get_rust_tokenizer(add_prefix_space=a    )
													SCREAMING_SNAKE_CASE__     :				List[str]   =					tokenizer.encode(a							,	add_prefix_space=a    )
													SCREAMING_SNAKE_CASE__     :				Any   =					rust_tokenizer.encode(a    )
													self.assertListEqual(a							,	a    )

													# Testing the unknown token
													SCREAMING_SNAKE_CASE__     :				Any   =					tokens + [rust_tokenizer.unk_token]
													SCREAMING_SNAKE_CASE__     :				Dict   =					[14, 15, 10, 9, 3, 2, 15, 19]
													self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(a    )							,	a    )

							def      A_  (      self				:     Any							,	*a				:     Tuple							,	**a				:     Optional[int]    )	->Tuple:
													# It's very difficult to mix/test pretokenization with byte-level
													# And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
													pass

							def      A_  (      self				:     Dict							,	a				:     Optional[int]=15    )	->Optional[int]:
													for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																			with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""    ):
																									SCREAMING_SNAKE_CASE__     :				Any   =					self.rust_tokenizer_class.from_pretrained(a							,	**a    )

																									# Simple input
																									SCREAMING_SNAKE_CASE__     :				int   =					"This is a simple input"
																									SCREAMING_SNAKE_CASE__     :				List[Any]   =					["This is a simple input 1", "This is a simple input 2"]
																									SCREAMING_SNAKE_CASE__     :				Union[str, Any]   =					("This is a simple input", "This is a pair")
																									SCREAMING_SNAKE_CASE__     :				Any   =					[
																									    ("This is a simple input 1", "This is a simple input 2"),
																									    ("This is a simple pair 1", "This is a simple pair 2"),
																									]

																									# Simple input tests
																									self.assertRaises(a							,	tokenizer_r.encode							,	a							,	max_length=a							,	padding="max_length"    )

																									# Simple input
																									self.assertRaises(a							,	tokenizer_r.encode_plus							,	a							,	max_length=a							,	padding="max_length"    )

																									# Simple input
																									self.assertRaises(
																									    a							,	tokenizer_r.batch_encode_plus							,	a							,	max_length=a							,	padding="max_length"							,	)

																									# Pair input
																									self.assertRaises(a							,	tokenizer_r.encode							,	a							,	max_length=a							,	padding="max_length"    )

																									# Pair input
																									self.assertRaises(a							,	tokenizer_r.encode_plus							,	a							,	max_length=a							,	padding="max_length"    )

																									# Pair input
																									self.assertRaises(
																									    a							,	tokenizer_r.batch_encode_plus							,	a							,	max_length=a							,	padding="max_length"							,	)

							def      A_  (      self				:     List[str]    )	->List[str]:
													SCREAMING_SNAKE_CASE__     :				int   =					CodeGenTokenizer.from_pretrained(self.tmpdirname							,	pad_token="<pad>"    )

													# Simple input
													SCREAMING_SNAKE_CASE__     :				Union[str, Any]   =					"This is a simple input"
													SCREAMING_SNAKE_CASE__     :				Tuple   =					["This is a simple input looooooooong", "This is a simple input"]
													SCREAMING_SNAKE_CASE__     :				Optional[int]   =					("This is a simple input", "This is a pair")
													SCREAMING_SNAKE_CASE__     :				List[Any]   =					[
													    ("This is a simple input loooooong", "This is a simple input"),
													    ("This is a simple pair loooooong", "This is a simple pair"),
													]

													SCREAMING_SNAKE_CASE__     :				List[str]   =					tokenizer.pad_token_id

													SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					tokenizer(a							,	padding="max_length"							,	max_length=30							,	return_tensors="np"    )
													SCREAMING_SNAKE_CASE__     :				Dict   =					tokenizer(a							,	padding=a							,	truncate=a							,	return_tensors="np"    )
													SCREAMING_SNAKE_CASE__     :				Tuple   =					tokenizer(*a							,	padding="max_length"							,	max_length=60							,	return_tensors="np"    )
													SCREAMING_SNAKE_CASE__     :				Dict   =					tokenizer(a							,	padding=a							,	truncate=a							,	return_tensors="np"    )

													# s
													# test single string max_length padding
													self.assertEqual(out_s["input_ids"].shape[-1]							,	30    )
													self.assertTrue(pad_token_id in out_s["input_ids"]    )
													self.assertTrue(0 in out_s["attention_mask"]    )

													# s2
													# test automatic padding
													self.assertEqual(out_sa["input_ids"].shape[-1]							,	33    )
													# long slice doesn't have padding
													self.assertFalse(pad_token_id in out_sa["input_ids"][0]    )
													self.assertFalse(0 in out_sa["attention_mask"][0]    )
													# short slice does have padding
													self.assertTrue(pad_token_id in out_sa["input_ids"][1]    )
													self.assertTrue(0 in out_sa["attention_mask"][1]    )

													# p
													# test single pair max_length padding
													self.assertEqual(out_p["input_ids"].shape[-1]							,	60    )
													self.assertTrue(pad_token_id in out_p["input_ids"]    )
													self.assertTrue(0 in out_p["attention_mask"]    )

													# p2
													# test automatic padding pair
													self.assertEqual(out_pa["input_ids"].shape[-1]							,	52    )
													# long slice pair doesn't have padding
													self.assertFalse(pad_token_id in out_pa["input_ids"][0]    )
													self.assertFalse(0 in out_pa["attention_mask"][0]    )
													# short slice pair does have padding
													self.assertTrue(pad_token_id in out_pa["input_ids"][1]    )
													self.assertTrue(0 in out_pa["attention_mask"][1]    )

							def      A_  (      self				:     Optional[int]    )	->Dict:
													SCREAMING_SNAKE_CASE__     :				List[Any]   =					"$$$"
													SCREAMING_SNAKE_CASE__     :				str   =					CodeGenTokenizer.from_pretrained(self.tmpdirname							,	bos_token=a							,	add_bos_token=a    )

													SCREAMING_SNAKE_CASE__     :				Union[str, Any]   =					"This is a simple input"
													SCREAMING_SNAKE_CASE__     :				List[Any]   =					["This is a simple input 1", "This is a simple input 2"]

													SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					tokenizer.bos_token_id

													SCREAMING_SNAKE_CASE__     :				Dict   =					tokenizer(a    )
													SCREAMING_SNAKE_CASE__     :				List[str]   =					tokenizer(a    )

													self.assertEqual(out_s.input_ids[0]							,	a    )
													self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids    )    )

													SCREAMING_SNAKE_CASE__     :				Tuple   =					tokenizer.decode(out_s.input_ids    )
													SCREAMING_SNAKE_CASE__     :				Tuple   =					tokenizer.batch_decode(out_sa.input_ids    )

													self.assertEqual(decode_s.split()[0]							,	a    )
													self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa    )    )

							@slow
							def      A_  (      self				:     int    )	->Dict:
													SCREAMING_SNAKE_CASE__     :				Dict   =					CodeGenTokenizer.from_pretrained("Salesforce/codegen-350M-mono"    )

													SCREAMING_SNAKE_CASE__     :				List[str]   =					"\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"
													SCREAMING_SNAKE_CASE__     :				Optional[int]   =					"\nif len_a > len_b:      result = a\nelse:      result = b"

													SCREAMING_SNAKE_CASE__     :				Any   =					tokenizer.encode(a    )
													SCREAMING_SNAKE_CASE__     :				List[str]   =					["^#", re.escape("<|endoftext|>"    ), "^'''", "^\"\"\"", "\n\n\n"]
													SCREAMING_SNAKE_CASE__     :				Dict   =					tokenizer.decode(a							,	truncate_before_pattern=a    )
													self.assertEqual(a							,	a    )






							def      A_  (      self				:     Union[str, Any]    )	->Tuple:
													pass






import unittest

from datasets import load_dataset

from transformers.pipelines import pipeline
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_torch, slow



@is_pipeline_test
@require_torch
class    _a     (				unittest.TestCase							):






							"""simple docstring"""

							@require_torch
							def      A_  (      self				:     Dict    )	->str:
													SCREAMING_SNAKE_CASE__     :				Any   =					pipeline(
													    task="zero-shot-audio-classification"							,	model="hf-internal-testing/tiny-clap-htsat-unfused"    )
													SCREAMING_SNAKE_CASE__     :				Optional[int]   =					load_dataset("ashraq/esc50"    )
													SCREAMING_SNAKE_CASE__     :				Optional[int]   =					dataset["train"]["audio"][-1]["array"]
													SCREAMING_SNAKE_CASE__     :				int   =					audio_classifier(a							,	candidate_labels=["Sound of a dog", "Sound of vaccum cleaner"]    )
													self.assertEqual(
													    nested_simplify(a    )							,	[{"score": 0.501, "label": "Sound of a dog"}, {"score": 0.499, "label": "Sound of vaccum cleaner"}]							,	)

							@unittest.skip("No models are available in TF"    )
							def      A_  (      self				:     int    )	->Union[str, Any]:
													pass

							@slow
							@require_torch
							def      A_  (      self				:     int    )	->str:
													SCREAMING_SNAKE_CASE__     :				List[str]   =					pipeline(
													    task="zero-shot-audio-classification"							,	model="laion/clap-htsat-unfused"							,	)
													# This is an audio of a dog
													SCREAMING_SNAKE_CASE__     :				int   =					load_dataset("ashraq/esc50"    )
													SCREAMING_SNAKE_CASE__     :				str   =					dataset["train"]["audio"][-1]["array"]
													SCREAMING_SNAKE_CASE__     :				List[Any]   =					audio_classifier(a							,	candidate_labels=["Sound of a dog", "Sound of vaccum cleaner"]    )

													self.assertEqual(
													    nested_simplify(a    )							,	[
													        {"score": 0.999, "label": "Sound of a dog"},
													        {"score": 0.001, "label": "Sound of vaccum cleaner"},
													    ]							,	)

													SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					audio_classifier([audio] * 5							,	candidate_labels=["Sound of a dog", "Sound of vaccum cleaner"]    )
													self.assertEqual(
													    nested_simplify(a    )							,	[
													        [
													            {"score": 0.999, "label": "Sound of a dog"},
													            {"score": 0.001, "label": "Sound of vaccum cleaner"},
													        ],
													    ]
													    * 5							,	)
													SCREAMING_SNAKE_CASE__     :				int   =					audio_classifier(
													    [audio] * 5							,	candidate_labels=["Sound of a dog", "Sound of vaccum cleaner"]							,	batch_size=5    )
													self.assertEqual(
													    nested_simplify(a    )							,	[
													        [
													            {"score": 0.999, "label": "Sound of a dog"},
													            {"score": 0.001, "label": "Sound of vaccum cleaner"},
													        ],
													    ]
													    * 5							,	)






							@unittest.skip("No models are available in TF"    )
							def      A_  (      self				:     Optional[int]    )	->Union[str, Any]:
													pass