


"""simple docstring"""
import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class      __A     (snake_case__						,		unittest.TestCase):





							'''simple docstring'''

							__lowercase:		List[Any]							      =    CodeGenTokenizer
							__lowercase:		Any							      =    CodeGenTokenizerFast
							__lowercase:		Dict							      =    True
							__lowercase:		Any							      =    {"""add_prefix_space""": True}
							__lowercase:		Dict							      =    False



							def 		lowerCAmelCase       (				self							:      Union[str, Any]       )					->Optional[int]:



														"""simple docstring"""



														super().setUp()

														# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
														snake_case_           =		[
														    """l""",
														    """o""",
														    """w""",
														    """e""",
														    """r""",
														    """s""",
														    """t""",
														    """i""",
														    """d""",
														    """n""",
														    """\u0120""",
														    """\u0120l""",
														    """\u0120n""",
														    """\u0120lo""",
														    """\u0120low""",
														    """er""",
														    """\u0120lowest""",
														    """\u0120newer""",
														    """\u0120wider""",
														    """<unk>""",
														    """<|endoftext|>""",
														]
														snake_case_           =		dict(zip(UpperCAmelCase_		,       range(len(UpperCAmelCase_       )       )       )       )
														snake_case_           =		["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
														snake_case_           =		{"""unk_token""": """<unk>"""}

														snake_case_           =		os.path.join(self.tmpdirname		,       VOCAB_FILES_NAMES["""vocab_file"""]       )
														snake_case_           =		os.path.join(self.tmpdirname		,       VOCAB_FILES_NAMES["""merges_file"""]       )
														with open(self.vocab_file		,       """w"""		,       encoding="""utf-8"""       ) as fp:
																					fp.write(json.dumps(UpperCAmelCase_       ) + """\n"""       )
														with open(self.merges_file		,       """w"""		,       encoding="""utf-8"""       ) as fp:
																					fp.write("""\n""".join(UpperCAmelCase_       )       )



							def 		lowerCAmelCase       (				self							:      List[str]		,       **UpperCAmelCase_							:      List[Any]       )					->List[Any]:



														"""simple docstring"""



														kwargs.update(self.special_tokens_map       )
														return CodeGenTokenizer.from_pretrained(self.tmpdirname		,       **UpperCAmelCase_       )



							def 		lowerCAmelCase       (				self							:      str		,       **UpperCAmelCase_							:      Union[str, Any]       )					->int:



														"""simple docstring"""



														kwargs.update(self.special_tokens_map       )
														return CodeGenTokenizerFast.from_pretrained(self.tmpdirname		,       **UpperCAmelCase_       )



							def 		lowerCAmelCase       (				self							:      Union[str, Any]		,       UpperCAmelCase_							:      List[Any]       )					->str:



														"""simple docstring"""



														snake_case_           =		"""lower newer"""
														snake_case_           =		"""lower newer"""
														return input_text, output_text



							def 		lowerCAmelCase       (				self							:      str       )					->str:



														"""simple docstring"""



														snake_case_           =		CodeGenTokenizer(self.vocab_file		,       self.merges_file		,       **self.special_tokens_map       )
														snake_case_           =		"""lower newer"""
														snake_case_           =		["""\u0120low""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
														snake_case_           =		tokenizer.tokenize(UpperCAmelCase_		,       add_prefix_space=UpperCAmelCase_       )
														self.assertListEqual(UpperCAmelCase_		,       UpperCAmelCase_       )

														snake_case_           =		tokens + [tokenizer.unk_token]
														snake_case_           =		[14, 15, 10, 9, 3, 2, 15, 19]
														self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_       )		,       UpperCAmelCase_       )



							def 		lowerCAmelCase       (				self							:      int       )					->Optional[Any]:



														"""simple docstring"""



														if not self.test_rust_tokenizer:
																					return

														snake_case_           =		self.get_tokenizer()
														snake_case_           =		self.get_rust_tokenizer(add_prefix_space=UpperCAmelCase_       )

														snake_case_           =		"""lower newer"""

														# Testing tokenization
														snake_case_           =		tokenizer.tokenize(UpperCAmelCase_		,       add_prefix_space=UpperCAmelCase_       )
														snake_case_           =		rust_tokenizer.tokenize(UpperCAmelCase_       )
														self.assertListEqual(UpperCAmelCase_		,       UpperCAmelCase_       )

														# Testing conversion to ids without special tokens
														snake_case_           =		tokenizer.encode(UpperCAmelCase_		,       add_special_tokens=UpperCAmelCase_		,       add_prefix_space=UpperCAmelCase_       )
														snake_case_           =		rust_tokenizer.encode(UpperCAmelCase_		,       add_special_tokens=UpperCAmelCase_       )
														self.assertListEqual(UpperCAmelCase_		,       UpperCAmelCase_       )

														# Testing conversion to ids with special tokens
														snake_case_           =		self.get_rust_tokenizer(add_prefix_space=UpperCAmelCase_       )
														snake_case_           =		tokenizer.encode(UpperCAmelCase_		,       add_prefix_space=UpperCAmelCase_       )
														snake_case_           =		rust_tokenizer.encode(UpperCAmelCase_       )
														self.assertListEqual(UpperCAmelCase_		,       UpperCAmelCase_       )

														# Testing the unknown token
														snake_case_           =		tokens + [rust_tokenizer.unk_token]
														snake_case_           =		[14, 15, 10, 9, 3, 2, 15, 19]
														self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(UpperCAmelCase_       )		,       UpperCAmelCase_       )



							def 		lowerCAmelCase       (				self							:      str		,       *UpperCAmelCase_							:      str		,       **UpperCAmelCase_							:      Optional[Any]       )					->str:



														"""simple docstring"""



														pass



							def 		lowerCAmelCase       (				self							:      Tuple		,       UpperCAmelCase_							:      Optional[Any]=15       )					->str:



														"""simple docstring"""



														for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																					with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
																												snake_case_           =		self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_		,       **UpperCAmelCase_       )

																												# Simple input
																												snake_case_           =		"""This is a simple input"""
																												snake_case_           =		["""This is a simple input 1""", """This is a simple input 2"""]
																												snake_case_           =		("""This is a simple input""", """This is a pair""")
																												snake_case_           =		[
																												    ("""This is a simple input 1""", """This is a simple input 2"""),
																												    ("""This is a simple pair 1""", """This is a simple pair 2"""),
																												]

																												# Simple input tests
																												self.assertRaises(UpperCAmelCase_		,       tokenizer_r.encode		,       UpperCAmelCase_		,       max_length=UpperCAmelCase_		,       padding="""max_length"""       )

																												# Simple input
																												self.assertRaises(UpperCAmelCase_		,       tokenizer_r.encode_plus		,       UpperCAmelCase_		,       max_length=UpperCAmelCase_		,       padding="""max_length"""       )

																												# Simple input
																												self.assertRaises(
																												    UpperCAmelCase_		,       tokenizer_r.batch_encode_plus		,       UpperCAmelCase_		,       max_length=UpperCAmelCase_		,       padding="""max_length"""		,       )

																												# Pair input
																												self.assertRaises(UpperCAmelCase_		,       tokenizer_r.encode		,       UpperCAmelCase_		,       max_length=UpperCAmelCase_		,       padding="""max_length"""       )

																												# Pair input
																												self.assertRaises(UpperCAmelCase_		,       tokenizer_r.encode_plus		,       UpperCAmelCase_		,       max_length=UpperCAmelCase_		,       padding="""max_length"""       )

																												# Pair input
																												self.assertRaises(
																												    UpperCAmelCase_		,       tokenizer_r.batch_encode_plus		,       UpperCAmelCase_		,       max_length=UpperCAmelCase_		,       padding="""max_length"""		,       )



							def 		lowerCAmelCase       (				self							:      str       )					->List[Any]:



														"""simple docstring"""



														snake_case_           =		CodeGenTokenizer.from_pretrained(self.tmpdirname		,       pad_token="""<pad>"""       )

														# Simple input
														snake_case_           =		"""This is a simple input"""
														snake_case_           =		["""This is a simple input looooooooong""", """This is a simple input"""]
														snake_case_           =		("""This is a simple input""", """This is a pair""")
														snake_case_           =		[
														    ("""This is a simple input loooooong""", """This is a simple input"""),
														    ("""This is a simple pair loooooong""", """This is a simple pair"""),
														]

														snake_case_           =		tokenizer.pad_token_id

														snake_case_           =		tokenizer(UpperCAmelCase_		,       padding="""max_length"""		,       max_length=30		,       return_tensors="""np"""       )
														snake_case_           =		tokenizer(UpperCAmelCase_		,       padding=UpperCAmelCase_		,       truncate=UpperCAmelCase_		,       return_tensors="""np"""       )
														snake_case_           =		tokenizer(*UpperCAmelCase_		,       padding="""max_length"""		,       max_length=60		,       return_tensors="""np"""       )
														snake_case_           =		tokenizer(UpperCAmelCase_		,       padding=UpperCAmelCase_		,       truncate=UpperCAmelCase_		,       return_tensors="""np"""       )

														# s
														# test single string max_length padding
														self.assertEqual(out_s["""input_ids"""].shape[-1]		,       30       )
														self.assertTrue(pad_token_id in out_s["""input_ids"""]       )
														self.assertTrue(0 in out_s["""attention_mask"""]       )

														# s2
														# test automatic padding
														self.assertEqual(out_sa["""input_ids"""].shape[-1]		,       33       )
														# long slice doesn't have padding
														self.assertFalse(pad_token_id in out_sa["""input_ids"""][0]       )
														self.assertFalse(0 in out_sa["""attention_mask"""][0]       )
														# short slice does have padding
														self.assertTrue(pad_token_id in out_sa["""input_ids"""][1]       )
														self.assertTrue(0 in out_sa["""attention_mask"""][1]       )

														# p
														# test single pair max_length padding
														self.assertEqual(out_p["""input_ids"""].shape[-1]		,       60       )
														self.assertTrue(pad_token_id in out_p["""input_ids"""]       )
														self.assertTrue(0 in out_p["""attention_mask"""]       )

														# p2
														# test automatic padding pair
														self.assertEqual(out_pa["""input_ids"""].shape[-1]		,       52       )
														# long slice pair doesn't have padding
														self.assertFalse(pad_token_id in out_pa["""input_ids"""][0]       )
														self.assertFalse(0 in out_pa["""attention_mask"""][0]       )
														# short slice pair does have padding
														self.assertTrue(pad_token_id in out_pa["""input_ids"""][1]       )
														self.assertTrue(0 in out_pa["""attention_mask"""][1]       )



							def 		lowerCAmelCase       (				self							:      str       )					->Optional[Any]:



														"""simple docstring"""



														snake_case_           =		"""$$$"""
														snake_case_           =		CodeGenTokenizer.from_pretrained(self.tmpdirname		,       bos_token=UpperCAmelCase_		,       add_bos_token=UpperCAmelCase_       )

														snake_case_           =		"""This is a simple input"""
														snake_case_           =		["""This is a simple input 1""", """This is a simple input 2"""]

														snake_case_           =		tokenizer.bos_token_id

														snake_case_           =		tokenizer(UpperCAmelCase_       )
														snake_case_           =		tokenizer(UpperCAmelCase_       )

														self.assertEqual(out_s.input_ids[0]		,       UpperCAmelCase_       )
														self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids       )       )

														snake_case_           =		tokenizer.decode(out_s.input_ids       )
														snake_case_           =		tokenizer.batch_decode(out_sa.input_ids       )

														self.assertEqual(decode_s.split()[0]		,       UpperCAmelCase_       )
														self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa       )       )



							@slow
							def 		lowerCAmelCase       (				self							:      List[str]       )					->List[Any]:



														"""simple docstring"""



														snake_case_           =		CodeGenTokenizer.from_pretrained("""Salesforce/codegen-350M-mono"""       )

														snake_case_           =		"""\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"""
														snake_case_           =		"""\nif len_a > len_b:      result = a\nelse:      result = b"""

														snake_case_           =		tokenizer.encode(UpperCAmelCase_       )
														snake_case_           =		["""^#""", re.escape("""<|endoftext|>"""       ), """^'''""", """^\"\"\"""", """\n\n\n"""]
														snake_case_           =		tokenizer.decode(UpperCAmelCase_		,       truncate_before_pattern=UpperCAmelCase_       )
														self.assertEqual(UpperCAmelCase_		,       UpperCAmelCase_       )



							def 		lowerCAmelCase       (				self							:      Union[str, Any]       )					->List[Any]:



														"""simple docstring"""



														pass







"""simple docstring"""
import json
import os
import tempfile
import unittest

import numpy as np
from datasets import load_dataset

from transformers.testing_utils import require_torch, require_vision, slow
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin


if is_torch_available():
				import torch

if is_vision_available():
				from PIL import Image

				from transformers import ImageGPTImageProcessor





class      __A     (unittest.TestCase):





							'''simple docstring'''

							def __init__(				self							:      int		,       UpperCAmelCase_							:      List[str]		,       UpperCAmelCase_							:      Tuple=7		,       UpperCAmelCase_							:      Dict=3		,       UpperCAmelCase_							:      Tuple=18		,       UpperCAmelCase_							:      Optional[Any]=30		,       UpperCAmelCase_							:      str=400		,       UpperCAmelCase_							:      int=True		,       UpperCAmelCase_							:      Optional[int]=None		,       UpperCAmelCase_							:      Optional[Any]=True		,       )					->Optional[Any]:



														"""simple docstring"""



														snake_case_           =		size if size is not None else {"""height""": 18, """width""": 18}
														snake_case_           =		parent
														snake_case_           =		batch_size
														snake_case_           =		num_channels
														snake_case_           =		image_size
														snake_case_           =		min_resolution
														snake_case_           =		max_resolution
														snake_case_           =		do_resize
														snake_case_           =		size
														snake_case_           =		do_normalize



							def 		lowerCAmelCase       (				self							:      List[str]       )					->Optional[Any]:



														"""simple docstring"""



														return {
														    # here we create 2 clusters for the sake of simplicity
														    "clusters": np.asarray(
														        [
														            [0.8_866_443_634_033_203, 0.6_618_829_369_544_983, 0.3_891_746_401_786_804],
														            [-0.6_042_559_146_881_104, -0.02_295_008_860_528_469, 0.5_423_797_369_003_296],
														        ]       ),
														    "do_resize": self.do_resize,
														    "size": self.size,
														    "do_normalize": self.do_normalize,
														}





@require_torch
@require_vision
class      __A     (snake_case__						,		unittest.TestCase):





							'''simple docstring'''

							__lowercase:		List[Any]							      =    ImageGPTImageProcessor if is_vision_available() else None



							def 		lowerCAmelCase       (				self							:      Optional[int]       )					->Optional[int]:



														"""simple docstring"""



														snake_case_           =		ImageGPTImageProcessingTester(self       )



							@property
							def 		lowerCAmelCase       (				self							:      Tuple       )					->List[Any]:



														"""simple docstring"""



														return self.image_processor_tester.prepare_image_processor_dict()



							def 		lowerCAmelCase       (				self							:      Optional[Any]       )					->List[str]:



														"""simple docstring"""



														snake_case_           =		self.image_processing_class(**self.image_processor_dict       )
														self.assertTrue(hasattr(UpperCAmelCase_		,       """clusters"""       )       )
														self.assertTrue(hasattr(UpperCAmelCase_		,       """do_resize"""       )       )
														self.assertTrue(hasattr(UpperCAmelCase_		,       """size"""       )       )
														self.assertTrue(hasattr(UpperCAmelCase_		,       """do_normalize"""       )       )



							def 		lowerCAmelCase       (				self							:      Optional[int]       )					->Optional[Any]:



														"""simple docstring"""



														snake_case_           =		self.image_processing_class.from_dict(self.image_processor_dict       )
														self.assertEqual(image_processor.size		,       {"""height""": 18, """width""": 18}       )

														snake_case_           =		self.image_processing_class.from_dict(self.image_processor_dict		,       size=42       )
														self.assertEqual(image_processor.size		,       {"""height""": 42, """width""": 42}       )



							def 		lowerCAmelCase       (				self							:      Any       )					->List[Any]:



														"""simple docstring"""



														snake_case_           =		self.image_processing_class(**self.image_processor_dict       )
														snake_case_           =		json.loads(image_processor.to_json_string()       )
														for key, value in self.image_processor_dict.items():
																					if key == "clusters":
																												self.assertTrue(np.array_equal(UpperCAmelCase_		,       obj[key]       )       )
																					else:
																												self.assertEqual(obj[key]		,       UpperCAmelCase_       )



							def 		lowerCAmelCase       (				self							:      Optional[int]       )					->Dict:



														"""simple docstring"""



														snake_case_           =		self.image_processing_class(**self.image_processor_dict       )

														with tempfile.TemporaryDirectory() as tmpdirname:
																					snake_case_           =		os.path.join(UpperCAmelCase_		,       """image_processor.json"""       )
																					image_processor_first.to_json_file(UpperCAmelCase_       )
																					snake_case_           =		self.image_processing_class.from_json_file(UpperCAmelCase_       ).to_dict()

														snake_case_           =		image_processor_first.to_dict()
														for key, value in image_processor_first.items():
																					if key == "clusters":
																												self.assertTrue(np.array_equal(UpperCAmelCase_		,       image_processor_second[key]       )       )
																					else:
																												self.assertEqual(image_processor_first[key]		,       UpperCAmelCase_       )



							def 		lowerCAmelCase       (				self							:      Optional[Any]       )					->List[Any]:



														"""simple docstring"""



														snake_case_           =		self.image_processing_class(**self.image_processor_dict       )

														with tempfile.TemporaryDirectory() as tmpdirname:
																					image_processor_first.save_pretrained(UpperCAmelCase_       )
																					snake_case_           =		self.image_processing_class.from_pretrained(UpperCAmelCase_       ).to_dict()

														snake_case_           =		image_processor_first.to_dict()
														for key, value in image_processor_first.items():
																					if key == "clusters":
																												self.assertTrue(np.array_equal(UpperCAmelCase_		,       image_processor_second[key]       )       )
																					else:
																												self.assertEqual(image_processor_first[key]		,       UpperCAmelCase_       )



							@unittest.skip("""ImageGPT requires clusters at initialization"""       )
							def 		lowerCAmelCase       (				self							:      List[Any]       )					->Tuple:



														"""simple docstring"""



														pass







def 						_a				(      )    ->				str:
							snake_case_           =		load_dataset("""hf-internal-testing/fixtures_image_utils"""				, split="""test"""				)

							snake_case_           =		Image.open(dataset[4]["""file"""]				)
							snake_case_           =		Image.open(dataset[5]["""file"""]				)

							snake_case_           =		[imagea, imagea]

							return images





@require_vision
@require_torch
class      __A     (unittest.TestCase):





							'''simple docstring'''

							@slow
							def 		lowerCAmelCase       (				self							:      Tuple       )					->List[str]:



														"""simple docstring"""



														snake_case_           =		ImageGPTImageProcessor.from_pretrained("""openai/imagegpt-small"""       )

														snake_case_           =		prepare_images()

														# test non-batched
														snake_case_           =		image_processing(images[0]		,       return_tensors="""pt"""       )

														self.assertIsInstance(encoding.input_ids		,       torch.LongTensor       )
														self.assertEqual(encoding.input_ids.shape		,       (1, 1_024)       )

														snake_case_           =		[306, 191, 191]
														self.assertEqual(encoding.input_ids[0, :3].tolist()		,       UpperCAmelCase_       )

														# test batched
														snake_case_           =		image_processing(UpperCAmelCase_		,       return_tensors="""pt"""       )

														self.assertIsInstance(encoding.input_ids		,       torch.LongTensor       )
														self.assertEqual(encoding.input_ids.shape		,       (2, 1_024)       )

														snake_case_           =		[303, 13, 13]
														self.assertEqual(encoding.input_ids[1, -3:].tolist()		,       UpperCAmelCase_       )



