



import json
import pathlib
import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision, slow
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
 import torch

if is_vision_available():
 from PIL import Image

 from transformers import YolosImageProcessor

class 					_A (			unittest.TestCase					):



       def __init__(	self							:  List[Any]  ,	__SCREAMING_SNAKE_CASE							:  Optional[int]  ,	__SCREAMING_SNAKE_CASE							:  List[str]=7  ,	__SCREAMING_SNAKE_CASE							:  Tuple=3  ,	__SCREAMING_SNAKE_CASE							:  Tuple=30  ,	__SCREAMING_SNAKE_CASE							:  Dict=400  ,	__SCREAMING_SNAKE_CASE							:  Dict=True  ,	__SCREAMING_SNAKE_CASE							:  Optional[int]=None  ,	__SCREAMING_SNAKE_CASE							:  List[Any]=True  ,	__SCREAMING_SNAKE_CASE							:  Dict=[0.5, 0.5, 0.5]  ,	__SCREAMING_SNAKE_CASE							:  List[str]=[0.5, 0.5, 0.5]  ,	__SCREAMING_SNAKE_CASE							:  Optional[Any]=True  ,	__SCREAMING_SNAKE_CASE							:  Union[str, Any]=1 / 255  ,	__SCREAMING_SNAKE_CASE							:  Dict=True  ,	):


           '''simple docstring'''

           __a		 =       size if size is not None else {'''shortest_edge''': 18, '''longest_edge''': 1_333}
           __a		 =       parent
           __a		 =       batch_size
           __a		 =       num_channels
           __a		 =       min_resolution
           __a		 =       max_resolution
           __a		 =       do_resize
           __a		 =       size
           __a		 =       do_normalize
           __a		 =       image_mean
           __a		 =       image_std
           __a		 =       do_rescale
           __a		 =       rescale_factor
           __a		 =       do_pad



       def 	_lowerCamelCase    (	self							:  List[str]):


           '''simple docstring'''

           return {
               "do_resize": self.do_resize,
               "size": self.size,
               "do_normalize": self.do_normalize,
               "image_mean": self.image_mean,
               "image_std": self.image_std,
               "do_rescale": self.do_rescale,
               "rescale_factor": self.rescale_factor,
               "do_pad": self.do_pad,
           }





       def 	_lowerCamelCase    (	self							:  Union[str, Any]  ,	__SCREAMING_SNAKE_CASE							:  List[str]  ,	__SCREAMING_SNAKE_CASE							:  Optional[Any]=False):


           '''simple docstring'''

           if not batched:
               __a		 =       image_inputs[0]
               if isinstance(__SCREAMING_SNAKE_CASE  ,	Image.Image):
                   __a ,     __a		 =       image.size
               else:
                   __a ,     __a		 =       image.shape[1], image.shape[2]
               if w < h:
                   __a		 =       int(self.size['''shortest_edge'''] * h / w)
                   __a		 =       self.size['''shortest_edge''']
               elif w > h:
                   __a		 =       self.size['''shortest_edge''']
                   __a		 =       int(self.size['''shortest_edge'''] * w / h)
               else:
                   __a		 =       self.size['''shortest_edge''']
                   __a		 =       self.size['''shortest_edge''']

           else:
               __a		 =       []
               for image in image_inputs:
                   __a ,     __a		 =       self.get_expected_values([image])
                   expected_values.append((expected_height, expected_width))
               __a		 =       max(__SCREAMING_SNAKE_CASE  ,	key=lambda __SCREAMING_SNAKE_CASE: item[0])[0]
               __a		 =       max(__SCREAMING_SNAKE_CASE  ,	key=lambda __SCREAMING_SNAKE_CASE: item[1])[1]

           return expected_height, expected_width


@require_torch
@require_vision
class 					_A (			__UpperCAmelCase   ,unittest.TestCase					):
       UpperCamelCase__						:					str		      =		YolosImageProcessor if is_vision_available() else None



       def 	_lowerCamelCase    (	self							:  Any):


           '''simple docstring'''

           __a		 =       YolosImageProcessingTester(self)



       @property
       def 	_lowerCamelCase    (	self							:  Any):


           '''simple docstring'''

           return self.image_processor_tester.prepare_image_processor_dict()



       def 	_lowerCamelCase    (	self							:  List[str]):


           '''simple docstring'''

           __a		 =       self.image_processing_class(**self.image_processor_dict)
           self.assertTrue(hasattr(__SCREAMING_SNAKE_CASE  ,	'''image_mean'''))
           self.assertTrue(hasattr(__SCREAMING_SNAKE_CASE  ,	'''image_std'''))
           self.assertTrue(hasattr(__SCREAMING_SNAKE_CASE  ,	'''do_normalize'''))
           self.assertTrue(hasattr(__SCREAMING_SNAKE_CASE  ,	'''do_resize'''))
           self.assertTrue(hasattr(__SCREAMING_SNAKE_CASE  ,	'''size'''))



       def 	_lowerCamelCase    (	self							:  int):


           '''simple docstring'''

           __a		 =       self.image_processing_class.from_dict(self.image_processor_dict)
           self.assertEqual(image_processor.size  ,	{'''shortest_edge''': 18, '''longest_edge''': 1_333})
           self.assertEqual(image_processor.do_pad  ,	__SCREAMING_SNAKE_CASE)

           __a		 =       self.image_processing_class.from_dict(
               self.image_processor_dict  ,	size=42  ,	max_size=84  ,	pad_and_return_pixel_mask=__SCREAMING_SNAKE_CASE)
           self.assertEqual(image_processor.size  ,	{'''shortest_edge''': 42, '''longest_edge''': 84})
           self.assertEqual(image_processor.do_pad  ,	__SCREAMING_SNAKE_CASE)



       def 	_lowerCamelCase    (	self							:  Tuple):


           '''simple docstring'''

           pass



       def 	_lowerCamelCase    (	self							:  Any):


           '''simple docstring'''

           __a		 =       self.image_processing_class(**self.image_processor_dict)
           # create random PIL images
           __a		 =       prepare_image_inputs(self.image_processor_tester  ,	equal_resolution=__SCREAMING_SNAKE_CASE)
           for image in image_inputs:
               self.assertIsInstance(__SCREAMING_SNAKE_CASE  ,	Image.Image)

           # Test not batched input
           __a		 =       image_processing(image_inputs[0]  ,	return_tensors='''pt''').pixel_values

           __a ,     __a		 =       self.image_processor_tester.get_expected_values(__SCREAMING_SNAKE_CASE)

           self.assertEqual(
               encoded_images.shape  ,	(1, self.image_processor_tester.num_channels, expected_height, expected_width)  ,	)

           # Test batched
           __a ,     __a		 =       self.image_processor_tester.get_expected_values(__SCREAMING_SNAKE_CASE  ,	batched=__SCREAMING_SNAKE_CASE)

           __a		 =       image_processing(__SCREAMING_SNAKE_CASE  ,	return_tensors='''pt''').pixel_values
           self.assertEqual(
               encoded_images.shape  ,	(
                   self.image_processor_tester.batch_size,
                   self.image_processor_tester.num_channels,
                   expected_height,
                   expected_width,
               )  ,	)



       def 	_lowerCamelCase    (	self							:  Union[str, Any]):


           '''simple docstring'''

           __a		 =       self.image_processing_class(**self.image_processor_dict)
           # create random numpy tensors
           __a		 =       prepare_image_inputs(self.image_processor_tester  ,	equal_resolution=__SCREAMING_SNAKE_CASE  ,	numpify=__SCREAMING_SNAKE_CASE)
           for image in image_inputs:
               self.assertIsInstance(__SCREAMING_SNAKE_CASE  ,	np.ndarray)

           # Test not batched input
           __a		 =       image_processing(image_inputs[0]  ,	return_tensors='''pt''').pixel_values

           __a ,     __a		 =       self.image_processor_tester.get_expected_values(__SCREAMING_SNAKE_CASE)

           self.assertEqual(
               encoded_images.shape  ,	(1, self.image_processor_tester.num_channels, expected_height, expected_width)  ,	)

           # Test batched
           __a		 =       image_processing(__SCREAMING_SNAKE_CASE  ,	return_tensors='''pt''').pixel_values

           __a ,     __a		 =       self.image_processor_tester.get_expected_values(__SCREAMING_SNAKE_CASE  ,	batched=__SCREAMING_SNAKE_CASE)

           self.assertEqual(
               encoded_images.shape  ,	(
                   self.image_processor_tester.batch_size,
                   self.image_processor_tester.num_channels,
                   expected_height,
                   expected_width,
               )  ,	)



       def 	_lowerCamelCase    (	self							:  Union[str, Any]):


           '''simple docstring'''

           __a		 =       self.image_processing_class(**self.image_processor_dict)
           # create random PyTorch tensors
           __a		 =       prepare_image_inputs(self.image_processor_tester  ,	equal_resolution=__SCREAMING_SNAKE_CASE  ,	torchify=__SCREAMING_SNAKE_CASE)
           for image in image_inputs:
               self.assertIsInstance(__SCREAMING_SNAKE_CASE  ,	torch.Tensor)

           # Test not batched input
           __a		 =       image_processing(image_inputs[0]  ,	return_tensors='''pt''').pixel_values

           __a ,     __a		 =       self.image_processor_tester.get_expected_values(__SCREAMING_SNAKE_CASE)

           self.assertEqual(
               encoded_images.shape  ,	(1, self.image_processor_tester.num_channels, expected_height, expected_width)  ,	)

           # Test batched
           __a		 =       image_processing(__SCREAMING_SNAKE_CASE  ,	return_tensors='''pt''').pixel_values

           __a ,     __a		 =       self.image_processor_tester.get_expected_values(__SCREAMING_SNAKE_CASE  ,	batched=__SCREAMING_SNAKE_CASE)

           self.assertEqual(
               encoded_images.shape  ,	(
                   self.image_processor_tester.batch_size,
                   self.image_processor_tester.num_channels,
                   expected_height,
                   expected_width,
               )  ,	)



       def 	_lowerCamelCase    (	self							:  List[Any]):


           '''simple docstring'''

           __a		 =       self.image_processing_class(**self.image_processor_dict)
           __a		 =       self.image_processing_class(do_resize=__SCREAMING_SNAKE_CASE  ,	do_normalize=__SCREAMING_SNAKE_CASE  ,	do_rescale=__SCREAMING_SNAKE_CASE)
           # create random PyTorch tensors
           __a		 =       prepare_image_inputs(self.image_processor_tester  ,	equal_resolution=__SCREAMING_SNAKE_CASE  ,	torchify=__SCREAMING_SNAKE_CASE)
           for image in image_inputs:
               self.assertIsInstance(__SCREAMING_SNAKE_CASE  ,	torch.Tensor)

           # Test whether the method "pad" and calling the image processor return the same tensors
           __a		 =       image_processing_a.pad(__SCREAMING_SNAKE_CASE  ,	return_tensors='''pt''')
           __a		 =       image_processing_a(__SCREAMING_SNAKE_CASE  ,	return_tensors='''pt''')

           self.assertTrue(
               torch.allclose(encoded_images_with_method['''pixel_values''']  ,	encoded_images['''pixel_values''']  ,	atol=1E-4))



       @slow
       def 	_lowerCamelCase    (	self							:  List[Any]):


           '''simple docstring'''

           __a		 =       Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png''')
           with open('''./tests/fixtures/tests_samples/COCO/coco_annotations.txt'''  ,	'''r''') as f:
               __a		 =       json.loads(f.read())

           __a		 =       {'''image_id''': 39_769, '''annotations''': target}

           # encode them
           __a		 =       YolosImageProcessor.from_pretrained('''hustvl/yolos-small''')
           __a		 =       image_processing(images=__SCREAMING_SNAKE_CASE  ,	annotations=__SCREAMING_SNAKE_CASE  ,	return_tensors='''pt''')

           # verify pixel values
           __a		 =       torch.Size([1, 3, 800, 1_066])
           self.assertEqual(encoding['''pixel_values'''].shape  ,	__SCREAMING_SNAKE_CASE)

           __a		 =       torch.tensor([0.27_96, 0.31_38, 0.34_81])
           self.assertTrue(torch.allclose(encoding['''pixel_values'''][0, 0, 0, :3]  ,	__SCREAMING_SNAKE_CASE  ,	atol=1E-4))

           # verify area
           __a		 =       torch.tensor([58_87.96_00, 1_12_50.20_61, 48_93_53.84_38, 83_71_22.75_00, 14_79_67.51_56, 16_57_32.34_38])
           self.assertTrue(torch.allclose(encoding['''labels'''][0]['''area''']  ,	__SCREAMING_SNAKE_CASE))
           # verify boxes
           __a		 =       torch.Size([6, 4])
           self.assertEqual(encoding['''labels'''][0]['''boxes'''].shape  ,	__SCREAMING_SNAKE_CASE)
           __a		 =       torch.tensor([0.55_03, 0.27_65, 0.06_04, 0.22_15])
           self.assertTrue(torch.allclose(encoding['''labels'''][0]['''boxes'''][0]  ,	__SCREAMING_SNAKE_CASE  ,	atol=1E-3))
           # verify image_id
           __a		 =       torch.tensor([39_769])
           self.assertTrue(torch.allclose(encoding['''labels'''][0]['''image_id''']  ,	__SCREAMING_SNAKE_CASE))
           # verify is_crowd
           __a		 =       torch.tensor([0, 0, 0, 0, 0, 0])
           self.assertTrue(torch.allclose(encoding['''labels'''][0]['''iscrowd''']  ,	__SCREAMING_SNAKE_CASE))
           # verify class_labels
           __a		 =       torch.tensor([75, 75, 63, 65, 17, 17])
           self.assertTrue(torch.allclose(encoding['''labels'''][0]['''class_labels''']  ,	__SCREAMING_SNAKE_CASE))
           # verify orig_size
           __a		 =       torch.tensor([480, 640])
           self.assertTrue(torch.allclose(encoding['''labels'''][0]['''orig_size''']  ,	__SCREAMING_SNAKE_CASE))
           # verify size
           __a		 =       torch.tensor([800, 1_066])
           self.assertTrue(torch.allclose(encoding['''labels'''][0]['''size''']  ,	__SCREAMING_SNAKE_CASE))





       @slow
       def 	_lowerCamelCase    (	self							:  Optional[Any]):


           '''simple docstring'''

           __a		 =       Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png''')
           with open('''./tests/fixtures/tests_samples/COCO/coco_panoptic_annotations.txt'''  ,	'''r''') as f:
               __a		 =       json.loads(f.read())

           __a		 =       {'''file_name''': '''000000039769.png''', '''image_id''': 39_769, '''segments_info''': target}

           __a		 =       pathlib.Path('''./tests/fixtures/tests_samples/COCO/coco_panoptic''')

           # encode them
           __a		 =       YolosImageProcessor(format='''coco_panoptic''')
           __a		 =       image_processing(images=__SCREAMING_SNAKE_CASE  ,	annotations=__SCREAMING_SNAKE_CASE  ,	masks_path=__SCREAMING_SNAKE_CASE  ,	return_tensors='''pt''')

           # verify pixel values
           __a		 =       torch.Size([1, 3, 800, 1_066])
           self.assertEqual(encoding['''pixel_values'''].shape  ,	__SCREAMING_SNAKE_CASE)

           __a		 =       torch.tensor([0.27_96, 0.31_38, 0.34_81])
           self.assertTrue(torch.allclose(encoding['''pixel_values'''][0, 0, 0, :3]  ,	__SCREAMING_SNAKE_CASE  ,	atol=1E-4))

           # verify area
           __a		 =       torch.tensor([14_79_79.68_75, 16_55_27.04_69, 48_46_38.59_38, 1_12_92.93_75, 58_79.65_62, 76_34.11_47])
           self.assertTrue(torch.allclose(encoding['''labels'''][0]['''area''']  ,	__SCREAMING_SNAKE_CASE))
           # verify boxes
           __a		 =       torch.Size([6, 4])
           self.assertEqual(encoding['''labels'''][0]['''boxes'''].shape  ,	__SCREAMING_SNAKE_CASE)
           __a		 =       torch.tensor([0.26_25, 0.54_37, 0.46_88, 0.86_25])
           self.assertTrue(torch.allclose(encoding['''labels'''][0]['''boxes'''][0]  ,	__SCREAMING_SNAKE_CASE  ,	atol=1E-3))
           # verify image_id
           __a		 =       torch.tensor([39_769])
           self.assertTrue(torch.allclose(encoding['''labels'''][0]['''image_id''']  ,	__SCREAMING_SNAKE_CASE))
           # verify is_crowd
           __a		 =       torch.tensor([0, 0, 0, 0, 0, 0])
           self.assertTrue(torch.allclose(encoding['''labels'''][0]['''iscrowd''']  ,	__SCREAMING_SNAKE_CASE))
           # verify class_labels
           __a		 =       torch.tensor([17, 17, 63, 75, 75, 93])
           self.assertTrue(torch.allclose(encoding['''labels'''][0]['''class_labels''']  ,	__SCREAMING_SNAKE_CASE))
           # verify masks
           __a		 =       822_873
           self.assertEqual(encoding['''labels'''][0]['''masks'''].sum().item()  ,	__SCREAMING_SNAKE_CASE)
           # verify orig_size
           __a		 =       torch.tensor([480, 640])
           self.assertTrue(torch.allclose(encoding['''labels'''][0]['''orig_size''']  ,	__SCREAMING_SNAKE_CASE))
           # verify size
           __a		 =       torch.tensor([800, 1_066])
           self.assertTrue(torch.allclose(encoding['''labels'''][0]['''size''']  ,	__SCREAMING_SNAKE_CASE))










import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__snake_case		:Dict   	=  get_tests_dir('''fixtures/test_sentencepiece_with_bytefallback.model''')

@require_sentencepiece
@require_tokenizers
class 					_A (			__UpperCAmelCase   ,unittest.TestCase					):
       UpperCamelCase__						:					List[str]		      =		GPTSwaTokenizer
       UpperCamelCase__						:					Dict		      =		False
       UpperCamelCase__						:					int		      =		True
       UpperCamelCase__						:					List[Any]		      =		False



       def 	_lowerCamelCase    (	self							:  List[Any]):


           '''simple docstring'''

           super().setUp()

           # We have a SentencePiece fixture for testing
           __a		 =       GPTSwaTokenizer(__SCREAMING_SNAKE_CASE  ,	eos_token='''<unk>'''  ,	bos_token='''<unk>'''  ,	pad_token='''<unk>''')

           tokenizer.save_pretrained(self.tmpdirname)



       def 	_lowerCamelCase    (	self							:  Tuple  ,	__SCREAMING_SNAKE_CASE							:  int):


           '''simple docstring'''

           __a		 =       '''This is a test'''
           __a		 =       '''This is a test'''
           return input_text, output_text



       def 	_lowerCamelCase    (	self							:  Dict):


           '''simple docstring'''

           __a		 =       '''<s>'''
           __a		 =       1

           self.assertEqual(self.get_tokenizer()._convert_token_to_id(__SCREAMING_SNAKE_CASE)  ,	__SCREAMING_SNAKE_CASE)
           self.assertEqual(self.get_tokenizer()._convert_id_to_token(__SCREAMING_SNAKE_CASE)  ,	__SCREAMING_SNAKE_CASE)



       def 	_lowerCamelCase    (	self							:  List[Any]):


           '''simple docstring'''

           __a		 =       list(self.get_tokenizer().get_vocab().keys())

           self.assertEqual(vocab_keys[0]  ,	'''<unk>''')
           self.assertEqual(vocab_keys[1]  ,	'''<s>''')
           self.assertEqual(vocab_keys[-1]  ,	'''j''')
           self.assertEqual(len(__SCREAMING_SNAKE_CASE)  ,	2_000)



       def 	_lowerCamelCase    (	self							:  Dict):


           '''simple docstring'''

           self.assertEqual(self.get_tokenizer().vocab_size  ,	2_000)



       def 	_lowerCamelCase    (	self							:  List[str]):


           '''simple docstring'''

           __a		 =       GPTSwaTokenizer(__SCREAMING_SNAKE_CASE)

           __a		 =       tokenizer.tokenize('''This is a test''')
           self.assertListEqual(__SCREAMING_SNAKE_CASE  ,	['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est'''])

           self.assertListEqual(tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE)  ,	[465, 287, 265, 631, 842])

           __a		 =       tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
           # fmt: off
           self.assertListEqual(
               __SCREAMING_SNAKE_CASE  ,	['''▁I''', '''▁was''', '''▁bor''', '''n''', '''▁in''', '''▁''', '''<0x39>''', '''2''', '''0''', '''0''', '''0''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁f''', '''al''', '''s''', '''<0xC3>''', '''<0xA9>''', '''.''']  ,	)
           # fmt: on

           __a		 =       tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE)
           self.assertListEqual(
               __SCREAMING_SNAKE_CASE  ,	[262, 272, 1_525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260]  ,	)

           __a		 =       tokenizer.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE)
           # fmt: off
           self.assertListEqual(
               __SCREAMING_SNAKE_CASE  ,	['''▁I''', '''▁was''', '''▁bor''', '''n''', '''▁in''', '''▁''', '''<0x39>''', '''2''', '''0''', '''0''', '''0''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁f''', '''al''', '''s''', '''<0xC3>''', '''<0xA9>''', '''.'''])
           # fmt: on



       def 	_lowerCamelCase    (	self							:  Any):


           '''simple docstring'''

           __a		 =       GPTSwaTokenizer(__SCREAMING_SNAKE_CASE)
           __a		 =       ['''This is a test''', '''I was born in 92000, and this is falsé.''']
           __a		 =       [
               [465, 287, 265, 631, 842],
               [262, 272, 1_525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260],
           ]

           # Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
           for text, expected_ids in zip(__SCREAMING_SNAKE_CASE  ,	__SCREAMING_SNAKE_CASE):
               self.assertListEqual(tokenizer.encode_fast(__SCREAMING_SNAKE_CASE)  ,	__SCREAMING_SNAKE_CASE)

           # Test that decode_fast returns the input text
           for text, token_ids in zip(__SCREAMING_SNAKE_CASE  ,	__SCREAMING_SNAKE_CASE):
               self.assertEqual(tokenizer.decode_fast(__SCREAMING_SNAKE_CASE)  ,	__SCREAMING_SNAKE_CASE)





       @slow
       def 	_lowerCamelCase    (	self							:  Any):


           '''simple docstring'''

           __a		 =       [
               '''<|python|>def fibonacci(n)\n    if n < 0:\n        print(\'Incorrect input\')''',
               '''Hey there, how are you doing this fine day?''',
               '''This is a text with a trailing spaces followed by a dot     .''',
               '''Häj sväjs lillebrör! =)''',
               '''Det är inget fel på Mr. Cool''',
           ]

           # fmt: off
           __a		 =       {'''input_ids''': [[63_423, 5, 6_811, 14_954, 282, 816, 3_821, 63_466, 63_425, 63_462, 18, 63_978, 678, 301, 1_320, 63_423, 63_455, 63_458, 18, 63_982, 4_246, 3_940, 1_901, 47_789, 5_547, 18_994], [19_630, 1_100, 63_446, 1_342, 633, 544, 4_488, 593, 5_102, 2_416, 63_495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1_652, 428, 268, 1_936, 515, 268, 58_593, 22_413, 9_106, 546, 268, 33_213, 63_979, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [55_130, 63_450, 924, 63_449, 2_249, 4_062, 1_558, 318, 63_504, 21_498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [509, 377, 2_827, 2_559, 332, 6_575, 63_443, 26_801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
           # fmt: on
           self.tokenizer_integration_test_util(
               expected_encoding=__SCREAMING_SNAKE_CASE  ,	model_name='''AI-Sweden/gpt-sw3-126m'''  ,	sequences=__SCREAMING_SNAKE_CASE  ,	)





