



import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class   UpperCamelCase_       (     UpperCamelCase__				,  unittest.TestCase    ):
							lowerCamelCase_           =					GPTaTokenizer
							lowerCamelCase_           =					GPTaTokenizerFast
							lowerCamelCase_           =					True
							lowerCamelCase_           =					{"add_prefix_space": True}
							lowerCamelCase_           =					False


							def      _snake_case  (							self    :Tuple		)   ->   int:



									"""simple docstring"""



									super().setUp()

									# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
									SCREAMING_SNAKE_CASE__			=      [
									    'l',
									    'o',
									    'w',
									    'e',
									    'r',
									    's',
									    't',
									    'i',
									    'd',
									    'n',
									    '\u0120',
									    '\u0120l',
									    '\u0120n',
									    '\u0120lo',
									    '\u0120low',
									    'er',
									    '\u0120lowest',
									    '\u0120newer',
									    '\u0120wider',
									    '<unk>',
									    '<|endoftext|>',
									]
									SCREAMING_SNAKE_CASE__			=      dict(zip(__A							,				range(len(__A		)		)		)		)
									SCREAMING_SNAKE_CASE__			=      ['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
									SCREAMING_SNAKE_CASE__			=      {'unk_token': '<unk>'}

									SCREAMING_SNAKE_CASE__			=      os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""vocab_file"""]		)
									SCREAMING_SNAKE_CASE__			=      os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""merges_file"""]		)
									with open(self.vocab_file							,				"""w"""							,				encoding="""utf-8"""		) as fp:
											fp.write(json.dumps(__A		) + """\n"""		)
									with open(self.merges_file							,				"""w"""							,				encoding="""utf-8"""		) as fp:
											fp.write("""\n""".join(__A		)		)


							def      _snake_case  (							self    :Any							,				**__A    :int		)   ->   Any:



									"""simple docstring"""



									kwargs.update(self.special_tokens_map		)
									return GPTaTokenizer.from_pretrained(self.tmpdirname							,				**__A		)


							def      _snake_case  (							self    :int							,				**__A    :str		)   ->   Tuple:



									"""simple docstring"""



									kwargs.update(self.special_tokens_map		)
									return GPTaTokenizerFast.from_pretrained(self.tmpdirname							,				**__A		)


							def      _snake_case  (							self    :str							,				__A    :Any		)   ->   Optional[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      'lower newer'
									SCREAMING_SNAKE_CASE__			=      'lower newer'
									return input_text, output_text


							def      _snake_case  (							self    :str		)   ->   Optional[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      GPTaTokenizer(self.vocab_file							,				self.merges_file							,				**self.special_tokens_map		)
									SCREAMING_SNAKE_CASE__			=      'lower newer'
									SCREAMING_SNAKE_CASE__			=      ['\u0120low', 'er', '\u0120', 'n', 'e', 'w', 'er']
									SCREAMING_SNAKE_CASE__			=      tokenizer.tokenize(__A							,				add_prefix_space=__A		)
									self.assertListEqual(__A							,				__A		)

									SCREAMING_SNAKE_CASE__			=      tokens + [tokenizer.unk_token]
									SCREAMING_SNAKE_CASE__			=      [14, 15, 10, 9, 3, 2, 15, 19]
									self.assertListEqual(tokenizer.convert_tokens_to_ids(__A		)							,				__A		)


							def      _snake_case  (							self    :Any		)   ->   List[str]:



									"""simple docstring"""



									if not self.test_rust_tokenizer:
											return

									SCREAMING_SNAKE_CASE__			=      self.get_tokenizer()
									SCREAMING_SNAKE_CASE__			=      self.get_rust_tokenizer(add_prefix_space=__A		)

									SCREAMING_SNAKE_CASE__			=      'lower newer'

									# Testing tokenization
									SCREAMING_SNAKE_CASE__			=      tokenizer.tokenize(__A							,				add_prefix_space=__A		)
									SCREAMING_SNAKE_CASE__			=      rust_tokenizer.tokenize(__A		)
									self.assertListEqual(__A							,				__A		)

									# Testing conversion to ids without special tokens
									SCREAMING_SNAKE_CASE__			=      tokenizer.encode(__A							,				add_special_tokens=__A							,				add_prefix_space=__A		)
									SCREAMING_SNAKE_CASE__			=      rust_tokenizer.encode(__A							,				add_special_tokens=__A		)
									self.assertListEqual(__A							,				__A		)

									# Testing conversion to ids with special tokens
									SCREAMING_SNAKE_CASE__			=      self.get_rust_tokenizer(add_prefix_space=__A		)
									SCREAMING_SNAKE_CASE__			=      tokenizer.encode(__A							,				add_prefix_space=__A		)
									SCREAMING_SNAKE_CASE__			=      rust_tokenizer.encode(__A		)
									self.assertListEqual(__A							,				__A		)

									# Testing the unknown token
									SCREAMING_SNAKE_CASE__			=      tokens + [rust_tokenizer.unk_token]
									SCREAMING_SNAKE_CASE__			=      [14, 15, 10, 9, 3, 2, 15, 19]
									self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(__A		)							,				__A		)


							def      _snake_case  (							self    :Any							,				*__A    :Optional[int]							,				**__A    :Tuple		)   ->   Tuple:



									"""simple docstring"""



									pass


							def      _snake_case  (							self    :Tuple							,				__A    :Optional[int]=15		)   ->   Optional[Any]:



									"""simple docstring"""



									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
													SCREAMING_SNAKE_CASE__			=      self.rust_tokenizer_class.from_pretrained(__A							,				**__A		)

													# Simple input
													SCREAMING_SNAKE_CASE__			=      'This is a simple input'
													SCREAMING_SNAKE_CASE__			=      ['This is a simple input 1', 'This is a simple input 2']
													SCREAMING_SNAKE_CASE__			=      ('This is a simple input', 'This is a pair')
													SCREAMING_SNAKE_CASE__			=      [
													    ('This is a simple input 1', 'This is a simple input 2'),
													    ('This is a simple pair 1', 'This is a simple pair 2'),
													]

													# Simple input tests
													self.assertRaises(__A							,				tokenizer_r.encode							,				__A							,				max_length=__A							,				padding="""max_length"""		)

													# Simple input
													self.assertRaises(__A							,				tokenizer_r.encode_plus							,				__A							,				max_length=__A							,				padding="""max_length"""		)

													# Simple input
													self.assertRaises(
													    __A							,				tokenizer_r.batch_encode_plus							,				__A							,				max_length=__A							,				padding="""max_length"""							,				)

													# Pair input
													self.assertRaises(__A							,				tokenizer_r.encode							,				__A							,				max_length=__A							,				padding="""max_length"""		)

													# Pair input
													self.assertRaises(__A							,				tokenizer_r.encode_plus							,				__A							,				max_length=__A							,				padding="""max_length"""		)

													# Pair input
													self.assertRaises(
													    __A							,				tokenizer_r.batch_encode_plus							,				__A							,				max_length=__A							,				padding="""max_length"""							,				)


							def      _snake_case  (							self    :Any		)   ->   str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      GPTaTokenizer.from_pretrained(self.tmpdirname							,				pad_token="""<pad>"""		)

									# Simple input
									SCREAMING_SNAKE_CASE__			=      'This is a simple input'
									SCREAMING_SNAKE_CASE__			=      ['This is a simple input looooooooong', 'This is a simple input']
									SCREAMING_SNAKE_CASE__			=      ('This is a simple input', 'This is a pair')
									SCREAMING_SNAKE_CASE__			=      [
									    ('This is a simple input loooooong', 'This is a simple input'),
									    ('This is a simple pair loooooong', 'This is a simple pair'),
									]

									SCREAMING_SNAKE_CASE__			=      tokenizer.pad_token_id

									SCREAMING_SNAKE_CASE__			=      tokenizer(__A							,				padding="""max_length"""							,				max_length=30							,				return_tensors="""np"""		)
									SCREAMING_SNAKE_CASE__			=      tokenizer(__A							,				padding=__A							,				truncate=__A							,				return_tensors="""np"""		)
									SCREAMING_SNAKE_CASE__			=      tokenizer(*__A							,				padding="""max_length"""							,				max_length=60							,				return_tensors="""np"""		)
									SCREAMING_SNAKE_CASE__			=      tokenizer(__A							,				padding=__A							,				truncate=__A							,				return_tensors="""np"""		)

									# s
									# test single string max_length padding
									self.assertEqual(out_s["""input_ids"""].shape[-1]							,				30		)
									self.assertTrue(pad_token_id in out_s["""input_ids"""]		)
									self.assertTrue(0 in out_s["""attention_mask"""]		)

									# s2
									# test automatic padding
									self.assertEqual(out_sa["""input_ids"""].shape[-1]							,				33		)
									# long slice doesn't have padding
									self.assertFalse(pad_token_id in out_sa["""input_ids"""][0]		)
									self.assertFalse(0 in out_sa["""attention_mask"""][0]		)
									# short slice does have padding
									self.assertTrue(pad_token_id in out_sa["""input_ids"""][1]		)
									self.assertTrue(0 in out_sa["""attention_mask"""][1]		)

									# p
									# test single pair max_length padding
									self.assertEqual(out_p["""input_ids"""].shape[-1]							,				60		)
									self.assertTrue(pad_token_id in out_p["""input_ids"""]		)
									self.assertTrue(0 in out_p["""attention_mask"""]		)

									# p2
									# test automatic padding pair
									self.assertEqual(out_pa["""input_ids"""].shape[-1]							,				52		)
									# long slice pair doesn't have padding
									self.assertFalse(pad_token_id in out_pa["""input_ids"""][0]		)
									self.assertFalse(0 in out_pa["""attention_mask"""][0]		)
									# short slice pair does have padding
									self.assertTrue(pad_token_id in out_pa["""input_ids"""][1]		)
									self.assertTrue(0 in out_pa["""attention_mask"""][1]		)


							def      _snake_case  (							self    :Optional[int]		)   ->   Tuple:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      '$$$'
									SCREAMING_SNAKE_CASE__			=      GPTaTokenizer.from_pretrained(self.tmpdirname							,				bos_token=__A							,				add_bos_token=__A		)

									SCREAMING_SNAKE_CASE__			=      'This is a simple input'
									SCREAMING_SNAKE_CASE__			=      ['This is a simple input 1', 'This is a simple input 2']

									SCREAMING_SNAKE_CASE__			=      tokenizer.bos_token_id

									SCREAMING_SNAKE_CASE__			=      tokenizer(__A		)
									SCREAMING_SNAKE_CASE__			=      tokenizer(__A		)

									self.assertEqual(out_s.input_ids[0]							,				__A		)
									self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids		)		)

									SCREAMING_SNAKE_CASE__			=      tokenizer.decode(out_s.input_ids		)
									SCREAMING_SNAKE_CASE__			=      tokenizer.batch_decode(out_sa.input_ids		)

									self.assertEqual(decode_s.split()[0]							,				__A		)
									self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa		)		)


							def      _snake_case  (							self    :Union[str, Any]		)   ->   Optional[Any]:



									"""simple docstring"""



									pass


							def      _snake_case  (							self    :Tuple		)   ->   str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      [self.get_tokenizer(do_lower_case=__A							,				add_bos_token=__A		)]
									for tokenizer in tokenizers:
											with self.subTest(f'''{tokenizer.__class__.__name__}'''		):
													SCREAMING_SNAKE_CASE__			=      'Encode this.'
													SCREAMING_SNAKE_CASE__			=      'This one too please.'
													SCREAMING_SNAKE_CASE__			=      tokenizer.encode(__A							,				add_special_tokens=__A		)
													encoded_sequence += tokenizer.encode(__A							,				add_special_tokens=__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer.encode_plus(
													    __A							,				__A							,				add_special_tokens=__A							,				return_special_tokens_mask=__A							,				)
													SCREAMING_SNAKE_CASE__			=      encoded_sequence_dict['input_ids']
													SCREAMING_SNAKE_CASE__			=      encoded_sequence_dict['special_tokens_mask']
													self.assertEqual(len(__A		)							,				len(__A		)		)

													SCREAMING_SNAKE_CASE__			=      [
													    (x if not special_tokens_mask[i] else None) for i, x in enumerate(__A		)
													]
													SCREAMING_SNAKE_CASE__			=      [x for x in filtered_sequence if x is not None]
													self.assertEqual(__A							,				__A		)




@require_tokenizers
class   UpperCamelCase_       (     unittest.TestCase    ):


							def      _snake_case  (							self    :Optional[Any]		)   ->   str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      AutoTokenizer.from_pretrained("""facebook/opt-350m"""							,				from_slow=__A		)
									SCREAMING_SNAKE_CASE__			=      'A photo of a cat'

									SCREAMING_SNAKE_CASE__			=      tokenizer.encode(
									    __A							,				)
									self.assertEqual(__A							,				[2, 250, 1345, 9, 10, 4758]		)
									tokenizer.save_pretrained("""test_opt"""		)

									SCREAMING_SNAKE_CASE__			=      AutoTokenizer.from_pretrained("""./test_opt"""		)
									SCREAMING_SNAKE_CASE__			=      tokenizer.encode(
									    __A							,				)
									self.assertEqual(__A							,				[2, 250, 1345, 9, 10, 4758]		)


							def      _snake_case  (							self    :Union[str, Any]		)   ->   str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      AutoTokenizer.from_pretrained("""facebook/opt-350m"""							,				use_slow=__A		)
									SCREAMING_SNAKE_CASE__			=      'A photo of a cat'

									SCREAMING_SNAKE_CASE__			=      tokenizer.encode(
									    __A							,				)
									# Same as above
									self.assertEqual(__A							,				[2, 250, 1345, 9, 10, 4758]		)


							@unittest.skip("""This test is failing because of a bug in the fast tokenizer"""		)
							def      _snake_case  (							self    :Any		)   ->   str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      AutoTokenizer.from_pretrained("""facebook/opt-350m"""							,				from_slow=__A		)

									SCREAMING_SNAKE_CASE__			=      'bos'
									SCREAMING_SNAKE_CASE__			=      tokenizer.get_vocab()['bos']

									SCREAMING_SNAKE_CASE__			=      'A photo of a cat'
									SCREAMING_SNAKE_CASE__			=      tokenizer.encode(
									    __A							,				)
									# We changed the bos token
									self.assertEqual(__A							,				[3_1957, 250, 1345, 9, 10, 4758]		)
									tokenizer.save_pretrained("""./tok"""		)
									SCREAMING_SNAKE_CASE__			=      AutoTokenizer.from_pretrained("""./tok"""		)
									self.assertTrue(tokenizer.is_fast		)
									SCREAMING_SNAKE_CASE__			=      tokenizer.encode(
									    __A							,				)
									self.assertEqual(__A							,				[3_1957, 250, 1345, 9, 10, 4758]		)






from argparse import ArgumentParser

from datasets.commands.convert import ConvertCommand
from datasets.commands.dummy_data import DummyDataCommand
from datasets.commands.env import EnvironmentCommand
from datasets.commands.run_beam import RunBeamCommand
from datasets.commands.test import TestCommand
from datasets.utils.logging import set_verbosity_info






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							Any    ):
		return {key.lstrip("""-"""    ): value for key, value in zip(unknown_args[::2]					,   unknown_args[1::2]    )}






def 				SCREAMING_SNAKE_CASE__							(       ):
		SCREAMING_SNAKE_CASE__			=      ArgumentParser(
		    """HuggingFace Datasets CLI tool"""					,   usage="""datasets-cli <command> [<args>]"""					,   allow_abbrev=UpperCamelCase__    )
		SCREAMING_SNAKE_CASE__			=      parser.add_subparsers(help="""datasets-cli command helpers"""    )
		set_verbosity_info()

		# Register commands
		ConvertCommand.register_subcommand(UpperCamelCase__    )
		EnvironmentCommand.register_subcommand(UpperCamelCase__    )
		TestCommand.register_subcommand(UpperCamelCase__    )
		RunBeamCommand.register_subcommand(UpperCamelCase__    )
		DummyDataCommand.register_subcommand(UpperCamelCase__    )

		# Parse args
		SCREAMING_SNAKE_CASE__	,						SCREAMING_SNAKE_CASE__			=      parser.parse_known_args()
		if not hasattr(UpperCamelCase__					,   """func"""    ):
				parser.print_help()
				exit(1    )
		SCREAMING_SNAKE_CASE__			=      parse_unknown_args(UpperCamelCase__    )

		# Run
		SCREAMING_SNAKE_CASE__			=      args.func(UpperCamelCase__					,   **UpperCamelCase__    )
		service.run()


if __name__ == "__main__":
	main()