



'''simple docstring'''

import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class 		lowerCamelCase__      (						A	,   unittest.TestCase     ):




					'''simple docstring'''



					A_						    =						CodeGenTokenizer
					A_						    =						CodeGenTokenizerFast
					A_						    =						True
					A_						    =						{"add_prefix_space": True}
					A_						    =						False




					def 						__UpperCAmelCase     (  self				:	List[str]					)  ->       Any:



						'''simple docstring'''





						super().setUp()

						# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
						_lowercase					:       Tuple       =		[
						    'l',
						    'o',
						    'w',
						    'e',
						    'r',
						    's',
						    't',
						    'i',
						    'd',
						    'n',
						    '\u0120',
						    '\u0120l',
						    '\u0120n',
						    '\u0120lo',
						    '\u0120low',
						    'er',
						    '\u0120lowest',
						    '\u0120newer',
						    '\u0120wider',
						    '<unk>',
						    '<|endoftext|>',
						]
						_lowercase					:       Optional[int]       =		dict(zip(UpperCamelCase__     ,  range(len(UpperCamelCase__					)					)					)					)
						_lowercase					:       List[str]       =		['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
						_lowercase					:       Optional[Any]       =		{'unk_token': '<unk>'}

						_lowercase					:       Dict       =		os.path.join(self.tmpdirname     ,  VOCAB_FILES_NAMES['vocab_file']					)
						_lowercase					:       Tuple       =		os.path.join(self.tmpdirname     ,  VOCAB_FILES_NAMES['merges_file']					)
						with open(self.vocab_file     ,  'w'     ,  encoding='utf-8'					) as fp:
							fp.write(json.dumps(UpperCamelCase__					) + '\n'					)
						with open(self.merges_file     ,  'w'     ,  encoding='utf-8'					) as fp:
							fp.write('\n'.join(UpperCamelCase__					)					)




					def 						__UpperCAmelCase     (  self				:	List[Any]     ,  **UpperCamelCase_				:	Optional[Any]					)  ->       Union[str, Any]:



						'''simple docstring'''





						kwargs.update(self.special_tokens_map					)
						return CodeGenTokenizer.from_pretrained(self.tmpdirname     ,  **UpperCamelCase__					)




					def 						__UpperCAmelCase     (  self				:	str     ,  **UpperCamelCase_				:	str					)  ->       List[Any]:



						'''simple docstring'''





						kwargs.update(self.special_tokens_map					)
						return CodeGenTokenizerFast.from_pretrained(self.tmpdirname     ,  **UpperCamelCase__					)




					def 						__UpperCAmelCase     (  self				:	Any     ,  UpperCamelCase_				:	Tuple					)  ->       Union[str, Any]:



						'''simple docstring'''





						_lowercase					:       Dict       =		'lower newer'
						_lowercase					:       Dict       =		'lower newer'
						return input_text, output_text




					def 						__UpperCAmelCase     (  self				:	int					)  ->       Union[str, Any]:



						'''simple docstring'''





						_lowercase					:       int       =		CodeGenTokenizer(self.vocab_file     ,  self.merges_file     ,  **self.special_tokens_map					)
						_lowercase					:       Optional[int]       =		'lower newer'
						_lowercase					:       Dict       =		['\u0120low', 'er', '\u0120', 'n', 'e', 'w', 'er']
						_lowercase					:       List[Any]       =		tokenizer.tokenize(UpperCamelCase__     ,  add_prefix_space=UpperCamelCase__					)
						self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__					)

						_lowercase					:       Optional[Any]       =		tokens + [tokenizer.unk_token]
						_lowercase					:       List[Any]       =		[14, 15, 10, 9, 3, 2, 15, 19]
						self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__					)     ,  UpperCamelCase__					)




					def 						__UpperCAmelCase     (  self				:	List[str]					)  ->       Dict:



						'''simple docstring'''





						if not self.test_rust_tokenizer:
							return

						_lowercase					:       Any       =		self.get_tokenizer()
						_lowercase					:       List[Any]       =		self.get_rust_tokenizer(add_prefix_space=UpperCamelCase__					)

						_lowercase					:       Any       =		'lower newer'

						# Testing tokenization
						_lowercase					:       Optional[int]       =		tokenizer.tokenize(UpperCamelCase__     ,  add_prefix_space=UpperCamelCase__					)
						_lowercase					:       Optional[int]       =		rust_tokenizer.tokenize(UpperCamelCase__					)
						self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__					)

						# Testing conversion to ids without special tokens
						_lowercase					:       List[Any]       =		tokenizer.encode(UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__     ,  add_prefix_space=UpperCamelCase__					)
						_lowercase					:       Union[str, Any]       =		rust_tokenizer.encode(UpperCamelCase__     ,  add_special_tokens=UpperCamelCase__					)
						self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__					)

						# Testing conversion to ids with special tokens
						_lowercase					:       Any       =		self.get_rust_tokenizer(add_prefix_space=UpperCamelCase__					)
						_lowercase					:       int       =		tokenizer.encode(UpperCamelCase__     ,  add_prefix_space=UpperCamelCase__					)
						_lowercase					:       Union[str, Any]       =		rust_tokenizer.encode(UpperCamelCase__					)
						self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__					)

						# Testing the unknown token
						_lowercase					:       Union[str, Any]       =		tokens + [rust_tokenizer.unk_token]
						_lowercase					:       List[Any]       =		[14, 15, 10, 9, 3, 2, 15, 19]
						self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(UpperCamelCase__					)     ,  UpperCamelCase__					)




					def 						__UpperCAmelCase     (  self				:	int     ,  *UpperCamelCase_				:	Dict     ,  **UpperCamelCase_				:	Optional[Any]					)  ->       Dict:



						'''simple docstring'''





						pass




					def 						__UpperCAmelCase     (  self				:	List[Any]     ,  UpperCamelCase_				:	Dict=15					)  ->       Optional[Any]:



						'''simple docstring'''





						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
								_lowercase					:       Any       =		self.rust_tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__					)

								# Simple input
								_lowercase					:       Any       =		'This is a simple input'
								_lowercase					:       Union[str, Any]       =		['This is a simple input 1', 'This is a simple input 2']
								_lowercase					:       Dict       =		('This is a simple input', 'This is a pair')
								_lowercase					:       Union[str, Any]       =		[
								    ('This is a simple input 1', 'This is a simple input 2'),
								    ('This is a simple pair 1', 'This is a simple pair 2'),
								]

								# Simple input tests
								self.assertRaises(UpperCamelCase__     ,  tokenizer_r.encode     ,  UpperCamelCase__     ,  max_length=UpperCamelCase__     ,  padding='max_length'					)

								# Simple input
								self.assertRaises(UpperCamelCase__     ,  tokenizer_r.encode_plus     ,  UpperCamelCase__     ,  max_length=UpperCamelCase__     ,  padding='max_length'					)

								# Simple input
								self.assertRaises(
								    UpperCamelCase__     ,  tokenizer_r.batch_encode_plus     ,  UpperCamelCase__     ,  max_length=UpperCamelCase__     ,  padding='max_length'     ,  )

								# Pair input
								self.assertRaises(UpperCamelCase__     ,  tokenizer_r.encode     ,  UpperCamelCase__     ,  max_length=UpperCamelCase__     ,  padding='max_length'					)

								# Pair input
								self.assertRaises(UpperCamelCase__     ,  tokenizer_r.encode_plus     ,  UpperCamelCase__     ,  max_length=UpperCamelCase__     ,  padding='max_length'					)

								# Pair input
								self.assertRaises(
								    UpperCamelCase__     ,  tokenizer_r.batch_encode_plus     ,  UpperCamelCase__     ,  max_length=UpperCamelCase__     ,  padding='max_length'     ,  )




					def 						__UpperCAmelCase     (  self				:	Optional[Any]					)  ->       int:



						'''simple docstring'''





						_lowercase					:       Optional[int]       =		CodeGenTokenizer.from_pretrained(self.tmpdirname     ,  pad_token='<pad>'					)

						# Simple input
						_lowercase					:       Tuple       =		'This is a simple input'
						_lowercase					:       List[Any]       =		['This is a simple input looooooooong', 'This is a simple input']
						_lowercase					:       Optional[Any]       =		('This is a simple input', 'This is a pair')
						_lowercase					:       str       =		[
						    ('This is a simple input loooooong', 'This is a simple input'),
						    ('This is a simple pair loooooong', 'This is a simple pair'),
						]

						_lowercase					:       Optional[int]       =		tokenizer.pad_token_id

						_lowercase					:       Union[str, Any]       =		tokenizer(UpperCamelCase__     ,  padding='max_length'     ,  max_length=30     ,  return_tensors='np'					)
						_lowercase					:       Tuple       =		tokenizer(UpperCamelCase__     ,  padding=UpperCamelCase__     ,  truncate=UpperCamelCase__     ,  return_tensors='np'					)
						_lowercase					:       Optional[Any]       =		tokenizer(*UpperCamelCase__     ,  padding='max_length'     ,  max_length=60     ,  return_tensors='np'					)
						_lowercase					:       Union[str, Any]       =		tokenizer(UpperCamelCase__     ,  padding=UpperCamelCase__     ,  truncate=UpperCamelCase__     ,  return_tensors='np'					)

						# s
						# test single string max_length padding
						self.assertEqual(out_s['input_ids'].shape[-1]     ,  30					)
						self.assertTrue(pad_token_id in out_s['input_ids']					)
						self.assertTrue(0 in out_s['attention_mask']					)

						# s2
						# test automatic padding
						self.assertEqual(out_sa['input_ids'].shape[-1]     ,  33					)
						# long slice doesn't have padding
						self.assertFalse(pad_token_id in out_sa['input_ids'][0]					)
						self.assertFalse(0 in out_sa['attention_mask'][0]					)
						# short slice does have padding
						self.assertTrue(pad_token_id in out_sa['input_ids'][1]					)
						self.assertTrue(0 in out_sa['attention_mask'][1]					)

						# p
						# test single pair max_length padding
						self.assertEqual(out_p['input_ids'].shape[-1]     ,  60					)
						self.assertTrue(pad_token_id in out_p['input_ids']					)
						self.assertTrue(0 in out_p['attention_mask']					)

						# p2
						# test automatic padding pair
						self.assertEqual(out_pa['input_ids'].shape[-1]     ,  52					)
						# long slice pair doesn't have padding
						self.assertFalse(pad_token_id in out_pa['input_ids'][0]					)
						self.assertFalse(0 in out_pa['attention_mask'][0]					)
						# short slice pair does have padding
						self.assertTrue(pad_token_id in out_pa['input_ids'][1]					)
						self.assertTrue(0 in out_pa['attention_mask'][1]					)




					def 						__UpperCAmelCase     (  self				:	str					)  ->       Tuple:



						'''simple docstring'''





						_lowercase					:       Tuple       =		'$$$'
						_lowercase					:       str       =		CodeGenTokenizer.from_pretrained(self.tmpdirname     ,  bos_token=UpperCamelCase__     ,  add_bos_token=UpperCamelCase__					)

						_lowercase					:       List[str]       =		'This is a simple input'
						_lowercase					:       int       =		['This is a simple input 1', 'This is a simple input 2']

						_lowercase					:       List[Any]       =		tokenizer.bos_token_id

						_lowercase					:       str       =		tokenizer(UpperCamelCase__					)
						_lowercase					:       Union[str, Any]       =		tokenizer(UpperCamelCase__					)

						self.assertEqual(out_s.input_ids[0]     ,  UpperCamelCase__					)
						self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids					)					)

						_lowercase					:       Optional[int]       =		tokenizer.decode(out_s.input_ids					)
						_lowercase					:       List[str]       =		tokenizer.batch_decode(out_sa.input_ids					)

						self.assertEqual(decode_s.split()[0]     ,  UpperCamelCase__					)
						self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa					)					)




					@slow
					def 						__UpperCAmelCase     (  self				:	str					)  ->       Any:



						'''simple docstring'''





						_lowercase					:       Optional[int]       =		CodeGenTokenizer.from_pretrained('Salesforce/codegen-350M-mono'					)

						_lowercase					:       Any       =		'\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'
						_lowercase					:       List[Any]       =		'\nif len_a > len_b:      result = a\nelse:      result = b'

						_lowercase					:       Union[str, Any]       =		tokenizer.encode(UpperCamelCase__					)
						_lowercase					:       List[Any]       =		['^#', re.escape('<|endoftext|>'					), '^\'\'\'', '^"""', '\n\n\n']
						_lowercase					:       Dict       =		tokenizer.decode(UpperCamelCase__     ,  truncate_before_pattern=UpperCamelCase__					)
						self.assertEqual(UpperCamelCase__     ,  UpperCamelCase__					)




					def 						__UpperCAmelCase     (  self				:	Any					)  ->       Any:



						'''simple docstring'''





						pass





'''simple docstring'''

import os
from collections.abc import Iterator







def 						__UpperCamelCase				(							_lowercase = "."					) ->			Iterator[str]:
	for dir_path, dir_names, filenames in os.walk(_lowercase					):
		_lowercase					:       Optional[int]       =		[d for d in dir_names if d != 'scripts' and d[0] not in '._']
		for filename in filenames:
			if filename == "__init__.py":
				continue
			if os.path.splitext(_lowercase					)[1] in (".py", ".ipynb"):
				yield os.path.join(_lowercase,				_lowercase					).lstrip('./'					)







def 						__UpperCamelCase				(							_lowercase					) ->			List[str]:
	return f'''{i * "  "}*''' if i else "\n##"







def 						__UpperCamelCase				(							_lowercase,				_lowercase					) ->			str:
	_lowercase					:       Optional[Any]       =		old_path.split(os.sep					)
	for i, new_part in enumerate(new_path.split(os.sep					)					):
		if (i + 1 > len(_lowercase					) or old_parts[i] != new_part) and new_part:
			print(f'''{md_prefix(_lowercase					)} {new_part.replace("_",				" "					).title()}'''					)
	return new_path







def 						__UpperCamelCase				(							_lowercase = "."					) ->			None:
	_lowercase					:       Dict       =		''
	for filepath in sorted(good_file_paths(_lowercase					)					):
		_lowercase  ,    _lowercase					:       Optional[Any]       =		os.path.split(_lowercase					)
		if filepath != old_path:
			_lowercase					:       Dict       =		print_path(_lowercase,				_lowercase					)
		_lowercase					:       Optional[int]       =		(filepath.count(os.sep					) + 1) if filepath else 0
		_lowercase					:       Dict       =		f'''{filepath}/{filename}'''.replace(' ',				'%20'					)
		_lowercase					:       Optional[int]       =		os.path.splitext(filename.replace('_',				' '					).title()					)[0]
		print(f'''{md_prefix(_lowercase					)} [{filename}]({url})'''					)


if __name__ == "__main__":
		print_directory_md('''.''')
