






"""simple docstring"""



import numpy as np

from transformers import BatchFeature
from transformers.testing_utils import require_tf, require_torch

from .test_feature_extraction_common import FeatureExtractionSavingTestMixin






class 							__SCREAMING_SNAKE_CASE						(				__SCREAMING_SNAKE_CASE		):
       # to overwrite at feature extractactor specific tests
       snake_case_     =							None
       snake_case_     =							None


       @property
       def 						_UpperCamelCase       (			self      :						Any							):
             '''simple docstring'''





             return self.feat_extract_tester.prepare_feat_extract_dict()


       def 						_UpperCamelCase       (			self      :						Tuple							):
             '''simple docstring'''





             A__						:  Dict		  =  self.feature_extraction_class(**self.feat_extract_dict							)
             self.assertTrue(hasattr(__snake_case			,    """feature_size"""							)							)
             self.assertTrue(hasattr(__snake_case			,    """sampling_rate"""							)							)
             self.assertTrue(hasattr(__snake_case			,    """padding_value"""							)							)


       def 						_UpperCamelCase       (			self      :						Dict							):
             '''simple docstring'''





             A__						:  int		  =  self.feat_extract_tester.prepare_inputs_for_common()
             A__						:  Any		  =  self.feature_extraction_class(**self.feat_extract_dict							)
             A__						:  str		  =  feat_extract.model_input_names[0]

             A__						:  Optional[Any]		  =  BatchFeature({input_name: speech_inputs}							)

             self.assertTrue(all(len(__snake_case							) == len(__snake_case							) for x, y in zip(__snake_case			,    processed_features[input_name]							)							)							)

             A__						:  Dict		  =  self.feat_extract_tester.prepare_inputs_for_common(equal_length=__snake_case							)
             A__						:  List[str]		  =  BatchFeature({input_name: speech_inputs}			,    tensor_type="""np"""							)

             A__						:  Dict		  =  processed_features[input_name]

             if len(batch_features_input.shape							) < 3:
                   A__						:  int		  =  batch_features_input[:, :, None]

             self.assertTrue(
                 batch_features_input.shape
                 == (self.feat_extract_tester.batch_size, len(speech_inputs[0]							), self.feat_extract_tester.feature_size)							)


       @require_torch
       def 						_UpperCamelCase       (			self      :						List[str]							):
             '''simple docstring'''





             A__						:  Dict		  =  self.feat_extract_tester.prepare_inputs_for_common(equal_length=__snake_case							)
             A__						:  List[str]		  =  self.feature_extraction_class(**self.feat_extract_dict							)
             A__						:  List[str]		  =  feat_extract.model_input_names[0]

             A__						:  Union[str, Any]		  =  BatchFeature({input_name: speech_inputs}			,    tensor_type="""pt"""							)

             A__						:  Dict		  =  processed_features[input_name]

             if len(batch_features_input.shape							) < 3:
                   A__						:  List[str]		  =  batch_features_input[:, :, None]

             self.assertTrue(
                 batch_features_input.shape
                 == (self.feat_extract_tester.batch_size, len(speech_inputs[0]							), self.feat_extract_tester.feature_size)							)


       @require_tf
       def 						_UpperCamelCase       (			self      :						Optional[Any]							):
             '''simple docstring'''





             A__						:  int		  =  self.feat_extract_tester.prepare_inputs_for_common(equal_length=__snake_case							)
             A__						:  List[str]		  =  self.feature_extraction_class(**self.feat_extract_dict							)
             A__						:  Tuple		  =  feat_extract.model_input_names[0]

             A__						:  str		  =  BatchFeature({input_name: speech_inputs}			,    tensor_type="""tf"""							)

             A__						:  List[Any]		  =  processed_features[input_name]

             if len(batch_features_input.shape							) < 3:
                   A__						:  Tuple		  =  batch_features_input[:, :, None]

             self.assertTrue(
                 batch_features_input.shape
                 == (self.feat_extract_tester.batch_size, len(speech_inputs[0]							), self.feat_extract_tester.feature_size)							)


       def 						_UpperCamelCase       (			self      :						Dict			,    snake_case      :						Tuple=False							):
             '''simple docstring'''





             def _inputs_have_equal_length(snake_case      :						int							):
                   A__						:  Any		  =  len(input[0]							)
                   for input_slice in input[1:]:
                         if len(__snake_case							) != length:
                               return False
                   return True

             def _inputs_are_equal(snake_case      :						Tuple			,    snake_case      :						Tuple							):
                   if len(__snake_case							) != len(__snake_case							):
                         return False

                   for input_slice_a, input_slice_a in zip(__snake_case			,    __snake_case							):
                         if not np.allclose(np.asarray(__snake_case							)			,    np.asarray(__snake_case							)			,    atol=1e-3							):
                               return False
                   return True

             A__						:  List[Any]		  =  self.feature_extraction_class(**self.feat_extract_dict							)
             A__						:  Tuple		  =  self.feat_extract_tester.prepare_inputs_for_common(numpify=__snake_case							)
             A__						:  Any		  =  feat_extract.model_input_names[0]

             A__						:  Tuple		  =  BatchFeature({input_name: speech_inputs}							)

             A__						:  List[Any]		  =  self.feat_extract_tester.seq_length_diff
             A__						:  List[str]		  =  self.feat_extract_tester.max_seq_length + pad_diff
             A__						:  List[str]		  =  self.feat_extract_tester.min_seq_length
             A__						:  Any		  =  self.feat_extract_tester.batch_size
             A__						:  List[Any]		  =  self.feat_extract_tester.feature_size

             # test padding for List[int] + numpy
             A__						:  Optional[int]		  =  feat_extract.pad(__snake_case			,    padding=__snake_case							)
             A__						:  List[str]		  =  input_a[input_name]

             A__						:  Union[str, Any]		  =  feat_extract.pad(__snake_case			,    padding="""longest"""							)
             A__						:  Dict		  =  input_a[input_name]

             A__						:  List[str]		  =  feat_extract.pad(__snake_case			,    padding="""max_length"""			,    max_length=len(speech_inputs[-1]							)							)
             A__						:  List[Any]		  =  input_a[input_name]

             A__						:  Tuple		  =  feat_extract.pad(__snake_case			,    padding="""longest"""			,    return_tensors="""np"""							)
             A__						:  Optional[Any]		  =  input_a[input_name]

             # max_length parameter has to be provided when setting `padding="max_length"`
             with self.assertRaises(__snake_case							):
                   feat_extract.pad(__snake_case			,    padding="""max_length"""							)[input_name]

             A__						:  Any		  =  feat_extract.pad(
                 __snake_case			,    padding="""max_length"""			,    max_length=__snake_case			,    return_tensors="""np"""							)
             A__						:  List[str]		  =  input_a[input_name]

             self.assertFalse(_inputs_have_equal_length(__snake_case							)							)
             self.assertTrue(_inputs_have_equal_length(__snake_case							)							)
             self.assertTrue(_inputs_have_equal_length(__snake_case							)							)
             self.assertTrue(_inputs_are_equal(__snake_case			,    __snake_case							)							)
             self.assertTrue(len(input_a[0]							) == pad_min_length							)
             self.assertTrue(len(input_a[1]							) == pad_min_length + pad_diff							)
             self.assertTrue(input_a.shape[:2] == (batch_size, len(input_a[0]							))							)
             self.assertTrue(input_a.shape[:2] == (batch_size, pad_max_length)							)

             if feature_size > 1:
                   self.assertTrue(input_a.shape[2] == input_a.shape[2] == feature_size							)

             # test padding for `pad_to_multiple_of` for List[int] + numpy
             A__						:  Tuple		  =  feat_extract.pad(__snake_case			,    pad_to_multiple_of=10							)
             A__						:  Union[str, Any]		  =  input_a[input_name]

             A__						:  Dict		  =  feat_extract.pad(__snake_case			,    padding="""longest"""			,    pad_to_multiple_of=10							)
             A__						:  Optional[int]		  =  input_a[input_name]

             A__						:  Tuple		  =  feat_extract.pad(
                 __snake_case			,    padding="""max_length"""			,    pad_to_multiple_of=10			,    max_length=__snake_case							)
             A__						:  Optional[Any]		  =  input_a[input_name]

             A__						:  Tuple		  =  feat_extract.pad(
                 __snake_case			,    padding="""max_length"""			,    pad_to_multiple_of=10			,    max_length=__snake_case			,    return_tensors="""np"""			,    )
             A__						:  str		  =  input_a[input_name]

             self.assertTrue(all(len(__snake_case							) % 10 == 0 for x in input_a							)							)
             self.assertTrue(_inputs_are_equal(__snake_case			,    __snake_case							)							)

             A__						:  Optional[Any]		  =  pad_max_length if pad_max_length % 10 == 0 else (pad_max_length // 10 + 1) * 10
             self.assertTrue(all(len(__snake_case							) == expected_mult_pad_length for x in input_a							)							)
             self.assertEqual(input_a.shape[:2]			,    (batch_size, expected_mult_pad_length)							)

             if feature_size > 1:
                   self.assertTrue(input_a.shape[2] == feature_size							)

             # Check padding value is correct
             A__						:  str		  =  (np.ones(self.feat_extract_tester.feature_size							) * feat_extract.padding_value).sum()
             self.assertTrue(
                 abs(np.asarray(input_a[0]							)[pad_min_length:].sum() - padding_vector_sum * (pad_max_length - pad_min_length)							)
                 < 1e-3							)
             self.assertTrue(
                 abs(
                     np.asarray(input_a[1]							)[pad_min_length + pad_diff :].sum()
                     - padding_vector_sum * (pad_max_length - pad_min_length - pad_diff)							)
                 < 1e-3							)
             self.assertTrue(
                 abs(
                     np.asarray(input_a[2]							)[pad_min_length + 2 * pad_diff :].sum()
                     - padding_vector_sum * (pad_max_length - pad_min_length - 2 * pad_diff)							)
                 < 1e-3							)
             self.assertTrue(
                 abs(input_a[0, pad_min_length:].sum() - padding_vector_sum * (pad_max_length - pad_min_length)							) < 1e-3							)
             self.assertTrue(
                 abs(input_a[0, pad_min_length:].sum() - padding_vector_sum * (expected_mult_pad_length - pad_min_length)							)
                 < 1e-3							)


       def 						_UpperCamelCase       (			self      :						Dict			,    snake_case      :						Union[str, Any]=False							):
             '''simple docstring'''





             def _inputs_have_equal_length(snake_case      :						List[str]							):
                   A__						:  str		  =  len(input[0]							)
                   for input_slice in input[1:]:
                         if len(__snake_case							) != length:
                               return False
                   return True

             def _inputs_are_equal(snake_case      :						Dict			,    snake_case      :						List[str]							):
                   if len(__snake_case							) != len(__snake_case							):
                         return False

                   for input_slice_a, input_slice_a in zip(__snake_case			,    __snake_case							):
                         if not np.allclose(np.asarray(__snake_case							)			,    np.asarray(__snake_case							)			,    atol=1e-3							):
                               return False
                   return True

             A__						:  str		  =  self.feature_extraction_class(**self.feat_extract_dict							)
             A__						:  Any		  =  self.feat_extract_tester.prepare_inputs_for_common(numpify=__snake_case							)
             A__						:  Tuple		  =  feat_extract.model_input_names[0]

             A__						:  Optional[int]		  =  BatchFeature({input_name: speech_inputs}							)

             # truncate to smallest
             A__						:  List[str]		  =  feat_extract.pad(
                 __snake_case			,    padding="""max_length"""			,    max_length=len(speech_inputs[0]							)			,    truncation=__snake_case							)
             A__						:  Optional[Any]		  =  input_a[input_name]

             A__						:  List[Any]		  =  feat_extract.pad(__snake_case			,    padding="""max_length"""			,    max_length=len(speech_inputs[0]							)							)
             A__						:  Any		  =  input_a[input_name]

             self.assertTrue(_inputs_have_equal_length(__snake_case							)							)
             self.assertFalse(_inputs_have_equal_length(__snake_case							)							)

             # truncate to smallest with np
             A__						:  Optional[Any]		  =  feat_extract.pad(
                 __snake_case			,    padding="""max_length"""			,    max_length=len(speech_inputs[0]							)			,    return_tensors="""np"""			,    truncation=__snake_case			,    )
             A__						:  int		  =  input_a[input_name]

             A__						:  List[Any]		  =  feat_extract.pad(
                 __snake_case			,    padding="""max_length"""			,    max_length=len(speech_inputs[0]							)			,    return_tensors="""np"""							)
             A__						:  Optional[Any]		  =  input_a[input_name]

             self.assertTrue(_inputs_have_equal_length(__snake_case							)							)
             self.assertTrue(input_a.shape[1] == len(speech_inputs[0]							)							)

             # since truncation forces padding to be smaller than longest input
             # function can't return `np.ndarray`, but has to return list
             self.assertFalse(_inputs_have_equal_length(__snake_case							)							)

             # truncate to middle
             A__						:  Any		  =  feat_extract.pad(
                 __snake_case			,    padding="""max_length"""			,    max_length=len(speech_inputs[1]							)			,    truncation=__snake_case			,    return_tensors="""np"""			,    )
             A__						:  Optional[int]		  =  input_a[input_name]

             A__						:  Optional[int]		  =  feat_extract.pad(
                 __snake_case			,    padding="""max_length"""			,    max_length=len(speech_inputs[1]							)			,    truncation=__snake_case							)
             A__						:  Dict		  =  input_a[input_name]

             A__						:  Optional[int]		  =  feat_extract.pad(
                 __snake_case			,    padding="""max_length"""			,    max_length=len(speech_inputs[1]							)			,    return_tensors="""np"""							)
             A__						:  List[Any]		  =  input_a[input_name]

             self.assertTrue(input_a.shape[1] == len(speech_inputs[1]							)							)
             self.assertTrue(_inputs_have_equal_length(__snake_case							)							)
             self.assertTrue(_inputs_have_equal_length(__snake_case							)							)
             self.assertTrue(_inputs_are_equal(__snake_case			,    __snake_case							)							)

             # since truncation forces padding to be smaller than longest input
             # function can't return `np.ndarray`, but has to return list
             self.assertFalse(_inputs_have_equal_length(__snake_case							)							)
             self.assertTrue(len(input_a[-1]							) == len(speech_inputs[-1]							)							)

             # padding has to be max_length when setting `truncation=True`
             with self.assertRaises(__snake_case							):
                   feat_extract.pad(__snake_case			,    truncation=__snake_case							)[input_name]

             # padding has to be max_length when setting `truncation=True`
             with self.assertRaises(__snake_case							):
                   feat_extract.pad(__snake_case			,    padding="""longest"""			,    truncation=__snake_case							)[input_name]

             # padding has to be max_length when setting `truncation=True`
             with self.assertRaises(__snake_case							):
                   feat_extract.pad(__snake_case			,    padding="""longest"""			,    truncation=__snake_case							)[input_name]

             # max_length parameter has to be provided when setting `truncation=True` and padding="max_length"
             with self.assertRaises(__snake_case							):
                   feat_extract.pad(__snake_case			,    padding="""max_length"""			,    truncation=__snake_case							)[input_name]

             # test truncation for `pad_to_multiple_of` for List[int] + numpy
             A__						:  Any		  =  12
             A__						:  Dict		  =  feat_extract.pad(
                 __snake_case			,    padding="""max_length"""			,    max_length=len(speech_inputs[0]							)			,    pad_to_multiple_of=__snake_case			,    truncation=__snake_case			,    )
             A__						:  List[str]		  =  input_a[input_name]

             A__						:  Dict		  =  feat_extract.pad(
                 __snake_case			,    padding="""max_length"""			,    max_length=len(speech_inputs[0]							)			,    pad_to_multiple_of=__snake_case			,    )
             A__						:  List[str]		  =  input_a[input_name]

             # retrieve expected_length as multiple of pad_to_multiple_of
             A__						:  List[str]		  =  len(speech_inputs[0]							)
             if expected_length % pad_to_multiple_of != 0:
                   A__						:  List[Any]		  =  ((len(speech_inputs[0]							) // pad_to_multiple_of) + 1) * pad_to_multiple_of

             self.assertTrue(len(input_a[0]							) == expected_length							)
             self.assertTrue(_inputs_have_equal_length(__snake_case							)							)
             self.assertFalse(_inputs_have_equal_length(__snake_case							)							)


       def 						_UpperCamelCase       (			self      :						Union[str, Any]							):
             '''simple docstring'''





             self._check_padding(numpify=__snake_case							)


       def 						_UpperCamelCase       (			self      :						int							):
             '''simple docstring'''





             self._check_padding(numpify=__snake_case							)


       def 						_UpperCamelCase       (			self      :						Union[str, Any]							):
             '''simple docstring'''





             self._check_truncation(numpify=__snake_case							)


       def 						_UpperCamelCase       (			self      :						str							):
             '''simple docstring'''





             self._check_truncation(numpify=__snake_case							)


       @require_torch
       def 						_UpperCamelCase       (			self      :						Optional[int]							):
             '''simple docstring'''





             A__						:  Optional[int]		  =  self.feature_extraction_class(**self.feat_extract_dict							)
             A__						:  Optional[Any]		  =  self.feat_extract_tester.prepare_inputs_for_common()
             A__						:  Dict		  =  feat_extract.model_input_names[0]

             A__						:  Any		  =  BatchFeature({input_name: speech_inputs}							)

             A__						:  Optional[Any]		  =  feat_extract.pad(__snake_case			,    padding="""longest"""			,    return_tensors="""np"""							)[input_name]
             A__						:  Dict		  =  feat_extract.pad(__snake_case			,    padding="""longest"""			,    return_tensors="""pt"""							)[input_name]

             self.assertTrue(abs(input_np.astype(np.floataa							).sum() - input_pt.numpy().astype(np.floataa							).sum()							) < 1e-2							)


       @require_tf
       def 						_UpperCamelCase       (			self      :						Optional[Any]							):
             '''simple docstring'''





             A__						:  List[Any]		  =  self.feature_extraction_class(**self.feat_extract_dict							)
             A__						:  Union[str, Any]		  =  self.feat_extract_tester.prepare_inputs_for_common()
             A__						:  Tuple		  =  feat_extract.model_input_names[0]

             A__						:  int		  =  BatchFeature({input_name: speech_inputs}							)

             A__						:  str		  =  feat_extract.pad(__snake_case			,    padding="""longest"""			,    return_tensors="""np"""							)[input_name]
             A__						:  List[Any]		  =  feat_extract.pad(__snake_case			,    padding="""longest"""			,    return_tensors="""tf"""							)[input_name]

             self.assertTrue(abs(input_np.astype(np.floataa							).sum() - input_tf.numpy().astype(np.floataa							).sum()							) < 1e-2							)


       def 						_UpperCamelCase       (			self      :						int							):
             '''simple docstring'''





             A__						:  Union[str, Any]		  =  self.feat_extract_dict
             A__						:  Union[str, Any]		  =  True
             A__						:  int		  =  self.feature_extraction_class(**__snake_case							)
             A__						:  Optional[int]		  =  self.feat_extract_tester.prepare_inputs_for_common()
             A__						:  Tuple		  =  [len(__snake_case							) for x in speech_inputs]
             A__						:  Optional[int]		  =  feat_extract.model_input_names[0]

             A__						:  int		  =  BatchFeature({input_name: speech_inputs}							)

             A__						:  int		  =  feat_extract.pad(__snake_case			,    padding="""longest"""			,    return_tensors="""np"""							)
             self.assertIn("""attention_mask"""			,    __snake_case							)
             self.assertListEqual(list(processed.attention_mask.shape							)			,    list(processed[input_name].shape[:2]							)							)
             self.assertListEqual(processed.attention_mask.sum(-1							).tolist()			,    __snake_case							)


       def 						_UpperCamelCase       (			self      :						Any							):
             '''simple docstring'''





             A__						:  str		  =  self.feat_extract_dict
             A__						:  Optional[int]		  =  True
             A__						:  int		  =  self.feature_extraction_class(**__snake_case							)
             A__						:  List[Any]		  =  self.feat_extract_tester.prepare_inputs_for_common()
             A__						:  Optional[int]		  =  [len(__snake_case							) for x in speech_inputs]
             A__						:  List[Any]		  =  feat_extract.model_input_names[0]

             A__						:  str		  =  BatchFeature({input_name: speech_inputs}							)
             A__						:  str		  =  min(__snake_case							)

             A__						:  List[str]		  =  feat_extract.pad(
                 __snake_case			,    padding="""max_length"""			,    max_length=__snake_case			,    truncation=__snake_case			,    return_tensors="""np"""							)
             self.assertIn("""attention_mask"""			,    __snake_case							)
             self.assertListEqual(
                 list(processed_pad.attention_mask.shape							)			,    [processed_pad[input_name].shape[0], max_length]							)
             self.assertListEqual(
                 processed_pad.attention_mask[:, :max_length].sum(-1							).tolist()			,    [max_length for x in speech_inputs]							)














"""simple docstring"""



import multiprocessing
import time

from arguments import PretokenizationArguments
from datasets import load_dataset

from transformers import AutoTokenizer, HfArgumentParser




def      _lowerCAmelCase		(							UpperCAmelCase__       :					List[str]      )		->str:
						A__						:  Tuple		  =  {}
						A__						:  Union[str, Any]		  =  tokenizer(example["""content"""],						truncation=UpperCAmelCase__      )["""input_ids"""]
						A__						:  Any		  =  len(example["""content"""]      ) / len(output["""input_ids"""]      )
						return output


A_    =							HfArgumentParser(PretokenizationArguments)
A_    =							parser.parse_args()
if args.num_workers is None:
						A_    =							multiprocessing.cpu_count()
A_    =							AutoTokenizer.from_pretrained(args.tokenizer_dir)

A_    =							time.time()
A_    =							load_dataset(args.dataset_name, split='''train''')
print(F'Dataset loaded in {time.time()-t_start:.2f}s')

A_    =							time.time()
A_    =							ds.map(
    tokenize,
    num_proc=args.num_workers,
    remove_columns=[
        '''repo_name''',
        '''path''',
        '''copies''',
        '''size''',
        '''content''',
        '''license''',
        '''hash''',
        '''line_mean''',
        '''line_max''',
        '''alpha_frac''',
        '''autogenerated''',
    ],
)
print(F'Dataset tokenized in {time.time()-t_start:.2f}s')

A_    =							time.time()
ds.push_to_hub(args.tokenized_data_repo)

print(F'Data pushed to the hub in {time.time()-t_start:.2f}s')






