






"""simple docstring"""



import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


A_    =							logging.getLogger(__name__)




def      _lowerCAmelCase		(							)		->List[str]:
      A__						:  List[str]		  =  argparse.ArgumentParser(
          description="""Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."""      )
      parser.add_argument(
          """--dataset_name""",						type=_lowercase,						default="""wikitext""",						help="""Name of the training. Explore datasets at: hf.co/datasets.""",						)
      parser.add_argument(
          """--dataset_config""",						type=_lowercase,						default="""wikitext-103-raw-v1""",						help="""Configuration name of the dataset."""      )
      parser.add_argument(
          """--tokenizer_name_or_path""",						type=_lowercase,						default="""sayakpaul/unigram-tokenizer-wikitext""",						help="""Tokenizer identifier. Can be a local filepath or a Hub identifier.""",						)
      parser.add_argument(
          """--shard_size""",						type=_lowercase,						default=1_0_0_0,						help="""Number of entries to go in a single shard.""",						)
      parser.add_argument("""--split""",						type=_lowercase,						default="""train""",						choices=["""train""", """test""", """validation"""]      )
      parser.add_argument(
          """--limit""",						default=_lowercase,						type=_lowercase,						help="""Limit the number of shards (used for debugging).""",						)
      parser.add_argument(
          """--max_length""",						type=_lowercase,						default=5_1_2,						help="""Maximum sequence length. For training on TPUs, it helps to have a maximum"""
          """ sequence length that is a multiple of 8.""",						)
      parser.add_argument(
          """--output_dir""",						default="""tf-tpu""",						type=_lowercase,						help="""Output directory where the TFRecord shards will be saved. If the"""
          """ path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord"""
          """ shards will be directly saved to a Google Cloud Storage bucket.""",						)

      A__						:  Dict		  =  parser.parse_args()
      return args




def      _lowerCAmelCase		(							UpperCAmelCase__       :					Tuple      )		->Dict:







      def fn(UpperCAmelCase__       :					int      ):
            return tokenizer(examples["""text"""]      )

      return fn




def      _lowerCAmelCase		(							UpperCAmelCase__       :					Optional[Any]      )		->Dict:
      A__						:  Optional[Any]		  =  []
      for i in range(len(tokenized_data["""input_ids"""]      )      ):
            A__						:  Optional[Any]		  =  {
                """input_ids""": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["""input_ids"""][i]      )      ),
                """attention_mask""": tf.train.Feature(
                    intaa_list=tf.train.IntaaList(value=tokenized_data["""attention_mask"""][i]      )      ),
            }
            A__						:  List[Any]		  =  tf.train.Features(feature=_lowercase      )
            A__						:  Dict		  =  tf.train.Example(features=_lowercase      )
            A__						:  List[Any]		  =  example.SerializeToString()
            records.append(_lowercase      )
      return records




def      _lowerCAmelCase		(							UpperCAmelCase__       :					Optional[int]      )		->Optional[int]:
      A__						:  Tuple		  =  datasets.load_dataset(args.dataset_name,						args.dataset_config,						split=args.split      )

      if args.limit is not None:
            A__						:  str		  =  min(len(_lowercase      ),						args.limit      )
            A__						:  Tuple		  =  dataset.select(range(_lowercase      )      )
            print(f'Limiting the dataset to {args.limit} entries.'      )

      A__						:  Any		  =  AutoTokenizer.from_pretrained(args.tokenizer_name_or_path      )

      # Handle output directory creation.
      # For serializing into a Google Cloud Storage Bucket, one needs to first
      # create a bucket.
      if "gs" not in args.output_dir:
            if not os.path.exists(args.output_dir      ):
                  os.makedirs(args.output_dir      )
            A__						:  int		  =  os.path.join(args.output_dir,						args.split      )
            if not os.path.exists(_lowercase      ):
                  os.makedirs(_lowercase      )
      else:
            A__						:  Dict		  =  os.path.join(args.output_dir,						args.split      )

      # Tokenize the whole dataset at once.
      A__						:  List[str]		  =  tokenize_function(_lowercase      )
      A__						:  Dict		  =  dataset.map(_lowercase,						batched=_lowercase,						num_proc=4,						remove_columns=["""text"""]      )

      # We need to concatenate all our texts together, and then split the result
      # into chunks of a fixed size, which we will call block_size. To do this, we
      # will use the map method again, with the option batched=True. When we use batched=True,
      # the function we pass to map() will be passed multiple inputs at once, allowing us
      # to group them into more or fewer examples than we had in the input.
      # This allows us to create our new fixed-length samples. The advantage of this
      # method is that we don't lose a whole lot of content from the dataset compared to the
      # case where we simply tokenize with a pre-defined max_length.

      def group_texts(UpperCAmelCase__       :					int      ):
            # Concatenate all texts.
            A__						:  List[Any]		  =  {k: sum(examples[k],						[]      ) for k in examples.keys()}
            A__						:  Optional[int]		  =  len(concatenated_examples[list(examples.keys()      )[0]]      )
            # We drop the small remainder, though you could add padding instead if the model supports it
            # In this, as in all things, we advise you to follow your heart 🫀
            A__						:  int		  =  (total_length // args.max_length) * args.max_length
            # Split by chunks of max_len.
            A__						:  Tuple		  =  {
                k: [t[i : i + args.max_length] for i in range(0,						_lowercase,						args.max_length      )]
                for k, t in concatenated_examples.items()
            }
            return result

      A__						:  Optional[int]		  =  dataset_tokenized.map(_lowercase,						batched=_lowercase,						batch_size=1_0_0_0,						num_proc=4      )

      A__						:  Dict		  =  0
      A__						:  List[Any]		  =  0
      for shard in range(0,						len(_lowercase      ),						args.shard_size      ):
            A__						:  Optional[Any]		  =  grouped_dataset[shard : shard + args.shard_size]
            A__						:  str		  =  len(dataset_snapshot["""input_ids"""]      )
            A__						:  Dict		  =  os.path.join(_lowercase,						f'dataset-{shard_count}-{records_containing}.tfrecord'      )
            A__						:  List[str]		  =  get_serialized_examples(_lowercase      )

            with tf.io.TFRecordWriter(_lowercase      ) as out_file:
                  for i in range(len(_lowercase      )      ):
                        A__						:  Dict		  =  serialized_examples[i]
                        out_file.write(_lowercase      )
                  print("""Wrote file {} containing {} records""".format(_lowercase,						_lowercase      )      )

            shard_count += 1
            total_records += records_containing

      with open(f'split-{args.split}-records-count.txt',						"""w"""      ) as f:
            print(f'Total {args.split} records: {total_records}',						file=_lowercase      )


if __name__ == "__main__":
      A_    =							parse_args()
      main(args)














"""simple docstring"""



import warnings
from typing import Any, Dict, List, Optional, Union

import numpy as np

from ...audio_utils import mel_filter_bank, optimal_fft_length, spectrogram, window_function
from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import PaddingStrategy, TensorType, logging


A_    =							logging.get_logger(__name__)






class 							__SCREAMING_SNAKE_CASE						(				UpperCamelCase		):
							snake_case_     =							['input_values', 'attention_mask']


							def __init__(			self      :						Union[str, Any]			,    snake_case      :						int = 1			,    snake_case      :						int = 1_6000			,    snake_case      :						float = 0.0			,    snake_case      :						bool = False			,    snake_case      :						int = 80			,    snake_case      :						int = 16			,    snake_case      :						int = 64			,    snake_case      :						str = "hann_window"			,    snake_case      :						float = 1.0			,    snake_case      :						float = 80			,    snake_case      :						float = 7600			,    snake_case      :						float = 1e-10			,    snake_case      :						int = 2			,    snake_case      :						bool = True			,    **snake_case      :						Dict			,    ):
													'''simple docstring'''





													super().__init__(feature_size=snake_case			,    sampling_rate=snake_case			,    padding_value=snake_case			,    **snake_case							)
													A__						:  Optional[int]		  =  do_normalize
													A__						:  List[Any]		  =  return_attention_mask

													A__						:  Tuple		  =  num_mel_bins
													A__						:  Optional[int]		  =  hop_length
													A__						:  List[str]		  =  win_length
													A__						:  List[Any]		  =  win_function
													A__						:  Tuple		  =  frame_signal_scale
													A__						:  Optional[Any]		  =  fmin
													A__						:  str		  =  fmax
													A__						:  str		  =  mel_floor
													A__						:  Dict		  =  reduction_factor

													A__						:  int		  =  win_length * sampling_rate // 1000
													A__						:  Optional[Any]		  =  hop_length * sampling_rate // 1000
													A__						:  Optional[int]		  =  optimal_fft_length(self.sample_size							)
													A__						:  List[Any]		  =  (self.n_fft // 2) + 1

													A__						:  str		  =  window_function(window_length=self.sample_size			,    name=self.win_function			,    periodic=snake_case							)

													A__						:  Optional[Any]		  =  mel_filter_bank(
													    num_frequency_bins=self.n_freqs			,    num_mel_filters=self.num_mel_bins			,    min_frequency=self.fmin			,    max_frequency=self.fmax			,    sampling_rate=self.sampling_rate			,    norm="""slaney"""			,    mel_scale="""slaney"""			,    )

													if frame_signal_scale != 1.0:
																			warnings.warn(
																			    """The argument `frame_signal_scale` is deprecated and will be removed in version 4.30.0 of Transformers"""			,    snake_case			,    )
													if reduction_factor != 2.0:
																			warnings.warn(
																			    """The argument `reduction_factor` is deprecated and will be removed in version 4.30.0 of Transformers"""			,    snake_case			,    )


							@staticmethod
							# Copied from transformers.models.wav2vec2.feature_extraction_wav2vec2.Wav2Vec2FeatureExtractor.zero_mean_unit_var_norm
							def 						_UpperCamelCase       (			snake_case      :						List[np.ndarray]			,    snake_case      :						List[np.ndarray]			,    snake_case      :						float = 0.0							):
													'''simple docstring'''





													if attention_mask is not None:
																			A__						:  Tuple		  =  np.array(snake_case			,    np.intaa							)
																			A__						:  List[Any]		  =  []

																			for vector, length in zip(snake_case			,    attention_mask.sum(-1							)							):
																									A__						:  str		  =  (vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1e-7							)
																									if length < normed_slice.shape[0]:
																															A__						:  int		  =  padding_value

																									normed_input_values.append(snake_case							)
													else:
																			A__						:  List[Any]		  =  [(x - x.mean()) / np.sqrt(x.var() + 1e-7							) for x in input_values]

													return normed_input_values


							def 						_UpperCamelCase       (			self      :						Optional[int]			,    snake_case      :						np.ndarray			,    ):
													'''simple docstring'''





													A__						:  List[Any]		  =  spectrogram(
													    snake_case			,    window=self.window			,    frame_length=self.sample_size			,    hop_length=self.sample_stride			,    fft_length=self.n_fft			,    mel_filters=self.mel_filters			,    mel_floor=self.mel_floor			,    log_mel="""log10"""			,    )
													return log_mel_spec.T


							def __call__(			self      :						str			,    snake_case      :						Optional[Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]] = None			,    snake_case      :						Optional[Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]] = None			,    snake_case      :						Union[bool, str, PaddingStrategy] = False			,    snake_case      :						Optional[int] = None			,    snake_case      :						bool = False			,    snake_case      :						Optional[int] = None			,    snake_case      :						Optional[bool] = None			,    snake_case      :						Optional[Union[str, TensorType]] = None			,    snake_case      :						Optional[int] = None			,    **snake_case      :						Union[str, Any]			,    ):
													'''simple docstring'''





													if audio is None and audio_target is None:
																			raise ValueError("""You must provide either `audio` or `audio_target` values."""							)

													if sampling_rate is not None:
																			if sampling_rate != self.sampling_rate:
																									raise ValueError(
																									    F'The model corresponding to this feature extractor: {self} was trained using a sampling rate of'
																									    F' {self.sampling_rate}. Please make sure that the provided audio input was sampled with'
																									    F' {self.sampling_rate} and not {sampling_rate}.'							)
													else:
																			logger.warning(
																			    """It is strongly recommended to pass the ``sampling_rate`` argument to this function. """
																			    """Failing to do so can result in silent errors that might be hard to debug."""							)

													if audio is not None:
																			A__						:  Dict		  =  self._process_audio(
																			    snake_case			,    snake_case			,    snake_case			,    snake_case			,    snake_case			,    snake_case			,    snake_case			,    snake_case			,    **snake_case			,    )
													else:
																			A__						:  Dict		  =  None

													if audio_target is not None:
																			A__						:  Union[str, Any]		  =  self._process_audio(
																			    snake_case			,    snake_case			,    snake_case			,    snake_case			,    snake_case			,    snake_case			,    snake_case			,    snake_case			,    **snake_case			,    )

																			if inputs is None:
																									return inputs_target
																			else:
																									A__						:  Union[str, Any]		  =  inputs_target["""input_values"""]
																									A__						:  str		  =  inputs_target.get("""attention_mask"""							)
																									if decoder_attention_mask is not None:
																															A__						:  List[str]		  =  decoder_attention_mask

													return inputs


							def 						_UpperCamelCase       (			self      :						str			,    snake_case      :						Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]			,    snake_case      :						bool = False			,    snake_case      :						Union[bool, str, PaddingStrategy] = False			,    snake_case      :						Optional[int] = None			,    snake_case      :						bool = False			,    snake_case      :						Optional[int] = None			,    snake_case      :						Optional[bool] = None			,    snake_case      :						Optional[Union[str, TensorType]] = None			,    **snake_case      :						Union[str, Any]			,    ):
													'''simple docstring'''





													A__						:  List[str]		  =  isinstance(snake_case			,    np.ndarray							) and len(speech.shape							) > 1
													if is_batched_numpy and len(speech.shape							) > 2:
																			raise ValueError(F'Only mono-channel audio is supported for input to {self}'							)
													A__						:  Tuple		  =  is_batched_numpy or (
													    isinstance(snake_case			,    (list, tuple)							) and (isinstance(speech[0]			,    (np.ndarray, tuple, list)							))
													)

													if is_batched:
																			A__						:  Tuple		  =  [np.asarray(snake_case			,    dtype=np.floataa							) for speech in speech]
													elif not is_batched and not isinstance(snake_case			,    np.ndarray							):
																			A__						:  Tuple		  =  np.asarray(snake_case			,    dtype=np.floataa							)
													elif isinstance(snake_case			,    np.ndarray							) and speech.dtype is np.dtype(np.floataa							):
																			A__						:  str		  =  speech.astype(np.floataa							)

													# always return batch
													if not is_batched:
																			A__						:  Optional[Any]		  =  [speech]

													# needed to make pad() work on spectrogram inputs
													A__						:  str		  =  self.feature_size

													# convert into correct format for padding
													if is_target:
																			A__						:  str		  =  [self._extract_mel_features(snake_case							) for waveform in speech]
																			A__						:  List[str]		  =  BatchFeature({"""input_values""": features}							)
																			A__						:  Dict		  =  self.num_mel_bins
													else:
																			A__						:  Tuple		  =  BatchFeature({"""input_values""": speech}							)

													A__						:  List[str]		  =  self.pad(
													    snake_case			,    padding=snake_case			,    max_length=snake_case			,    truncation=snake_case			,    pad_to_multiple_of=snake_case			,    return_attention_mask=snake_case			,    **snake_case			,    )

													A__						:  List[str]		  =  feature_size_hack

													# convert input values to correct format
													A__						:  str		  =  padded_inputs["""input_values"""]
													if not isinstance(input_values[0]			,    np.ndarray							):
																			A__						:  Tuple		  =  [np.asarray(snake_case			,    dtype=np.floataa							) for array in input_values]
													elif (
													    not isinstance(snake_case			,    np.ndarray							)
													    and isinstance(input_values[0]			,    np.ndarray							)
													    and input_values[0].dtype is np.dtype(np.floataa							)
													):
																			A__						:  Dict		  =  [array.astype(np.floataa							) for array in input_values]
													elif isinstance(snake_case			,    np.ndarray							) and input_values.dtype is np.dtype(np.floataa							):
																			A__						:  Any		  =  input_values.astype(np.floataa							)

													# convert attention_mask to correct format
													A__						:  List[Any]		  =  padded_inputs.get("""attention_mask"""							)
													if attention_mask is not None:
																			A__						:  Any		  =  [np.asarray(snake_case			,    dtype=np.intaa							) for array in attention_mask]

													# zero-mean and unit-variance normalization
													if not is_target and self.do_normalize:
																			A__						:  Optional[int]		  =  (
																			    attention_mask
																			    if self._get_padding_strategies(snake_case			,    max_length=snake_case							) is not PaddingStrategy.DO_NOT_PAD
																			    else None
																			)
																			A__						:  Optional[int]		  =  self.zero_mean_unit_var_norm(
																			    padded_inputs["""input_values"""]			,    attention_mask=snake_case			,    padding_value=self.padding_value							)

													if return_tensors is not None:
																			A__						:  Any		  =  padded_inputs.convert_to_tensors(snake_case							)

													return padded_inputs


							def 						_UpperCamelCase       (			self      :						Union[str, Any]							):
													'''simple docstring'''





													A__						:  List[str]		  =  super().to_dict()

													# Don't serialize these as they are derived from the other properties.
													A__						:  Dict		  =  ["""window""", """mel_filters""", """sample_size""", """sample_stride""", """n_fft""", """n_freqs"""]
													for name in names:
																			if name in output:
																									del output[name]

													return output






