





import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


_lowerCamelCase   				=			logging.getLogger(__name__)






def 				SCREAMING_SNAKE_CASE__							(       ):
  SCREAMING_SNAKE_CASE__			=      argparse.ArgumentParser(
      description="""Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."""    )
  parser.add_argument(
      """--dataset_name"""					,   type=UpperCamelCase__					,   default="""wikitext"""					,   help="""Name of the training. Explore datasets at: hf.co/datasets."""					,   )
  parser.add_argument(
      """--dataset_config"""					,   type=UpperCamelCase__					,   default="""wikitext-103-raw-v1"""					,   help="""Configuration name of the dataset."""    )
  parser.add_argument(
      """--tokenizer_name_or_path"""					,   type=UpperCamelCase__					,   default="""sayakpaul/unigram-tokenizer-wikitext"""					,   help="""Tokenizer identifier. Can be a local filepath or a Hub identifier."""					,   )
  parser.add_argument(
      """--shard_size"""					,   type=UpperCamelCase__					,   default=1_000					,   help="""Number of entries to go in a single shard."""					,   )
  parser.add_argument("""--split"""					,   type=UpperCamelCase__					,   default="""train"""					,   choices=["""train""", """test""", """validation"""]    )
  parser.add_argument(
      """--limit"""					,   default=UpperCamelCase__					,   type=UpperCamelCase__					,   help="""Limit the number of shards (used for debugging)."""					,   )
  parser.add_argument(
      """--max_length"""					,   type=UpperCamelCase__					,   default=512					,   help="""Maximum sequence length. For training on TPUs, it helps to have a maximum"""
      """ sequence length that is a multiple of 8."""					,   )
  parser.add_argument(
      """--output_dir"""					,   default="""tf-tpu"""					,   type=UpperCamelCase__					,   help="""Output directory where the TFRecord shards will be saved. If the"""
      """ path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"""
      """ shards will be directly saved to a Google Cloud Storage bucket."""					,   )

  SCREAMING_SNAKE_CASE__			=      parser.parse_args()
  return args






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							List[Any]    ):
  def fn(UpperCamelCase__:							Any    ):
    return tokenizer(examples["""text"""]    )

  return fn






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							Any    ):
  SCREAMING_SNAKE_CASE__			=      []
  for i in range(len(tokenized_data["""input_ids"""]    )    ):
    SCREAMING_SNAKE_CASE__			=      {
        """input_ids""": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["""input_ids"""][i]    )    ),
        """attention_mask""": tf.train.Feature(
            intaa_list=tf.train.IntaaList(value=tokenized_data["""attention_mask"""][i]    )    ),
    }
    SCREAMING_SNAKE_CASE__			=      tf.train.Features(feature=UpperCamelCase__    )
    SCREAMING_SNAKE_CASE__			=      tf.train.Example(features=UpperCamelCase__    )
    SCREAMING_SNAKE_CASE__			=      example.SerializeToString()
    records.append(UpperCamelCase__    )
  return records






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							List[str]    ):
  SCREAMING_SNAKE_CASE__			=      datasets.load_dataset(args.dataset_name					,   args.dataset_config					,   split=args.split    )

  if args.limit is not None:
    SCREAMING_SNAKE_CASE__			=      min(len(UpperCamelCase__    )					,   args.limit    )
    SCREAMING_SNAKE_CASE__			=      dataset.select(range(UpperCamelCase__    )    )
    print(f'''Limiting the dataset to {args.limit} entries.'''    )

  SCREAMING_SNAKE_CASE__			=      AutoTokenizer.from_pretrained(args.tokenizer_name_or_path    )

  # Handle output directory creation.
  # For serializing into a Google Cloud Storage Bucket, one needs to first
  # create a bucket.
  if "gs" not in args.output_dir:
    if not os.path.exists(args.output_dir    ):
      os.makedirs(args.output_dir    )
    SCREAMING_SNAKE_CASE__			=      os.path.join(args.output_dir					,   args.split    )
    if not os.path.exists(UpperCamelCase__    ):
      os.makedirs(UpperCamelCase__    )
  else:
    SCREAMING_SNAKE_CASE__			=      os.path.join(args.output_dir					,   args.split    )

  # Tokenize the whole dataset at once.
  SCREAMING_SNAKE_CASE__			=      tokenize_function(UpperCamelCase__    )
  SCREAMING_SNAKE_CASE__			=      dataset.map(UpperCamelCase__					,   batched=UpperCamelCase__					,   num_proc=4					,   remove_columns=["""text"""]    )

  # We need to concatenate all our texts together, and then split the result
  # into chunks of a fixed size, which we will call block_size. To do this, we
  # will use the map method again, with the option batched=True. When we use batched=True,
  # the function we pass to map() will be passed multiple inputs at once, allowing us
  # to group them into more or fewer examples than we had in the input.
  # This allows us to create our new fixed-length samples. The advantage of this
  # method is that we don't lose a whole lot of content from the dataset compared to the
  # case where we simply tokenize with a pre-defined max_length.

  def group_texts(UpperCamelCase__:							int    ):
    # Concatenate all texts.
    SCREAMING_SNAKE_CASE__			=      {k: sum(examples[k]					,   []    ) for k in examples.keys()}
    SCREAMING_SNAKE_CASE__			=      len(concatenated_examples[list(examples.keys()    )[0]]    )
    # We drop the small remainder, though you could add padding instead if the model supports it
    # In this, as in all things, we advise you to follow your heart 🫀
    SCREAMING_SNAKE_CASE__			=      (total_length // args.max_length) * args.max_length
    # Split by chunks of max_len.
    SCREAMING_SNAKE_CASE__			=      {
        k: [t[i : i + args.max_length] for i in range(0					,   UpperCamelCase__					,   args.max_length    )]
        for k, t in concatenated_examples.items()
    }
    return result

  SCREAMING_SNAKE_CASE__			=      dataset_tokenized.map(UpperCamelCase__					,   batched=UpperCamelCase__					,   batch_size=1_000					,   num_proc=4    )

  SCREAMING_SNAKE_CASE__			=      0
  SCREAMING_SNAKE_CASE__			=      0
  for shard in range(0					,   len(UpperCamelCase__    )					,   args.shard_size    ):
    SCREAMING_SNAKE_CASE__			=      grouped_dataset[shard : shard + args.shard_size]
    SCREAMING_SNAKE_CASE__			=      len(dataset_snapshot["""input_ids"""]    )
    SCREAMING_SNAKE_CASE__			=      os.path.join(UpperCamelCase__					,   f'''dataset-{shard_count}-{records_containing}.tfrecord'''    )
    SCREAMING_SNAKE_CASE__			=      get_serialized_examples(UpperCamelCase__    )

    with tf.io.TFRecordWriter(UpperCamelCase__    ) as out_file:
      for i in range(len(UpperCamelCase__    )    ):
        SCREAMING_SNAKE_CASE__			=      serialized_examples[i]
        out_file.write(UpperCamelCase__    )
      print("""Wrote file {} containing {} records""".format(UpperCamelCase__					,   UpperCamelCase__    )    )

    shard_count += 1
    total_records += records_containing

  with open(f'''split-{args.split}-records-count.txt'''					,   """w"""    ) as f:
    print(f'''Total {args.split} records: {total_records}'''					,   file=UpperCamelCase__    )


if __name__ == "__main__":
 _lowerCamelCase   				=			parse_args()
 main(args)






import inspect
import os
import unittest

import torch

import accelerate
from accelerate import Accelerator
from accelerate.test_utils import execute_subprocess_async, require_multi_gpu
from accelerate.utils import patch_environment




class   UpperCamelCase_       (     unittest.TestCase    ):


							def      _snake_case  (							self    :Any		)   ->   int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      inspect.getfile(accelerate.test_utils		)
									SCREAMING_SNAKE_CASE__			=      os.path.sep.join(mod_file.split(os.path.sep		)[:-1] + ["""scripts""", """test_script.py"""]		)
									SCREAMING_SNAKE_CASE__			=      os.path.sep.join(
									    mod_file.split(os.path.sep		)[:-1] + ["""scripts""", """test_distributed_data_loop.py"""]		)
									SCREAMING_SNAKE_CASE__			=      os.path.sep.join(mod_file.split(os.path.sep		)[:-1] + ["""scripts""", """test_ops.py"""]		)


							@require_multi_gpu
							def      _snake_case  (							self    :Optional[Any]		)   ->   Tuple:



									"""simple docstring"""



									print(f'''Found {torch.cuda.device_count()} devices.'''		)
									SCREAMING_SNAKE_CASE__			=      ["""torchrun""", f'''--nproc_per_node={torch.cuda.device_count()}''', self.test_file_path]
									with patch_environment(omp_num_threads=1		):
											execute_subprocess_async(__A							,				env=os.environ.copy()		)


							@require_multi_gpu
							def      _snake_case  (							self    :Tuple		)   ->   Optional[Any]:



									"""simple docstring"""



									print(f'''Found {torch.cuda.device_count()} devices.'''		)
									SCREAMING_SNAKE_CASE__			=      ["""torchrun""", f'''--nproc_per_node={torch.cuda.device_count()}''', self.operation_file_path]
									print(f'''Command: {cmd}'''		)
									with patch_environment(omp_num_threads=1		):
											execute_subprocess_async(__A							,				env=os.environ.copy()		)


							@require_multi_gpu
							def      _snake_case  (							self    :Optional[Any]		)   ->   Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      ["""torchrun""", f'''--nproc_per_node={torch.cuda.device_count()}''', inspect.getfile(self.__class__		)]
									with patch_environment(omp_num_threads=1		):
											execute_subprocess_async(__A							,				env=os.environ.copy()		)


							@require_multi_gpu
							def      _snake_case  (							self    :Optional[int]		)   ->   str:



									"""simple docstring"""



									print(f'''Found {torch.cuda.device_count()} devices, using 2 devices only'''		)
									SCREAMING_SNAKE_CASE__			=      ["""torchrun""", f'''--nproc_per_node={torch.cuda.device_count()}''', self.data_loop_file_path]
									with patch_environment(omp_num_threads=1							,				cuda_visible_devices="""0,1"""		):
											execute_subprocess_async(__A							,				env=os.environ.copy()		)


if __name__ == "__main__":
	_lowerCamelCase   				=			Accelerator()
	_lowerCamelCase   				=			(accelerator.state.process_index + 2, 10)
	_lowerCamelCase   				=			torch.randint(0, 10, shape).to(accelerator.device)

	_lowerCamelCase   				=			''

	_lowerCamelCase   				=			accelerator.pad_across_processes(tensor)
	if tensora.shape[0] != accelerator.state.num_processes + 1:
		error_msg += F"Found shape {tensora.shape} but should have {accelerator.state.num_processes + 1} at dim 0."
	if not torch.equal(tensora[: accelerator.state.process_index + 2], tensor):
		error_msg += "Tensors have different values."
	if not torch.all(tensora[accelerator.state.process_index + 2 :] == 0):
		error_msg += "Padding was not done with the right value (0)."

	_lowerCamelCase   				=			accelerator.pad_across_processes(tensor, pad_first=True)
	if tensora.shape[0] != accelerator.state.num_processes + 1:
		error_msg += F"Found shape {tensora.shape} but should have {accelerator.state.num_processes + 1} at dim 0."
	_lowerCamelCase   				=			accelerator.state.num_processes - accelerator.state.process_index - 1
	if not torch.equal(tensora[index:], tensor):
		error_msg += "Tensors have different values."
	if not torch.all(tensora[:index] == 0):
		error_msg += "Padding was not done with the right value (0)."

	# Raise error at the end to make sure we don't stop at the first failure.
	if len(error_msg) > 0:
		raise ValueError(error_msg)