



import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


__lowerCamelCase     :							Optional[Any]  							=				logging.getLogger(__name__)

def    lowerCamelCase_() ->       Optional[Any]:
						UpperCAmelCase         =     argparse.ArgumentParser(
						    description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."    )
						parser.add_argument(
						    "--dataset_name"    ,   type=lowerCamelCase_    ,   default="wikitext"    ,   help="Name of the training. Explore datasets at: hf.co/datasets."    ,   )
						parser.add_argument(
						    "--dataset_config"    ,   type=lowerCamelCase_    ,   default="wikitext-103-raw-v1"    ,   help="Configuration name of the dataset."    )
						parser.add_argument(
						    "--tokenizer_name_or_path"    ,   type=lowerCamelCase_    ,   default="sayakpaul/unigram-tokenizer-wikitext"    ,   help="Tokenizer identifier. Can be a local filepath or a Hub identifier."    ,   )
						parser.add_argument(
						    "--shard_size"    ,   type=lowerCamelCase_    ,   default=1_000    ,   help="Number of entries to go in a single shard."    ,   )
						parser.add_argument("--split"    ,   type=lowerCamelCase_    ,   default="train"    ,   choices=["train", "test", "validation"]    )
						parser.add_argument(
						    "--limit"    ,   default=lowerCamelCase_    ,   type=lowerCamelCase_    ,   help="Limit the number of shards (used for debugging)."    ,   )
						parser.add_argument(
						    "--max_length"    ,   type=lowerCamelCase_    ,   default=512    ,   help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
						    " sequence length that is a multiple of 8."    ,   )
						parser.add_argument(
						    "--output_dir"    ,   default="tf-tpu"    ,   type=lowerCamelCase_    ,   help="Output directory where the TFRecord shards will be saved. If the"
						    " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
						    " shards will be directly saved to a Google Cloud Storage bucket."    ,   )

						UpperCAmelCase         =     parser.parse_args()
						return args

def    lowerCamelCase_(lowerCamelCase_    ) ->       int:
						def fn(lowerCamelCase_    ):
												return tokenizer(examples["text"]    )

						return fn

def    lowerCamelCase_(lowerCamelCase_    ) ->       List[Any]:
						UpperCAmelCase         =     []
						for i in range(len(tokenized_data["input_ids"]    )    ):
												UpperCAmelCase         =     {
												    "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]    )    ),
												    "attention_mask": tf.train.Feature(
												        intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]    )    ),
												}
												UpperCAmelCase         =     tf.train.Features(feature=lowerCamelCase_    )
												UpperCAmelCase         =     tf.train.Example(features=lowerCamelCase_    )
												UpperCAmelCase         =     example.SerializeToString()
												records.append(lowerCamelCase_    )
						return records

def    lowerCamelCase_(lowerCamelCase_    ) ->       Any:
						UpperCAmelCase         =     datasets.load_dataset(args.dataset_name    ,   args.dataset_config    ,   split=args.split    )

						if args.limit is not None:
												UpperCAmelCase         =     min(len(lowerCamelCase_    )    ,   args.limit    )
												UpperCAmelCase         =     dataset.select(range(lowerCamelCase_    )    )
												print(F'Limiting the dataset to {args.limit} entries.'    )

						UpperCAmelCase         =     AutoTokenizer.from_pretrained(args.tokenizer_name_or_path    )

						# Handle output directory creation.
						# For serializing into a Google Cloud Storage Bucket, one needs to first
						# create a bucket.
						if "gs" not in args.output_dir:
												if not os.path.exists(args.output_dir    ):
																		os.makedirs(args.output_dir    )
												UpperCAmelCase         =     os.path.join(args.output_dir    ,   args.split    )
												if not os.path.exists(lowerCamelCase_    ):
																		os.makedirs(lowerCamelCase_    )
						else:
												UpperCAmelCase         =     os.path.join(args.output_dir    ,   args.split    )

						# Tokenize the whole dataset at once.
						UpperCAmelCase         =     tokenize_function(lowerCamelCase_    )
						UpperCAmelCase         =     dataset.map(lowerCamelCase_    ,   batched=lowerCamelCase_    ,   num_proc=4    ,   remove_columns=["text"]    )

						# We need to concatenate all our texts together, and then split the result
						# into chunks of a fixed size, which we will call block_size. To do this, we
						# will use the map method again, with the option batched=True. When we use batched=True,
						# the function we pass to map() will be passed multiple inputs at once, allowing us
						# to group them into more or fewer examples than we had in the input.
						# This allows us to create our new fixed-length samples. The advantage of this
						# method is that we don't lose a whole lot of content from the dataset compared to the
						# case where we simply tokenize with a pre-defined max_length.

						def group_texts(lowerCamelCase_    ):
												# Concatenate all texts.
												UpperCAmelCase         =     {k: sum(examples[k]    ,   []    ) for k in examples.keys()}
												UpperCAmelCase         =     len(concatenated_examples[list(examples.keys()    )[0]]    )
												# We drop the small remainder, though you could add padding instead if the model supports it
												# In this, as in all things, we advise you to follow your heart 🫀
												UpperCAmelCase         =     (total_length // args.max_length) * args.max_length
												# Split by chunks of max_len.
												UpperCAmelCase         =     {
												    k: [t[i : i + args.max_length] for i in range(0    ,   lowerCamelCase_    ,   args.max_length    )]
												    for k, t in concatenated_examples.items()
												}
												return result

						UpperCAmelCase         =     dataset_tokenized.map(lowerCamelCase_    ,   batched=lowerCamelCase_    ,   batch_size=1_000    ,   num_proc=4    )

						UpperCAmelCase         =     0
						UpperCAmelCase         =     0
						for shard in range(0    ,   len(lowerCamelCase_    )    ,   args.shard_size    ):
												UpperCAmelCase         =     grouped_dataset[shard : shard + args.shard_size]
												UpperCAmelCase         =     len(dataset_snapshot["input_ids"]    )
												UpperCAmelCase         =     os.path.join(lowerCamelCase_    ,   F'dataset-{shard_count}-{records_containing}.tfrecord'    )
												UpperCAmelCase         =     get_serialized_examples(lowerCamelCase_    )

												with tf.io.TFRecordWriter(lowerCamelCase_    ) as out_file:
																		for i in range(len(lowerCamelCase_    )    ):
																								UpperCAmelCase         =     serialized_examples[i]
																								out_file.write(lowerCamelCase_    )
																		print("Wrote file {} containing {} records".format(lowerCamelCase_    ,   lowerCamelCase_    )    )

												shard_count += 1
												total_records += records_containing

						with open(F'split-{args.split}-records-count.txt'    ,   "w"    ) as f:
												print(F'Total {args.split} records: {total_records}'    ,   file=lowerCamelCase_    )


if __name__ == "__main__":
		__lowerCamelCase     :							Dict  							=				parse_args()
		main(args)








import fire

from utils import calculate_rouge, save_json

def    lowerCamelCase_(lowerCamelCase_    ,   lowerCamelCase_    ,   lowerCamelCase_=None    ,   **lowerCamelCase_    ) ->       Optional[int]:
						UpperCAmelCase         =     [x.strip() for x in open(lowerCamelCase_    ).readlines()]
						UpperCAmelCase         =     [x.strip() for x in open(lowerCamelCase_    ).readlines()][: len(lowerCamelCase_    )]
						UpperCAmelCase         =     calculate_rouge(lowerCamelCase_    ,   lowerCamelCase_    ,   **lowerCamelCase_    )
						if save_path is not None:
												save_json(lowerCamelCase_    ,   lowerCamelCase_    ,   indent=lowerCamelCase_    )
						return metrics  # these print nicely


if __name__ == "__main__":
		fire.Fire(calculate_rouge_path)



