





"""simple docstring"""


import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


lowerCamelCase__				  =   logging.getLogger(__name__)




def 					lowercase__    (					)					->		Dict:




						"""simple docstring"""

						_UpperCamelCase      :	Optional[Any]						=       argparse.ArgumentParser(
						    description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."			)
						parser.add_argument(
						    "--dataset_name"		,type=lowercase_		,default="wikitext"		,help="Name of the training. Explore datasets at: hf.co/datasets."		,)
						parser.add_argument(
						    "--dataset_config"		,type=lowercase_		,default="wikitext-103-raw-v1"		,help="Configuration name of the dataset."			)
						parser.add_argument(
						    "--tokenizer_name_or_path"		,type=lowercase_		,default="sayakpaul/unigram-tokenizer-wikitext"		,help="Tokenizer identifier. Can be a local filepath or a Hub identifier."		,)
						parser.add_argument(
						    "--shard_size"		,type=lowercase_		,default=1_000		,help="Number of entries to go in a single shard."		,)
						parser.add_argument("--split"		,type=lowercase_		,default="train"		,choices=["train", "test", "validation"]			)
						parser.add_argument(
						    "--limit"		,default=lowercase_		,type=lowercase_		,help="Limit the number of shards (used for debugging)."		,)
						parser.add_argument(
						    "--max_length"		,type=lowercase_		,default=512		,help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
						    " sequence length that is a multiple of 8."		,)
						parser.add_argument(
						    "--output_dir"		,default="tf-tpu"		,type=lowercase_		,help="Output directory where the TFRecord shards will be saved. If the"
						    " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
						    " shards will be directly saved to a Google Cloud Storage bucket."		,)

						_UpperCamelCase      :	Any						=       parser.parse_args()
						return args




def 					lowercase__    (					lowercase_			)					->		Tuple:




						"""simple docstring"""

						def fn(lowercase_			):
												return tokenizer(examples["text"]			)

						return fn




def 					lowercase__    (					lowercase_			)					->		List[Any]:




						"""simple docstring"""

						_UpperCamelCase      :	Optional[int]						=       []
						for i in range(len(tokenized_data["input_ids"]			)			):
												_UpperCamelCase      :	Tuple						=       {
												    "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]			)			),
												    "attention_mask": tf.train.Feature(
												        intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]			)			),
												}
												_UpperCamelCase      :	Union[str, Any]						=       tf.train.Features(feature=lowercase_			)
												_UpperCamelCase      :	Tuple						=       tf.train.Example(features=lowercase_			)
												_UpperCamelCase      :	List[Any]						=       example.SerializeToString()
												records.append(lowercase_			)
						return records




def 					lowercase__    (					lowercase_			)					->		Optional[int]:




						"""simple docstring"""

						_UpperCamelCase      :	Union[str, Any]						=       datasets.load_dataset(args.dataset_name		,args.dataset_config		,split=args.split			)

						if args.limit is not None:
												_UpperCamelCase      :	List[Any]						=       min(len(lowercase_			)		,args.limit			)
												_UpperCamelCase      :	Union[str, Any]						=       dataset.select(range(lowercase_			)			)
												print(F'''Limiting the dataset to {args.limit} entries.'''			)

						_UpperCamelCase      :	Tuple						=       AutoTokenizer.from_pretrained(args.tokenizer_name_or_path			)

						# Handle output directory creation.
						# For serializing into a Google Cloud Storage Bucket, one needs to first
						# create a bucket.
						if "gs" not in args.output_dir:
												if not os.path.exists(args.output_dir			):
																		os.makedirs(args.output_dir			)
												_UpperCamelCase      :	int						=       os.path.join(args.output_dir		,args.split			)
												if not os.path.exists(lowercase_			):
																		os.makedirs(lowercase_			)
						else:
												_UpperCamelCase      :	Tuple						=       os.path.join(args.output_dir		,args.split			)

						# Tokenize the whole dataset at once.
						_UpperCamelCase      :	Optional[Any]						=       tokenize_function(lowercase_			)
						_UpperCamelCase      :	Optional[Any]						=       dataset.map(lowercase_		,batched=lowercase_		,num_proc=4		,remove_columns=["text"]			)

						# We need to concatenate all our texts together, and then split the result
						# into chunks of a fixed size, which we will call block_size. To do this, we
						# will use the map method again, with the option batched=True. When we use batched=True,
						# the function we pass to map() will be passed multiple inputs at once, allowing us
						# to group them into more or fewer examples than we had in the input.
						# This allows us to create our new fixed-length samples. The advantage of this
						# method is that we don't lose a whole lot of content from the dataset compared to the
						# case where we simply tokenize with a pre-defined max_length.

						def group_texts(lowercase_			):
												# Concatenate all texts.
												_UpperCamelCase      :	Tuple						=       {k: sum(examples[k]		,[]			) for k in examples.keys()}
												_UpperCamelCase      :	Union[str, Any]						=       len(concatenated_examples[list(examples.keys()			)[0]]			)
												# We drop the small remainder, though you could add padding instead if the model supports it
												# In this, as in all things, we advise you to follow your heart 🫀
												_UpperCamelCase      :	int						=       (total_length // args.max_length) * args.max_length
												# Split by chunks of max_len.
												_UpperCamelCase      :	Optional[int]						=       {
												    k: [t[i : i + args.max_length] for i in range(0		,lowercase_		,args.max_length			)]
												    for k, t in concatenated_examples.items()
												}
												return result

						_UpperCamelCase      :	Any						=       dataset_tokenized.map(lowercase_		,batched=lowercase_		,batch_size=1_000		,num_proc=4			)

						_UpperCamelCase      :	str						=       0
						_UpperCamelCase      :	Any						=       0
						for shard in range(0		,len(lowercase_			)		,args.shard_size			):
												_UpperCamelCase      :	Optional[Any]						=       grouped_dataset[shard : shard + args.shard_size]
												_UpperCamelCase      :	str						=       len(dataset_snapshot["input_ids"]			)
												_UpperCamelCase      :	List[str]						=       os.path.join(lowercase_		,F'''dataset-{shard_count}-{records_containing}.tfrecord'''			)
												_UpperCamelCase      :	Any						=       get_serialized_examples(lowercase_			)

												with tf.io.TFRecordWriter(lowercase_			) as out_file:
																		for i in range(len(lowercase_			)			):
																								_UpperCamelCase      :	Any						=       serialized_examples[i]
																								out_file.write(lowercase_			)
																		print("Wrote file {} containing {} records".format(lowercase_		,lowercase_			)			)

												shard_count += 1
												total_records += records_containing

						with open(F'''split-{args.split}-records-count.txt'''		,"w"			) as f:
												print(F'''Total {args.split} records: {total_records}'''		,file=lowercase_			)


if __name__ == "__main__":
		lowerCamelCase__				  =   parse_args()
		main(args)







"""simple docstring"""


import os
import zipfile

import requests
from get_ci_error_statistics import download_artifact, get_artifacts_links




def 					lowercase__    (					lowercase_		,lowercase_=7			)					->		Tuple:




      """simple docstring"""

      _UpperCamelCase      :	Optional[int]						=       None
      if token is not None:
            _UpperCamelCase      :	Optional[Any]						=       {"Accept": "application/vnd.github+json", "Authorization": F'''Bearer {token}'''}

      # The id of a workflow (not of a workflow run)
      _UpperCamelCase      :	Any						=       "636036"

      _UpperCamelCase      :	Tuple						=       F'''https://api.github.com/repos/huggingface/transformers/actions/workflows/{workflow_id}/runs'''
      # On `main` branch + event being `schedule` + not returning PRs + only `num_runs` results
      url += F'''?branch=main&event=schedule&exclude_pull_requests=true&per_page={num_runs}'''

      _UpperCamelCase      :	Dict						=       requests.get(lowercase_		,headers=lowercase_			).json()

      return result["workflow_runs"]




def 					lowercase__    (					lowercase_			)					->		List[str]:




      """simple docstring"""

      _UpperCamelCase      :	List[Any]						=       get_daily_ci_runs(lowercase_			)
      _UpperCamelCase      :	Tuple						=       None
      for workflow_run in workflow_runs:
            if workflow_run["status"] == "completed":
                  _UpperCamelCase      :	Union[str, Any]						=       workflow_run["id"]
                  break

      return workflow_run_id




def 					lowercase__    (					lowercase_		,lowercase_		,lowercase_			)					->		Optional[int]:




      """simple docstring"""

      _UpperCamelCase      :	str						=       get_last_daily_ci_runs(lowercase_			)
      if workflow_run_id is not None:
            _UpperCamelCase      :	int						=       get_artifacts_links(worflow_run_id=lowercase_		,token=lowercase_			)
            for artifact_name in artifact_names:
                  if artifact_name in artifacts_links:
                        _UpperCamelCase      :	Dict						=       artifacts_links[artifact_name]
                        download_artifact(
                            artifact_name=lowercase_		,artifact_url=lowercase_		,output_dir=lowercase_		,token=lowercase_			)




def 					lowercase__    (					lowercase_		,lowercase_		,lowercase_			)					->		int:




      """simple docstring"""

      get_last_daily_ci_artifacts(lowercase_		,lowercase_		,lowercase_			)

      _UpperCamelCase      :	Dict						=       {}
      for artifact_name in artifact_names:
            _UpperCamelCase      :	Union[str, Any]						=       os.path.join(lowercase_		,F'''{artifact_name}.zip'''			)
            if os.path.isfile(lowercase_			):
                  _UpperCamelCase      :	int						=       {}
                  with zipfile.ZipFile(lowercase_			) as z:
                        for filename in z.namelist():
                              if not os.path.isdir(lowercase_			):
                                    # read the file
                                    with z.open(lowercase_			) as f:
                                          _UpperCamelCase      :	int						=       f.read().decode("UTF-8"			)

      return results
