




'''simple docstring'''


import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


UpperCAmelCase_ 		=      logging.getLogger(__name__)


def 						SCREAMING_SNAKE_CASE       (      ):
  __a			=					argparse.ArgumentParser(
      description='Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'			)
  parser.add_argument(
      '--dataset_name'				,					type=a_				,					default='wikitext'				,					help='Name of the training. Explore datasets at: hf.co/datasets.'				,					)
  parser.add_argument(
      '--dataset_config'				,					type=a_				,					default='wikitext-103-raw-v1'				,					help='Configuration name of the dataset.'			)
  parser.add_argument(
      '--tokenizer_name_or_path'				,					type=a_				,					default='sayakpaul/unigram-tokenizer-wikitext'				,					help='Tokenizer identifier. Can be a local filepath or a Hub identifier.'				,					)
  parser.add_argument(
      '--shard_size'				,					type=a_				,					default=1000				,					help='Number of entries to go in a single shard.'				,					)
  parser.add_argument('--split'				,					type=a_				,					default='train'				,					choices=['train', 'test', 'validation']			)
  parser.add_argument(
      '--limit'				,					default=a_				,					type=a_				,					help='Limit the number of shards (used for debugging).'				,					)
  parser.add_argument(
      '--max_length'				,					type=a_				,					default=512				,					help='Maximum sequence length. For training on TPUs, it helps to have a maximum'
      ' sequence length that is a multiple of 8.'				,					)
  parser.add_argument(
      '--output_dir'				,					default='tf-tpu'				,					type=a_				,					help='Output directory where the TFRecord shards will be saved. If the'
      ' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'
      ' shards will be directly saved to a Google Cloud Storage bucket.'				,					)

  __a			=					parser.parse_args()
  return args


def 						SCREAMING_SNAKE_CASE       (      a_	:							Optional[int]			):
  def fn(a_	:							Optional[Any]			):
    return tokenizer(examples['text']			)

  return fn


def 						SCREAMING_SNAKE_CASE       (      a_	:							Optional[int]			):
  __a			=					[]
  for i in range(len(tokenized_data['input_ids']			)			):
    __a			=					{
        'input_ids': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['input_ids'][i]			)			),
        'attention_mask': tf.train.Feature(
            intaa_list=tf.train.IntaaList(value=tokenized_data['attention_mask'][i]			)			),
    }
    __a			=					tf.train.Features(feature=a_			)
    __a			=					tf.train.Example(features=a_			)
    __a			=					example.SerializeToString()
    records.append(a_			)
  return records


def 						SCREAMING_SNAKE_CASE       (      a_	:							Dict			):
  __a			=					datasets.load_dataset(args.dataset_name				,					args.dataset_config				,					split=args.split			)

  if args.limit is not None:
    __a			=					min(len(a_			)				,					args.limit			)
    __a			=					dataset.select(range(a_			)			)
    print(f"Limiting the dataset to {args.limit} entries."			)

  __a			=					AutoTokenizer.from_pretrained(args.tokenizer_name_or_path			)

  # Handle output directory creation.
  # For serializing into a Google Cloud Storage Bucket, one needs to first
  # create a bucket.
  if "gs" not in args.output_dir:
    if not os.path.exists(args.output_dir			):
      os.makedirs(args.output_dir			)
    __a			=					os.path.join(args.output_dir				,					args.split			)
    if not os.path.exists(a_			):
      os.makedirs(a_			)
  else:
    __a			=					os.path.join(args.output_dir				,					args.split			)

  # Tokenize the whole dataset at once.
  __a			=					tokenize_function(a_			)
  __a			=					dataset.map(a_				,					batched=a_				,					num_proc=4				,					remove_columns=['text']			)

  # We need to concatenate all our texts together, and then split the result
  # into chunks of a fixed size, which we will call block_size. To do this, we
  # will use the map method again, with the option batched=True. When we use batched=True,
  # the function we pass to map() will be passed multiple inputs at once, allowing us
  # to group them into more or fewer examples than we had in the input.
  # This allows us to create our new fixed-length samples. The advantage of this
  # method is that we don't lose a whole lot of content from the dataset compared to the
  # case where we simply tokenize with a pre-defined max_length.

  def group_texts(a_	:							Tuple			):
    # Concatenate all texts.
    __a			=					{k: sum(examples[k]				,					[]			) for k in examples.keys()}
    __a			=					len(concatenated_examples[list(examples.keys()			)[0]]			)
    # We drop the small remainder, though you could add padding instead if the model supports it
    # In this, as in all things, we advise you to follow your heart 🫀
    __a			=					(total_length // args.max_length) * args.max_length
    # Split by chunks of max_len.
    __a			=					{
        k: [t[i : i + args.max_length] for i in range(0				,					a_				,					args.max_length			)]
        for k, t in concatenated_examples.items()
    }
    return result

  __a			=					dataset_tokenized.map(a_				,					batched=a_				,					batch_size=1000				,					num_proc=4			)

  __a			=					0
  __a			=					0
  for shard in range(0				,					len(a_			)				,					args.shard_size			):
    __a			=					grouped_dataset[shard : shard + args.shard_size]
    __a			=					len(dataset_snapshot['input_ids']			)
    __a			=					os.path.join(a_				,					f"dataset-{shard_count}-{records_containing}.tfrecord"			)
    __a			=					get_serialized_examples(a_			)

    with tf.io.TFRecordWriter(a_			) as out_file:
      for i in range(len(a_			)			):
        __a			=					serialized_examples[i]
        out_file.write(a_			)
      print('Wrote file {} containing {} records'.format(a_				,					a_			)			)

    shard_count += 1
    total_records += records_containing

  with open(f"split-{args.split}-records-count.txt"				,					'w'			) as f:
    print(f"Total {args.split} records: {total_records}"				,					file=a_			)


if __name__ == "__main__":
  UpperCAmelCase_ 		=      parse_args()
  main(args)









'''simple docstring'''


# Usage:
# ./gen-card-allenai-wmt16.py

import os
from pathlib import Path


def 						SCREAMING_SNAKE_CASE       (      a_	:							Optional[Any]				,					a_	:							int				,					a_	:							Tuple				,					a_	:							Tuple			):
  __a			=					{
      'en': 'Machine learning is great, isn\'t it?',
      'ru': 'Машинное обучение - это здорово, не так ли?',
      'de': 'Maschinelles Lernen ist großartig, nicht wahr?',
  }

  # BLUE scores as follows:
  # "pair": [fairseq, transformers]
  __a			=					{
      'wmt16-en-de-dist-12-1': [28.3, 27.52],
      'wmt16-en-de-dist-6-1': [27.4, 27.11],
      'wmt16-en-de-12-1': [26.9, 25.75],
  }
  __a			=					f"{src_lang}-{tgt_lang}"

  __a			=					f"\n---\nlanguage:\n- {src_lang}\n- {tgt_lang}\nthumbnail:\ntags:\n- translation\n- wmt16\n- allenai\nlicense: apache-2.0\ndatasets:\n- wmt16\nmetrics:\n- bleu\n---\n\n# FSMT\n\n## Model description\n\nThis is a ported version of fairseq-based [wmt16 transformer](https://github.com/jungokasai/deep-shallow/) for {src_lang}-{tgt_lang}.\n\nFor more details, please, see [Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation](https://arxiv.org/abs/2006.10369).\n\nAll 3 models are available:\n\n* [wmt16-en-de-dist-12-1](https://huggingface.co/allenai/wmt16-en-de-dist-12-1)\n* [wmt16-en-de-dist-6-1](https://huggingface.co/allenai/wmt16-en-de-dist-6-1)\n* [wmt16-en-de-12-1](https://huggingface.co/allenai/wmt16-en-de-12-1)\n\n\n## Intended uses & limitations\n\n#### How to use\n\n```python\nfrom transformers import FSMTForConditionalGeneration, FSMTTokenizer\nmname = \"allenai/{model_name}\"\ntokenizer = FSMTTokenizer.from_pretrained(mname)\nmodel = FSMTForConditionalGeneration.from_pretrained(mname)\n\ninput = \"{texts[src_lang]}\"\ninput_ids = tokenizer.encode(input, return_tensors=\"pt\")\noutputs = model.generate(input_ids)\ndecoded = tokenizer.decode(outputs[0], skip_special_tokens=True)\nprint(decoded) # {texts[tgt_lang]}\n\n```\n\n#### Limitations and bias\n\n\n## Training data\n\nPretrained weights were left identical to the original model released by allenai. For more details, please, see the [paper](https://arxiv.org/abs/2006.10369).\n\n## Eval results\n\nHere are the BLEU scores:\n\nmodel   | fairseq | transformers\n-------|---------|----------\n{model_name}  | {scores[model_name][0]} | {scores[model_name][1]}\n\nThe score is slightly below the score reported in the paper, as the researchers don't use `sacrebleu` and measure the score on tokenized outputs. `transformers` score was measured using `sacrebleu` on detokenized outputs.\n\nThe score was calculated using this code:\n\n```bash\ngit clone https://github.com/huggingface/transformers\ncd transformers\nexport PAIR={pair}\nexport DATA_DIR=data/$PAIR\nexport SAVE_DIR=data/$PAIR\nexport BS=8\nexport NUM_BEAMS=5\nmkdir -p $DATA_DIR\nsacrebleu -t wmt16 -l $PAIR --echo src > $DATA_DIR/val.source\nsacrebleu -t wmt16 -l $PAIR --echo ref > $DATA_DIR/val.target\necho $PAIR\nPYTHONPATH=\"src:examples/seq2seq\" python examples/seq2seq/run_eval.py allenai/{model_name} $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS\n```\n\n## Data Sources\n\n- [training, etc.](http://www.statmt.org/wmt16/)\n- [test set](http://matrix.statmt.org/test_sets/newstest2016.tgz?1504722372)\n\n\n### BibTeX entry and citation info\n\n```\n@misc{{kasai2020deep,\n    title={{Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation}},\n    author={{Jungo Kasai and Nikolaos Pappas and Hao Peng and James Cross and Noah A. Smith}},\n    year={{2020}},\n    eprint={{2006.10369}},\n    archivePrefix={{arXiv}},\n    primaryClass={{cs.CL}}\n}}\n```\n\n"
  model_card_dir.mkdir(parents=a_				,					exist_ok=a_			)
  __a			=					os.path.join(a_				,					'README.md'			)
  print(f"Generating {path}"			)
  with open(a_				,					'w'				,					encoding='utf-8'			) as f:
    f.write(a_			)


# make sure we are under the root of the project
UpperCAmelCase_ 		=      Path(__file__).resolve().parent.parent.parent
UpperCAmelCase_ 		=      repo_dir / "model_cards"

for model_name in ["wmt16-en-de-dist-12-1", "wmt16-en-de-dist-6-1", "wmt16-en-de-12-1"]:
  UpperCAmelCase_ 		=      model_cards_dir / "allenai" / model_name
  write_model_card(model_card_dir, src_lang="en", tgt_lang="de", model_name=model_name)



