


"""simple docstring"""

import importlib
import sys
from argparse import REMAINDER, ArgumentParser
from pathlib import Path

import torch_xla.distributed.xla_multiprocessing as xmp


def lowerCAmelCase_      (				):






 """simple docstring"""







 __lowercase									=	ArgumentParser(
     description=(
         """PyTorch TPU distributed training launch helper utility that will spawn up multiple distributed processes"""
     ) )

 # Optional arguments for the launch helper
 parser.add_argument("""--num_cores"""				,   type=UpperCamelCase__				,   default=1				,   help="""Number of TPU cores to use (1 or 8).""" )

 # positional
 parser.add_argument(
     """training_script"""				,   type=UpperCamelCase__				,   help=(
         """The full path to the single TPU training """
         """program/script to be launched in parallel, """
         """followed by all the arguments for the """
         """training script"""
     )				,   )

 # rest from the training program
 parser.add_argument("""training_script_args"""				,   nargs=UpperCamelCase__ )

 return parser.parse_args()


def lowerCAmelCase_      (				):






 """simple docstring"""







 __lowercase									=	parse_args()

 # Import training_script as a module.
 __lowercase									=	Path(args.training_script )
 sys.path.append(str(script_fpath.parent.resolve() ) )
 __lowercase									=	script_fpath.stem
 __lowercase									=	importlib.import_module(UpperCamelCase__ )

 # Patch sys.argv
 __lowercase									=	[args.training_script] + args.training_script_args + ["""--tpu_num_cores""", str(args.num_cores )]

 xmp.spawn(mod._mp_fn				,   args=()				,   nprocs=args.num_cores )


if __name__ == "__main__":
       main()




"""simple docstring"""

import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO
)
UpperCAmelCase__  							=logging.getLogger(__name__)

if __name__ == "__main__":
       UpperCAmelCase__  							=argparse.ArgumentParser(
           description="Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)"
       )
       parser.add_argument(
           "--data_file", type=str, default="data/dump.bert-base-uncased.pickle", help="The binarized dataset."
       )
       parser.add_argument(
           "--token_counts_dump", type=str, default="data/token_counts.bert-base-uncased.pickle", help="The dump file."
       )
       parser.add_argument("--vocab_size", default=3_0522, type=int)
       UpperCAmelCase__  							=parser.parse_args()

       logger.info(f"""Loading data from {args.data_file}""")
       with open(args.data_file, "rb") as fp:
              UpperCAmelCase__  							=pickle.load(fp)

       logger.info("Counting occurrences for MLM.")
       UpperCAmelCase__  							=Counter()
       for tk_ids in data:
              counter.update(tk_ids)
       UpperCAmelCase__  							=[0] * args.vocab_size
       for k, v in counter.items():
              UpperCAmelCase__  							=v

       logger.info(f"""Dump to {args.token_counts_dump}""")
       with open(args.token_counts_dump, "wb") as handle:
              pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)
