


"""simple docstring"""

import unittest
from dataclasses import dataclass

import pytest

from accelerate.commands.config.config_args import SageMakerConfig
from accelerate.utils import ComputeEnvironment
from accelerate.utils.launch import _convert_nargs_to_dict





@dataclass
class    lowerCamelCase__						(		_a    ):
      a :     Any											=		ComputeEnvironment.AMAZON_SAGEMAKER
      a :     str											=		True
      a :     Union[str, Any]											=		"""ml.p3.2xlarge"""
      a :     str											=		"""accelerate_sagemaker_execution_role"""
      a :     Optional[Any]											=		"""hf-sm"""
      a :     str											=		"""us-east-1"""
      a :     int											=		1
      a :     List[str]											=		"""accelerate-sagemaker-1"""
      a :     Optional[int]											=		"""1.6"""
      a :     Tuple											=		"""4.4"""
      a :     Optional[int]											=		"""train.py"""
      a :     Optional[Any]											=		[
          """--model_name_or_path""",
          """bert""",
          """--do_train""",
          """False""",
          """--epochs""",
          """3""",
          """--learning_rate""",
          """5e-5""",
          """--max_steps""",
          """50.5""",
      ]
      a :     Tuple											=		[
          """--model_name_or_path""",
          """bert""",
          """--do_train""",
          """--do_test""",
          """False""",
          """--do_predict""",
          """--epochs""",
          """3""",
          """--learning_rate""",
          """5e-5""",
          """--max_steps""",
          """50.5""",
      ]






class    lowerCamelCase__						(		unittest.TestCase    ):
      def SCREAMING_SNAKE_CASE_	(		self	:	str							):






       '''simple docstring'''


       __lowercase									=	_convert_nargs_to_dict(MockLaunchConfig.success_training_script_args							)
       assert isinstance(converted_args["""model_name_or_path"""]      ,       A_							)
       assert isinstance(converted_args["""do_train"""]      ,       A_							)
       assert isinstance(converted_args["""epochs"""]      ,       A_							)
       assert isinstance(converted_args["""learning_rate"""]      ,       A_							)
       assert isinstance(converted_args["""max_steps"""]      ,       A_							)

       with pytest.raises(A_							):
        _convert_nargs_to_dict(MockLaunchConfig.fail_training_script_args							)




"""simple docstring"""

import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO
)
UpperCAmelCase__  							=logging.getLogger(__name__)

if __name__ == "__main__":
       UpperCAmelCase__  							=argparse.ArgumentParser(
           description="Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)"
       )
       parser.add_argument(
           "--data_file", type=str, default="data/dump.bert-base-uncased.pickle", help="The binarized dataset."
       )
       parser.add_argument(
           "--token_counts_dump", type=str, default="data/token_counts.bert-base-uncased.pickle", help="The dump file."
       )
       parser.add_argument("--vocab_size", default=3_0522, type=int)
       UpperCAmelCase__  							=parser.parse_args()

       logger.info(f"""Loading data from {args.data_file}""")
       with open(args.data_file, "rb") as fp:
              UpperCAmelCase__  							=pickle.load(fp)

       logger.info("Counting occurrences for MLM.")
       UpperCAmelCase__  							=Counter()
       for tk_ids in data:
              counter.update(tk_ids)
       UpperCAmelCase__  							=[0] * args.vocab_size
       for k, v in counter.items():
              UpperCAmelCase__  							=v

       logger.info(f"""Dump to {args.token_counts_dump}""")
       with open(args.token_counts_dump, "wb") as handle:
              pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)
