





'''simple docstring'''

import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

a        =						logging.get_logger(__name__)


a        =						{name: getattr(transformers, name + "Fast") for name in SLOW_TO_FAST_CONVERTERS}

def 						__magic_name__       (							__UpperCAmelCase							,	__UpperCAmelCase							,	__UpperCAmelCase							,	__UpperCAmelCase      )						->					str:

   '''simple docstring'''







   if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
      raise ValueError(f"""Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()      )}."""      )

   if tokenizer_name is None:
      __SCREAMING_SNAKE_CASE      							=       TOKENIZER_CLASSES
   else:
      __SCREAMING_SNAKE_CASE      							=       {tokenizer_name: getattr(__UpperCAmelCase							,	tokenizer_name + """Fast"""      )}

   logger.info(f"""Loading tokenizer classes: {tokenizer_names}"""      )

   for tokenizer_name in tokenizer_names:
      __SCREAMING_SNAKE_CASE      							=       TOKENIZER_CLASSES[tokenizer_name]

      __SCREAMING_SNAKE_CASE      							=       True
      if checkpoint_name is None:
         __SCREAMING_SNAKE_CASE      							=       list(tokenizer_class.max_model_input_sizes.keys()      )
      else:
         __SCREAMING_SNAKE_CASE      							=       [checkpoint_name]

      logger.info(f"""For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}"""      )

      for checkpoint in checkpoint_names:
         logger.info(f"""Loading {tokenizer_class.__class__.__name__} {checkpoint}"""      )

         # Load tokenizer
         __SCREAMING_SNAKE_CASE      							=       tokenizer_class.from_pretrained(__UpperCAmelCase							,	force_download=__UpperCAmelCase      )

         # Save fast tokenizer
         logger.info(f"""Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}"""      )

         # For organization names we create sub-directories
         if "/" in checkpoint:
            __SCREAMING_SNAKE_CASE							,     __SCREAMING_SNAKE_CASE      							=       checkpoint.split("""/"""      )
            __SCREAMING_SNAKE_CASE      							=       os.path.join(__UpperCAmelCase							,	__UpperCAmelCase      )
         elif add_prefix:
            __SCREAMING_SNAKE_CASE      							=       checkpoint
            __SCREAMING_SNAKE_CASE      							=       dump_path
         else:
            __SCREAMING_SNAKE_CASE      							=       None
            __SCREAMING_SNAKE_CASE      							=       dump_path

         logger.info(f"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""      )

         if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()      )[0]:
            __SCREAMING_SNAKE_CASE      							=       list(tokenizer.pretrained_vocab_files_map.values()      )[0][checkpoint]
            __SCREAMING_SNAKE_CASE      							=       file_path.split(__UpperCAmelCase      )[-1][0]
            if next_char == "/":
               __SCREAMING_SNAKE_CASE      							=       os.path.join(__UpperCAmelCase							,	__UpperCAmelCase      )
               __SCREAMING_SNAKE_CASE      							=       None

            logger.info(f"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""      )

         __SCREAMING_SNAKE_CASE      							=       tokenizer.save_pretrained(
             __UpperCAmelCase							,	legacy_format=__UpperCAmelCase							,	filename_prefix=__UpperCAmelCase      )
         logger.info(f"""=> File names {file_names}"""      )

         for file_name in file_names:
            if not file_name.endswith("""tokenizer.json"""      ):
               os.remove(__UpperCAmelCase      )
               logger.info(f"""=> removing {file_name}"""      )


if __name__ == "__main__":
    a        =						argparse.ArgumentParser()
    # Required parameters
    parser.add_argument(
        "--dump_path", default=None, type=str, required=True, help="Path to output generated fast tokenizer files."
    )
    parser.add_argument(
        "--tokenizer_name",
        default=None,
        type=str,
        help=(
            F'''Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will '''
            "download and convert all the checkpoints from AWS."
        ),
    )
    parser.add_argument(
        "--checkpoint_name",
        default=None,
        type=str,
        help="Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.",
    )
    parser.add_argument(
        "--force_download",
        action="store_true",
        help="Re-download checkpoints.",
    )
    a        =						parser.parse_args()

    convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)












'''simple docstring'''

from .dependency_versions_table import deps
from .utils.versions import require_version, require_version_core


# define which module versions we always want to check at run time
# (usually the ones defined in `install_requires` in setup.py)
#
# order specific notes:
# - tqdm must be checked before tokenizers

a        =						[
    "python",
    "tqdm",
    "regex",
    "requests",
    "packaging",
    "filelock",
    "numpy",
    "tokenizers",
    "huggingface-hub",
    "safetensors",
    "accelerate",
    "pyyaml",
]

for pkg in pkgs_to_check_at_runtime:
    if pkg in deps:
        if pkg == "tokenizers":
            # must be loaded here, or else tqdm check may fail
            from .utils import is_tokenizers_available

            if not is_tokenizers_available():
                continue  # not required, check version only if installed
        elif pkg == "accelerate":
            # must be loaded here, or else tqdm check may fail
            from .utils import is_accelerate_available

            # Maybe switch to is_torch_available in the future here so that Accelerate is hard dep of
            # Transformers with PyTorch
            if not is_accelerate_available():
                continue  # not required, check version only if installed

        require_version_core(deps[pkg])
    else:
        raise ValueError(F'''can\'t find {pkg} in {deps.keys()}, check dependency_versions_table.py''')

def 						__magic_name__       (							__UpperCAmelCase							,	__UpperCAmelCase=None      )						->					Optional[Any]:

   '''simple docstring'''







   require_version(deps[pkg]							,	__UpperCAmelCase      )





