


# Copyright 2022 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os
import platform

import numpy as np
import psutil
import torch

from accelerate import __version__ as version
from accelerate.commands.config import default_config_file, load_config_from_file

from ..utils import is_npu_available, is_xpu_available





def   _UpperCAmelCase       (     UpperCAmelCase      :			List[Any]=None      ):
      """simple docstring"""




      if subparsers is not None:
            __lowerCamelCase     :      List[str]       						=						subparsers.add_parser("""env"""      )
      else:
            __lowerCamelCase     :      Union[str, Any]       						=						argparse.ArgumentParser("""Accelerate env command"""      )

      parser.add_argument(
          """--config_file"""						,	default=UpperCAmelCase						,	help="""The config file to use for the default values in the launching script."""      )

      if subparsers is not None:
            parser.set_defaults(func=UpperCAmelCase      )
      return parser





def   _UpperCAmelCase       (     UpperCAmelCase      :			Optional[Any]      ):
      """simple docstring"""




      __lowerCamelCase     :      Any       						=						torch.__version__
      __lowerCamelCase     :      Tuple       						=						torch.cuda.is_available()
      __lowerCamelCase     :      List[Any]       						=						is_xpu_available()
      __lowerCamelCase     :      Optional[int]       						=						is_npu_available()

      __lowerCamelCase     :      Optional[int]       						=						"""Not found"""
      # Get the default from the config file.
      if args.config_file is not None or os.path.isfile(UpperCAmelCase      ):
            __lowerCamelCase     :      Optional[Any]       						=						load_config_from_file(args.config_file      ).to_dict()

      __lowerCamelCase     :      Tuple       						=						{
          """`Accelerate` version""": version,
          """Platform""": platform.platform(),
          """Python version""": platform.python_version(),
          """Numpy version""": np.__version__,
          """PyTorch version (GPU?)""": f"""{pt_version} ({pt_cuda_available})""",
          """PyTorch XPU available""": str(UpperCAmelCase      ),
          """PyTorch NPU available""": str(UpperCAmelCase      ),
          """System RAM""": f"""{psutil.virtual_memory().total / 1_024 ** 3:.2f} GB""",
      }
      if pt_cuda_available:
            __lowerCamelCase     :      Any       						=						torch.cuda.get_device_name()

      print("""\nCopy-and-paste the text below in your GitHub issue\n"""      )
      print("""\n""".join([f"""- {prop}: {val}""" for prop, val in info.items()]      )      )

      print("""- `Accelerate` default config:""" if args.config_file is None else """- `Accelerate` config passed:"""      )
      __lowerCamelCase     :      str       						=						(
          """\n""".join([f"""\t- {prop}: {val}""" for prop, val in accelerate_config.items()]      )
          if isinstance(UpperCAmelCase						,	UpperCAmelCase      )
          else f"""\t{accelerate_config}"""
      )
      print(UpperCAmelCase      )

      __lowerCamelCase     :      str       						=						accelerate_config

      return info





def   _UpperCAmelCase       (     ):
      """simple docstring"""




      __lowerCamelCase     :      Optional[int]       						=						env_command_parser()
      __lowerCamelCase     :      str       						=						parser.parse_args()
      env_command(UpperCAmelCase      )
      return 0


if __name__ == "__main__":
    raise SystemExit(main())










import argparse
import shutil
from pathlib import Path

from tqdm import tqdm

from transformers import AutoTokenizer





def   _UpperCAmelCase       (     UpperCAmelCase      :			Any						,	UpperCAmelCase      :			int						,	UpperCAmelCase      :			int						,	UpperCAmelCase      :			Any=1_024      ):
      """simple docstring"""




      __lowerCamelCase						, __lowerCamelCase     :      str       						=						[], []

      __lowerCamelCase     :      Any       						=						list(zip(UpperCAmelCase						,	UpperCAmelCase      )      )
      __lowerCamelCase						, __lowerCamelCase     :      List[str]       						=						sorted_examples[0]

      def is_too_big(UpperCAmelCase      :			Optional[Any]      ):
            return tok(UpperCAmelCase						,	return_tensors="""pt"""      ).input_ids.shape[1] > max_tokens

      for src, tgt in tqdm(sorted_examples[1:]      ):
            __lowerCamelCase     :      Union[str, Any]       						=						new_src + """ """ + src
            __lowerCamelCase     :      str       						=						new_tgt + """ """ + tgt
            if is_too_big(UpperCAmelCase      ) or is_too_big(UpperCAmelCase      ):  # cant fit, finalize example
                  finished_src.append(UpperCAmelCase      )
                  finished_tgt.append(UpperCAmelCase      )
                  __lowerCamelCase						, __lowerCamelCase     :      str       						=						src, tgt
            else:  # can fit, keep adding
                  __lowerCamelCase						, __lowerCamelCase     :      int       						=						cand_src, cand_tgt

    # cleanup
      if new_src:
            assert new_tgt
            finished_src.append(UpperCAmelCase      )
            finished_tgt.append(UpperCAmelCase      )
      return finished_src, finished_tgt





def   _UpperCAmelCase       (     UpperCAmelCase      :			Tuple						,	UpperCAmelCase      :			Path						,	UpperCAmelCase      :			Dict						,	UpperCAmelCase      :			Optional[Any]      ):
      """simple docstring"""




      __lowerCamelCase     :      List[Any]       						=						Path(UpperCAmelCase      )
      save_path.mkdir(exist_ok=UpperCAmelCase      )
      for split in ["train"]:
            __lowerCamelCase						, __lowerCamelCase     :      List[Any]       						=						data_dir / f"""{split}.source""", data_dir / f"""{split}.target"""
            __lowerCamelCase     :      Tuple       						=						[x.rstrip() for x in Path(UpperCAmelCase      ).open().readlines()]
            __lowerCamelCase     :      Tuple       						=						[x.rstrip() for x in Path(UpperCAmelCase      ).open().readlines()]
            __lowerCamelCase						, __lowerCamelCase     :      int       						=						pack_examples(UpperCAmelCase						,	UpperCAmelCase						,	UpperCAmelCase						,	UpperCAmelCase      )
            print(f"""packed {split} split from {len(UpperCAmelCase      )} examples -> {len(UpperCAmelCase      )}."""      )
            Path(save_path / f"""{split}.source"""      ).open("""w"""      ).write("""\n""".join(UpperCAmelCase      )      )
            Path(save_path / f"""{split}.target"""      ).open("""w"""      ).write("""\n""".join(UpperCAmelCase      )      )
      for split in ["val", "test"]:
            __lowerCamelCase						, __lowerCamelCase     :      Optional[Any]       						=						data_dir / f"""{split}.source""", data_dir / f"""{split}.target"""
            shutil.copyfile(UpperCAmelCase						,	save_path / f"""{split}.source"""      )
            shutil.copyfile(UpperCAmelCase						,	save_path / f"""{split}.target"""      )





def   _UpperCAmelCase       (     ):
      """simple docstring"""




      __lowerCamelCase     :      List[Any]       						=						argparse.ArgumentParser()
      parser.add_argument("""--tok_name"""						,	type=UpperCAmelCase						,	help="""like facebook/bart-large-cnn,t5-base, etc."""      )
      parser.add_argument("""--max_seq_len"""						,	type=UpperCAmelCase						,	default=128      )
      parser.add_argument("""--data_dir"""						,	type=UpperCAmelCase      )
      parser.add_argument("""--save_path"""						,	type=UpperCAmelCase      )
      __lowerCamelCase     :      Union[str, Any]       						=						parser.parse_args()
      __lowerCamelCase     :      Tuple       						=						AutoTokenizer.from_pretrained(args.tok_name      )
      return pack_data_dir(UpperCAmelCase						,	Path(args.data_dir      )						,	args.max_seq_len						,	args.save_path      )


if __name__ == "__main__":
    packer_cli()



