

'''simple docstring'''


import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase__						:       List[str]										=						logging.get_logger(__name__)

lowerCAmelCase__						:       Dict										=						"""▁"""

lowerCAmelCase__						:       Tuple										=						{"""vocab_file""": """sentencepiece.bpe.model""", """monolingual_vocab_file""": """dict.txt"""}

lowerCAmelCase__						:       int										=						{
    """vocab_file""": {
        """vinai/bartpho-syllable""": """https://huggingface.co/vinai/bartpho-syllable/resolve/main/sentencepiece.bpe.model""",
    },
    """monolingual_vocab_file""": {
        """vinai/bartpho-syllable""": """https://huggingface.co/vinai/bartpho-syllable/resolve/main/dict.txt""",
    },
}

lowerCAmelCase__						:       str										=						{"""vinai/bartpho-syllable""": 1024}




class     a			(				SCREAMING_SNAKE_CASE       ):






						"""simple docstring"""




						__UpperCAmelCase			     =    VOCAB_FILES_NAMES
						__UpperCAmelCase			     =    PRETRAINED_VOCAB_FILES_MAP
						__UpperCAmelCase			     =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						__UpperCAmelCase			     =    ["""input_ids""", """attention_mask"""]







						def __init__(							self					:      List[str]			,			snake_case_					:      List[str]			,			snake_case_					:      Optional[int]			,			snake_case_					:      Dict="<s>"			,			snake_case_					:      List[Any]="</s>"			,			snake_case_					:      List[Any]="</s>"			,			snake_case_					:      Optional[int]="<s>"			,			snake_case_					:      Tuple="<unk>"			,			snake_case_					:      Optional[Any]="<pad>"			,			snake_case_					:      Optional[int]="<mask>"			,			snake_case_					:      Optional[Dict[str, Any]] = None			,			**snake_case_					:      List[str]			,			):



								'''simple docstring'''



								snake_case__						:       str       						=  AddedToken(snake_case_			,			lstrip=snake_case_			,			rstrip=snake_case_  ) if isinstance(snake_case_			,			snake_case_  ) else mask_token

								snake_case__						:       Any       						=  {} if sp_model_kwargs is None else sp_model_kwargs

								super().__init__(
								    bos_token=snake_case_			,			eos_token=snake_case_			,			unk_token=snake_case_			,			sep_token=snake_case_			,			cls_token=snake_case_			,			pad_token=snake_case_			,			mask_token=snake_case_			,			sp_model_kwargs=self.sp_model_kwargs			,			**snake_case_			,			)

								snake_case__						:       str       						=  vocab_file
								snake_case__						:       Tuple       						=  monolingual_vocab_file
								snake_case__						:       Union[str, Any]       						=  spm.SentencePieceProcessor(**self.sp_model_kwargs  )
								self.sp_model.Load(str(snake_case_  )  )

								# Load the reduced vocab

								# Keep order of special tokens for backward compatibility
								snake_case__						:       List[str]       						=  {}
								snake_case__						:       Any       						=  0
								for token in [bos_token, pad_token, eos_token, unk_token, sep_token, cls_token]:
										if str(snake_case_  ) not in self.fairseq_tokens_to_ids:
												snake_case__						:       Union[str, Any]       						=  cnt
												cnt += 1
								with open(snake_case_			,			'''r'''			,			encoding='''utf-8'''  ) as f:
										for line in f.readlines():
												snake_case__						:       str       						=  line.strip().split()[0]
												snake_case__						:       List[Any]       						=  len(self.fairseq_tokens_to_ids  )
								if str(snake_case_  ) not in self.fairseq_tokens_to_ids:
										snake_case__						:       Union[str, Any]       						=  len(self.fairseq_tokens_to_ids  )

								snake_case__						:       List[str]       						=  {v: k for k, v in self.fairseq_tokens_to_ids.items()}







						def __getstate__(							self					:      Tuple  ):



								'''simple docstring'''



								snake_case__						:       str       						=  self.__dict__.copy()
								snake_case__						:       Optional[int]       						=  None
								snake_case__						:       str       						=  self.sp_model.serialized_model_proto()
								return state







						def __setstate__(							self					:      Tuple			,			snake_case_					:      Dict  ):



								'''simple docstring'''



								snake_case__						:       Any       						=  d

								# for backward compatibility
								if not hasattr(self			,			'''sp_model_kwargs'''  ):
										snake_case__						:       Dict       						=  {}

								snake_case__						:       Optional[Any]       						=  spm.SentencePieceProcessor(**self.sp_model_kwargs  )
								self.sp_model.LoadFromSerializedProto(self.sp_model_proto  )







						def       __magic_name__							(							self					:      Any			,			snake_case_					:      List[int]			,			snake_case_					:      Optional[List[int]] = None  ):



								'''simple docstring'''



								if token_ids_a is None:
										return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
								snake_case__						:       int       						=  [self.cls_token_id]
								snake_case__						:       List[Any]       						=  [self.sep_token_id]
								return cls + token_ids_a + sep + sep + token_ids_a + sep







						def       __magic_name__							(							self					:      Optional[int]			,			snake_case_					:      List[int]			,			snake_case_					:      Optional[List[int]] = None			,			snake_case_					:      bool = False  ):



								'''simple docstring'''



								if already_has_special_tokens:
										return super().get_special_tokens_mask(
										    token_ids_a=snake_case_			,			token_ids_a=snake_case_			,			already_has_special_tokens=snake_case_  )

								if token_ids_a is None:
										return [1] + ([0] * len(snake_case_  )) + [1]
								return [1] + ([0] * len(snake_case_  )) + [1, 1] + ([0] * len(snake_case_  )) + [1]







						def       __magic_name__							(							self					:      str			,			snake_case_					:      List[int]			,			snake_case_					:      Optional[List[int]] = None  ):



								'''simple docstring'''



								snake_case__						:       List[Any]       						=  [self.sep_token_id]
								snake_case__						:       Optional[int]       						=  [self.cls_token_id]

								if token_ids_a is None:
										return len(cls + token_ids_a + sep  ) * [0]
								return len(cls + token_ids_a + sep + sep + token_ids_a + sep  ) * [0]







						@property
						def       __magic_name__							(							self					:      int  ):



								'''simple docstring'''



								return len(self.fairseq_ids_to_tokens  )







						def       __magic_name__							(							self					:      str  ):



								'''simple docstring'''



								snake_case__						:       int       						=  {self.convert_ids_to_tokens(snake_case_  ): i for i in range(self.vocab_size  )}
								vocab.update(self.added_tokens_encoder  )
								return vocab







						def       __magic_name__							(							self					:      Tuple			,			snake_case_					:      str  ):



								'''simple docstring'''



								return self.sp_model.encode(snake_case_			,			out_type=snake_case_  )







						def       __magic_name__							(							self					:      Dict			,			snake_case_					:      Union[str, Any]  ):



								'''simple docstring'''



								if token in self.fairseq_tokens_to_ids:
										return self.fairseq_tokens_to_ids[token]
								else:
										return self.unk_token_id







						def       __magic_name__							(							self					:      Union[str, Any]			,			snake_case_					:      Optional[int]  ):



								'''simple docstring'''



								return self.fairseq_ids_to_tokens[index]







						def       __magic_name__							(							self					:      List[Any]			,			snake_case_					:      Optional[int]  ):



								'''simple docstring'''



								snake_case__						:       Union[str, Any]       						=  ''''''.join(snake_case_  ).replace(snake_case_			,			''' '''  ).strip()
								return out_string







						def       __magic_name__							(							self					:      Union[str, Any]			,			snake_case_					:      str			,			snake_case_					:      Optional[str] = None  ):



								'''simple docstring'''



								if not os.path.isdir(snake_case_  ):
										logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""  )
										return
								snake_case__						:       str       						=  os.path.join(
								    snake_case_			,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']  )
								snake_case__						:       Dict       						=  os.path.join(
								    snake_case_			,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''monolingual_vocab_file''']			,			)

								if os.path.abspath(self.vocab_file  ) != os.path.abspath(snake_case_  ) and os.path.isfile(self.vocab_file  ):
										copyfile(self.vocab_file			,			snake_case_  )
								elif not os.path.isfile(self.vocab_file  ):
										with open(snake_case_			,			'''wb'''  ) as fi:
												snake_case__						:       Any       						=  self.sp_model.serialized_model_proto()
												fi.write(snake_case_  )

								if os.path.abspath(self.monolingual_vocab_file  ) != os.path.abspath(
								    snake_case_  ) and os.path.isfile(self.monolingual_vocab_file  ):
										copyfile(self.monolingual_vocab_file			,			snake_case_  )
								elif not os.path.isfile(self.monolingual_vocab_file  ):
										with open(snake_case_			,			'''w'''			,			encoding='''utf-8'''  ) as fp:
												for token in self.fairseq_tokens_to_ids:
														if token not in self.all_special_tokens:
																fp.write(F"""{str(snake_case_  )} \n"""  )

								return out_vocab_file, out_monolingual_vocab_file






'''simple docstring'''


# Copyright 2022 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os
import subprocess

from packaging.version import Version, parse

from accelerate.commands.config.config_args import default_config_file, load_config_from_file


lowerCAmelCase__						:       List[Any]										=						"""Run commands across TPU VMs for initial setup before running `accelerate launch`."""




def 	_a       (				__lowerCAmelCase		:      Any=None     ):

		"""simple docstring"""

		if subparsers is not None:
				snake_case__						:       Tuple       						=  subparsers.add_parser('''tpu-config'''   ,							description=_description     )
		else:
				snake_case__						:       str       						=  argparse.ArgumentParser('''Accelerate tpu-config command'''   ,							description=_description     )
		# Core arguments
		snake_case__						:       Any       						=  parser.add_argument_group(
		    '''Config Arguments'''   ,							'''Arguments that can be configured through `accelerate config`.'''     )
		config_args.add_argument(
		    '''--config_file'''   ,							type=__lowerCAmelCase   ,							default=__lowerCAmelCase   ,							help='''Path to the config file to use for accelerate.'''   ,							)
		config_args.add_argument(
		    '''--tpu_name'''   ,							default=__lowerCAmelCase   ,							help='''The name of the TPU to use. If not specified, will use the TPU specified in the config file.'''   ,							)
		config_args.add_argument(
		    '''--tpu_zone'''   ,							default=__lowerCAmelCase   ,							help='''The zone of the TPU to use. If not specified, will use the zone specified in the config file.'''   ,							)
		snake_case__						:       Optional[Any]       						=  parser.add_argument_group('''TPU Arguments'''   ,							'''Arguments for options ran inside the TPU.'''     )
		pod_args.add_argument(
		    '''--use_alpha'''   ,							action='''store_true'''   ,							help='''Whether to use `gcloud alpha` when running the TPU training script instead of `gcloud`.'''   ,							)
		pod_args.add_argument(
		    '''--command_file'''   ,							default=__lowerCAmelCase   ,							help='''The path to the file containing the commands to run on the pod on startup.'''   ,							)
		pod_args.add_argument(
		    '''--command'''   ,							action='''append'''   ,							nargs='''+'''   ,							help='''A command to run on the pod. Can be passed multiple times.'''   ,							)
		pod_args.add_argument(
		    '''--install_accelerate'''   ,							action='''store_true'''   ,							help='''Whether to install accelerate on the pod. Defaults to False.'''   ,							)
		pod_args.add_argument(
		    '''--accelerate_version'''   ,							default='''latest'''   ,							help='''The version of accelerate to install on the pod. If not specified, will use the latest pypi version. Specify \'dev\' to install from GitHub.'''   ,							)
		pod_args.add_argument(
		    '''--debug'''   ,							action='''store_true'''   ,							help='''If set, will print the command that would be run instead of running it.'''     )

		if subparsers is not None:
				parser.set_defaults(func=__lowerCAmelCase     )
		return parser




def 	_a       (				__lowerCAmelCase		:      Optional[Any]     ):

		"""simple docstring"""

		snake_case__						:       Optional[int]       						=  None

		# Get the default from the config file if it exists.
		if args.config_file is not None or os.path.isfile(__lowerCAmelCase     ):
				snake_case__						:       Tuple       						=  load_config_from_file(args.config_file     )
				if not args.command_file and defaults.command_file is not None and not args.command:
						snake_case__						:       Optional[int]       						=  defaults.command_file
				if not args.command and defaults.commands is not None:
						snake_case__						:       Union[str, Any]       						=  defaults.commands
				if not args.tpu_name:
						snake_case__						:       Union[str, Any]       						=  defaults.tpu_name
				if not args.tpu_zone:
						snake_case__						:       Optional[Any]       						=  defaults.tpu_zone
		if args.accelerate_version == "dev":
				snake_case__						:       Tuple       						=  '''git+https://github.com/huggingface/accelerate.git'''
		elif args.accelerate_version == "latest":
				snake_case__						:       Union[str, Any]       						=  '''accelerate -U'''
		elif isinstance(parse(args.accelerate_version     )   ,							__lowerCAmelCase     ):
				snake_case__						:       Any       						=  F"""accelerate=={args.accelerate_version}"""

		if not args.command_file and not args.command:
				raise ValueError('''You must specify either a command file or a command to run on the pod.'''     )

		if args.command_file:
				with open(args.command_file   ,							'''r'''     ) as f:
						snake_case__						:       str       						=  [f.read().splitlines()]

    # To turn list of lists into list of strings
		if isinstance(args.command[0]   ,							__lowerCAmelCase     ):
				snake_case__						:       List[Any]       						=  [line for cmd in args.command for line in cmd]
		# Default to the shared folder and install accelerate
		snake_case__						:       Optional[int]       						=  ['''cd /usr/share''']
		if args.install_accelerate:
				new_cmd += [F"""pip install {args.accelerate_version}"""]
		new_cmd += args.command
		snake_case__						:       Tuple       						=  '''; '''.join(__lowerCAmelCase     )

		# Then send it to gcloud
		# Eventually try to use google-api-core to do this instead of subprocess
		snake_case__						:       str       						=  ['''gcloud''']
		if args.use_alpha:
				cmd += ["alpha"]
		cmd += [
		    "compute",
		    "tpus",
		    "tpu-vm",
		    "ssh",
		    args.tpu_name,
		    "--zone",
		    args.tpu_zone,
		    "--command",
		    args.command,
		    "--worker",
		    "all",
		]
		if args.debug:
				print(F"""Running {' '.join(__lowerCAmelCase     )}"""     )
				return
		subprocess.run(__lowerCAmelCase     )
		print('''Successfully setup pod.'''     )





def 	_a       (				):

		"""simple docstring"""

		snake_case__						:       Dict       						=  tpu_command_parser()
		snake_case__						:       Dict       						=  parser.parse_args()

		tpu_command_launcher(__lowerCAmelCase     )



