




from argparse import ArgumentParser

from . import BaseTransformersCLICommand



def       UpperCAmelCase       (     _snake_case							):
							return DownloadCommand(args.model						, args.cache_dir						, args.force						, args.trust_remote_code							)


class 							__UpperCamelCase			( lowerCamelCase__  ):





				'''simple docstring'''



				@staticmethod
				def       __snake_case		( UpperCAmelCase_						):
											lowerCAmelCase   =		parser.add_parser('''download'''						)
											download_parser.add_argument(
											    '''--cache-dir'''       , type=__lowerCamelCase       , default=__lowerCamelCase       , help='''Path to location to store the models'''						)
											download_parser.add_argument(
											    '''--force'''       , action='''store_true'''       , help='''Force the model to be download even if already in cache-dir'''						)
											download_parser.add_argument(
											    '''--trust-remote-code'''       , action='''store_true'''       , help='''Whether or not to allow for custom models defined on the Hub in their own modeling files. Use only if you\'ve reviewed the code as it will execute on your local machine'''       , )
											download_parser.add_argument('''model'''       , type=__lowerCamelCase       , help='''Name of the model to download'''						)
											download_parser.set_defaults(func=__lowerCamelCase						)



				def __init__( self       , UpperCAmelCase_       , UpperCAmelCase_       , UpperCAmelCase_       , UpperCAmelCase_						):
											lowerCAmelCase   =		model
											lowerCAmelCase   =		cache
											lowerCAmelCase   =		force
											lowerCAmelCase   =		trust_remote_code






				def       __snake_case		( self						):
											from ..models.auto import AutoModel, AutoTokenizer

											AutoModel.from_pretrained(
											    self._model       , cache_dir=self._cache       , force_download=self._force       , trust_remote_code=self._trust_remote_code						)
											AutoTokenizer.from_pretrained(
											    self._model       , cache_dir=self._cache       , force_download=self._force       , trust_remote_code=self._trust_remote_code						)









import os
import re
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase_						=logging.get_logger(__name__)

UpperCAmelCase_						={
    """vocab_file""": """vocab.txt""",
    """merges_file""": """bpe.codes""",
}

UpperCAmelCase_						={
    """vocab_file""": {
        """vinai/phobert-base""": """https://huggingface.co/vinai/phobert-base/resolve/main/vocab.txt""",
        """vinai/phobert-large""": """https://huggingface.co/vinai/phobert-large/resolve/main/vocab.txt""",
    },
    """merges_file""": {
        """vinai/phobert-base""": """https://huggingface.co/vinai/phobert-base/resolve/main/bpe.codes""",
        """vinai/phobert-large""": """https://huggingface.co/vinai/phobert-large/resolve/main/bpe.codes""",
    },
}

UpperCAmelCase_						={
    """vinai/phobert-base""": 256,
    """vinai/phobert-large""": 256,
}



def       UpperCAmelCase       (     _snake_case							):
							lowerCAmelCase   =		set()
							lowerCAmelCase   =		word[0]
							for char in word[1:]:
														pairs.add((prev_char, char)							)
														lowerCAmelCase   =		char

							lowerCAmelCase   =		set(_snake_case							)
							return pairs


class 							__UpperCamelCase			( __UpperCAmelCase  ):





				'''simple docstring'''



				__a							: Union[str, Any]										=VOCAB_FILES_NAMES
				__a							: Optional[Any]										=PRETRAINED_VOCAB_FILES_MAP
				__a							: Optional[int]										=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES



				def __init__( self       , UpperCAmelCase_       , UpperCAmelCase_       , UpperCAmelCase_="<s>"       , UpperCAmelCase_="</s>"       , UpperCAmelCase_="</s>"       , UpperCAmelCase_="<s>"       , UpperCAmelCase_="<unk>"       , UpperCAmelCase_="<pad>"       , UpperCAmelCase_="<mask>"       , **UpperCAmelCase_       , ):
											super().__init__(
											    bos_token=UpperCAmelCase_       , eos_token=UpperCAmelCase_       , unk_token=UpperCAmelCase_       , sep_token=UpperCAmelCase_       , cls_token=UpperCAmelCase_       , pad_token=UpperCAmelCase_       , mask_token=UpperCAmelCase_       , **UpperCAmelCase_       , )

											lowerCAmelCase   =		vocab_file
											lowerCAmelCase   =		merges_file

											lowerCAmelCase   =		{}
											lowerCAmelCase   =		0
											lowerCAmelCase   =		1
											lowerCAmelCase   =		2
											lowerCAmelCase   =		3

											self.add_from_file(UpperCAmelCase_						)

											lowerCAmelCase   =		{v: k for k, v in self.encoder.items()}

											with open(UpperCAmelCase_       , encoding='''utf-8'''						) as merges_handle:
																		lowerCAmelCase   =		merges_handle.read().split('''\n'''						)[:-1]
											lowerCAmelCase   =		[tuple(merge.split()[:-1]						) for merge in merges]
											lowerCAmelCase   =		dict(zip(UpperCAmelCase_       , range(len(UpperCAmelCase_						)						)						)						)
											lowerCAmelCase   =		{}



				def       __snake_case		( self       , UpperCAmelCase_       , UpperCAmelCase_ = None						):

											if token_ids_a is None:
																		return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
											lowerCAmelCase   =		[self.cls_token_id]
											lowerCAmelCase   =		[self.sep_token_id]
											return cls + token_ids_a + sep + sep + token_ids_a + sep



				def       __snake_case		( self       , UpperCAmelCase_       , UpperCAmelCase_ = None       , UpperCAmelCase_ = False						):

											if already_has_special_tokens:
																		return super().get_special_tokens_mask(
																		    token_ids_a=UpperCAmelCase_       , token_ids_a=UpperCAmelCase_       , already_has_special_tokens=UpperCAmelCase_						)

											if token_ids_a is None:
																		return [1] + ([0] * len(UpperCAmelCase_						)) + [1]
											return [1] + ([0] * len(UpperCAmelCase_						)) + [1, 1] + ([0] * len(UpperCAmelCase_						)) + [1]



				def       __snake_case		( self       , UpperCAmelCase_       , UpperCAmelCase_ = None						):
											lowerCAmelCase   =		[self.sep_token_id]
											lowerCAmelCase   =		[self.cls_token_id]

											if token_ids_a is None:
																		return len(cls + token_ids_a + sep						) * [0]
											return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]



				@property
				def       __snake_case		( self						):
											return len(self.encoder						)



				def       __snake_case		( self						):
											return dict(self.encoder       , **self.added_tokens_encoder						)



				def       __snake_case		( self       , UpperCAmelCase_						):
											if token in self.cache:
																		return self.cache[token]
											lowerCAmelCase   =		tuple(UpperCAmelCase_						)
											lowerCAmelCase   =		tuple(list(word[:-1]						) + [word[-1] + '''</w>''']						)
											lowerCAmelCase   =		get_pairs(UpperCAmelCase_						)

											if not pairs:
																		return token

											while True:
																		lowerCAmelCase   =		min(UpperCAmelCase_       , key=lambda UpperCAmelCase_						: self.bpe_ranks.get(UpperCAmelCase_       , float('''inf'''						)						)						)
																		if bigram not in self.bpe_ranks:
																									break
																		lowerCAmelCase   ,     lowerCAmelCase   =		bigram
																		lowerCAmelCase   =		[]
																		lowerCAmelCase   =		0
																		while i < len(UpperCAmelCase_						):
																									try:
																																lowerCAmelCase   =		word.index(UpperCAmelCase_       , UpperCAmelCase_						)
																									except ValueError:
																																new_word.extend(word[i:]						)
																																break
																									else:
																																new_word.extend(word[i:j]						)
																																lowerCAmelCase   =		j

																									if word[i] == first and i < len(UpperCAmelCase_						) - 1 and word[i + 1] == second:
																																new_word.append(first + second						)
																																i += 2
																									else:
																																new_word.append(word[i]						)
																																i += 1
																		lowerCAmelCase   =		tuple(UpperCAmelCase_						)
																		lowerCAmelCase   =		new_word
																		if len(UpperCAmelCase_						) == 1:
																									break
																		else:
																									lowerCAmelCase   =		get_pairs(UpperCAmelCase_						)
											lowerCAmelCase   =		'''@@ '''.join(UpperCAmelCase_						)
											lowerCAmelCase   =		word[:-4]
											lowerCAmelCase   =		word
											return word



				def       __snake_case		( self       , UpperCAmelCase_						):
											lowerCAmelCase   =		[]

											lowerCAmelCase   =		re.findall(r'''\S+\n?'''       , UpperCAmelCase_						)

											for token in words:
																		split_tokens.extend(list(self.bpe(UpperCAmelCase_						).split(''' '''						)						)						)
											return split_tokens



				def       __snake_case		( self       , UpperCAmelCase_						):
											return self.encoder.get(UpperCAmelCase_       , self.encoder.get(self.unk_token						)						)



				def       __snake_case		( self       , UpperCAmelCase_						):
											return self.decoder.get(UpperCAmelCase_       , self.unk_token						)



				def       __snake_case		( self       , UpperCAmelCase_						):
											lowerCAmelCase   =		''' '''.join(UpperCAmelCase_						).replace('''@@ '''       , ''''''						).strip()
											return out_string



				def       __snake_case		( self       , UpperCAmelCase_       , UpperCAmelCase_ = None						):
											if not os.path.isdir(UpperCAmelCase_						):
																		logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""						)
																		return
											lowerCAmelCase   =		os.path.join(
											    UpperCAmelCase_       , (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']						)
											lowerCAmelCase   =		os.path.join(
											    UpperCAmelCase_       , (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']						)

											if os.path.abspath(self.vocab_file						) != os.path.abspath(UpperCAmelCase_						):
																		copyfile(self.vocab_file       , UpperCAmelCase_						)

											if os.path.abspath(self.merges_file						) != os.path.abspath(UpperCAmelCase_						):
																		copyfile(self.merges_file       , UpperCAmelCase_						)

											return out_vocab_file, out_merge_file






				def       __snake_case		( self       , UpperCAmelCase_						):
											if isinstance(UpperCAmelCase_       , UpperCAmelCase_						):
																		try:
																									with open(UpperCAmelCase_       , '''r'''       , encoding='''utf-8'''						) as fd:
																																self.add_from_file(UpperCAmelCase_						)
																		except FileNotFoundError as fnfe:
																									raise fnfe
																		except UnicodeError:
																									raise Exception(F"""Incorrect encoding detected in {f}, please rebuild the dataset"""						)
																		return

											lowerCAmelCase   =		f.readlines()
											for lineTmp in lines:
																		lowerCAmelCase   =		lineTmp.strip()
																		lowerCAmelCase   =		line.rfind(''' '''						)
																		if idx == -1:
																									raise ValueError('''Incorrect dictionary format, expected \'<token> <cnt>\''''						)
																		lowerCAmelCase   =		line[:idx]
																		lowerCAmelCase   =		len(self.encoder						)



