




import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__lowerCamelCase   :						int				=			logging.get_logger(__name__)
__lowerCamelCase   :						Any				=			{'vocab_file': 'spiece.model'}

__lowerCamelCase   :						str				=			{
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    }
}

__lowerCamelCase   :						List[Any]				=			{
    'albert-base-v1': 5_12,
    'albert-large-v1': 5_12,
    'albert-xlarge-v1': 5_12,
    'albert-xxlarge-v1': 5_12,
    'albert-base-v2': 5_12,
    'albert-large-v2': 5_12,
    'albert-xlarge-v2': 5_12,
    'albert-xxlarge-v2': 5_12,
}

__lowerCamelCase   :						Dict				=			'▁'


class    lowerCamelCase    (		UpperCamelCase_						):






   '''simple docstring'''





   UpperCamelCase__								=VOCAB_FILES_NAMES
   UpperCamelCase__								=PRETRAINED_VOCAB_FILES_MAP
   UpperCamelCase__								=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES

   def __init__( self			:   List[Any]       ,						lowerCamelCase_			:   List[Any]       ,						lowerCamelCase_			:   Any=True       ,						lowerCamelCase_			:   str=True       ,						lowerCamelCase_			:   str=False       ,						lowerCamelCase_			:   Optional[int]="[CLS]"       ,						lowerCamelCase_			:   str="[SEP]"       ,						lowerCamelCase_			:   Optional[int]="<unk>"       ,						lowerCamelCase_			:   str="[SEP]"       ,						lowerCamelCase_			:   Optional[int]="<pad>"       ,						lowerCamelCase_			:   Optional[Any]="[CLS]"       ,						lowerCamelCase_			:   List[str]="[MASK]"       ,						lowerCamelCase_			:   Optional[Dict[str, Any]] = None       ,						**lowerCamelCase_			:   List[Any]       ,						)						->   Union[str, Any]:
        __magic_name__							:	Dict    							=							(
            AddedToken(UpperCamelCase__       ,						lstrip=UpperCamelCase__       ,						rstrip=UpperCamelCase__       ,						normalized=UpperCamelCase__  )
            if isinstance(UpperCamelCase__       ,						UpperCamelCase__  )
            else mask_token
        )

        __magic_name__							:	str    							=							{} if sp_model_kwargs is None else sp_model_kwargs

        super().__init__(
            do_lower_case=UpperCamelCase__       ,						remove_space=UpperCamelCase__       ,						keep_accents=UpperCamelCase__       ,						bos_token=UpperCamelCase__       ,						eos_token=UpperCamelCase__       ,						unk_token=UpperCamelCase__       ,						sep_token=UpperCamelCase__       ,						pad_token=UpperCamelCase__       ,						cls_token=UpperCamelCase__       ,						mask_token=UpperCamelCase__       ,						sp_model_kwargs=self.sp_model_kwargs       ,						**UpperCamelCase__       ,						)

        __magic_name__							:	str    							=							do_lower_case
        __magic_name__							:	List[str]    							=							remove_space
        __magic_name__							:	Tuple    							=							keep_accents
        __magic_name__							:	Tuple    							=							vocab_file

        __magic_name__							:	List[str]    							=							spm.SentencePieceProcessor(**self.sp_model_kwargs  )
        self.sp_model.Load(UpperCamelCase__  )

   @property
   def 							UpperCAmelCase__       ( self			:   Optional[Any]  )						->   Optional[Any]:
        return len(self.sp_model  )

   def 							UpperCAmelCase__       ( self			:   Tuple  )						->   Tuple:
        __magic_name__							:	List[str]    							=							{self.convert_ids_to_tokens(UpperCamelCase__  ): i for i in range(self.vocab_size  )}
        vocab.update(self.added_tokens_encoder  )
        return vocab

   def __getstate__( self			:   Optional[Any]  )						->   int:
        __magic_name__							:	Dict    							=							self.__dict__.copy()
        __magic_name__							:	Tuple    							=							None
        return state

   def __setstate__( self			:   List[Any]       ,						lowerCamelCase_			:   List[str]  )						->   int:
        __magic_name__							:	str    							=							d

        # for backward compatibility
        if not hasattr(self       ,						'''sp_model_kwargs'''  ):
             __magic_name__							:	str    							=							{}

        __magic_name__							:	Union[str, Any]    							=							spm.SentencePieceProcessor(**self.sp_model_kwargs  )
        self.sp_model.Load(self.vocab_file  )

   def 							UpperCAmelCase__       ( self			:   Dict       ,						lowerCamelCase_			:   Optional[Any]  )						->   List[Any]:





        if self.remove_space:
             __magic_name__							:	str    							=							''' '''.join(inputs.strip().split()  )
        else:
             __magic_name__							:	Optional[int]    							=							inputs
        __magic_name__							:	List[str]    							=							outputs.replace('''``'''       ,						'''"'''  ).replace('''\'\''''       ,						'''"'''  )

        if not self.keep_accents:
             __magic_name__							:	Any    							=							unicodedata.normalize('''NFKD'''       ,						UpperCamelCase__  )
             __magic_name__							:	int    							=							''''''.join([c for c in outputs if not unicodedata.combining(UpperCamelCase__  )]  )
        if self.do_lower_case:
             __magic_name__							:	Union[str, Any]    							=							outputs.lower()

        return outputs

   def 							UpperCAmelCase__       ( self			:   Any       ,						lowerCamelCase_			:   str  )						->   Optional[int]:
        __magic_name__							:	Optional[Any]    							=							self.preprocess_text(UpperCamelCase__  )
        __magic_name__							:	Union[str, Any]    							=							self.sp_model.encode(UpperCamelCase__       ,						out_type=UpperCamelCase__  )
        __magic_name__							:	str    							=							[]
        for piece in pieces:
             if len(UpperCamelCase__  ) > 1 and piece[-1] == str(''','''  ) and piece[-2].isdigit():
                  __magic_name__							:	Any    							=							self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCamelCase__       ,						''''''  )  )
                  if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                       if len(cur_pieces[0]  ) == 1:
                            __magic_name__							:	Tuple    							=							cur_pieces[1:]
                       else:
                            __magic_name__							:	List[Any]    							=							cur_pieces[0][1:]
                  cur_pieces.append(piece[-1]  )
                  new_pieces.extend(UpperCamelCase__  )
             else:
                  new_pieces.append(UpperCamelCase__  )

        return new_pieces

   def 							UpperCAmelCase__       ( self			:   Dict       ,						lowerCamelCase_			:   Optional[Any]  )						->   List[str]:
        return self.sp_model.PieceToId(UpperCamelCase__  )

   def 							UpperCAmelCase__       ( self			:   Union[str, Any]       ,						lowerCamelCase_			:   Tuple  )						->   Optional[int]:
        return self.sp_model.IdToPiece(UpperCamelCase__  )

   def 							UpperCAmelCase__       ( self			:   List[Any]       ,						lowerCamelCase_			:   List[Any]  )						->   Union[str, Any]:
        __magic_name__							:	List[Any]    							=							[]
        __magic_name__							:	List[str]    							=							''''''
        __magic_name__							:	str    							=							False
        for token in tokens:
             # make sure that special tokens are not decoded using sentencepiece model
             if token in self.all_special_tokens:
                  if not prev_is_special:
                       out_string += " "
                  out_string += self.sp_model.decode(UpperCamelCase__  ) + token
                  __magic_name__							:	Union[str, Any]    							=							True
                  __magic_name__							:	List[str]    							=							[]
             else:
                  current_sub_tokens.append(UpperCamelCase__  )
                  __magic_name__							:	Tuple    							=							False
        out_string += self.sp_model.decode(UpperCamelCase__  )
        return out_string.strip()

   def 							UpperCAmelCase__       ( self			:   Union[str, Any]       ,						lowerCamelCase_			:   List[int]       ,						lowerCamelCase_			:   Optional[List[int]] = None  )						->   Tuple:
        __magic_name__							:	Optional[int]    							=							[self.sep_token_id]
        __magic_name__							:	int    							=							[self.cls_token_id]
        if token_ids_a is None:
             return cls + token_ids_a + sep
        return cls + token_ids_a + sep + token_ids_a + sep

   def 							UpperCAmelCase__       ( self			:   Dict       ,						lowerCamelCase_			:   List[int]       ,						lowerCamelCase_			:   Optional[List[int]] = None       ,						lowerCamelCase_			:   bool = False  )						->   Optional[int]:





        if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=UpperCamelCase__       ,						token_ids_a=UpperCamelCase__       ,						already_has_special_tokens=UpperCamelCase__  )

        if token_ids_a is not None:
             return [1] + ([0] * len(UpperCamelCase__  )) + [1] + ([0] * len(UpperCamelCase__  )) + [1]
        return [1] + ([0] * len(UpperCamelCase__  )) + [1]

   def 							UpperCAmelCase__       ( self			:   Tuple       ,						lowerCamelCase_			:   List[int]       ,						lowerCamelCase_			:   Optional[List[int]] = None  )						->   Dict:
        __magic_name__							:	List[str]    							=							[self.sep_token_id]
        __magic_name__							:	Optional[Any]    							=							[self.cls_token_id]

        if token_ids_a is None:
             return len(cls + token_ids_a + sep  ) * [0]
        return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]


   def 							UpperCAmelCase__       ( self			:   Dict       ,						lowerCamelCase_			:   str       ,						lowerCamelCase_			:   Optional[str] = None  )						->   Optional[int]:





        if not os.path.isdir(UpperCamelCase__  ):
             logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''  )
             return
        __magic_name__							:	Dict    							=							os.path.join(
            UpperCamelCase__       ,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']  )

        if os.path.abspath(self.vocab_file  ) != os.path.abspath(UpperCamelCase__  ) and os.path.isfile(self.vocab_file  ):
             copyfile(self.vocab_file       ,						UpperCamelCase__  )
        elif not os.path.isfile(self.vocab_file  ):
             with open(UpperCamelCase__       ,						'''wb'''  ) as fi:
                  __magic_name__							:	int    							=							self.sp_model.serialized_model_proto()
                  fi.write(UpperCamelCase__  )

        return (out_vocab_file,)









import inspect
import os
import unittest
from pathlib import Path

import torch

import accelerate
from accelerate.test_utils import execute_subprocess_async
from accelerate.test_utils.testing import run_command


class    lowerCamelCase    (		unittest.TestCase						):






			'''simple docstring'''





			UpperCamelCase__								=inspect.getfile(accelerate.test_utils						)
			UpperCamelCase__								=os.path.sep.join(mod_file.split(os.path.sep						)[:-1] + ['''scripts''', '''test_cli.py''']						)

			UpperCamelCase__								=['''accelerate''', '''launch''']
			UpperCamelCase__								=Path.home() / '''.cache/huggingface/accelerate'''
			UpperCamelCase__								='''default_config.yaml'''
			UpperCamelCase__								=config_folder / config_file
			UpperCamelCase__								=config_folder / '''_default_config.yaml'''

			UpperCamelCase__								=Path('''tests/test_configs'''						)

			@classmethod
			def 							UpperCAmelCase__       ( cls			:   Optional[int]  )						->   int:
								if cls.config_path.is_file():
													cls.config_path.rename(cls.changed_path  )

			@classmethod
			def 							UpperCAmelCase__       ( cls			:   Dict  )						->   List[str]:
								if cls.changed_path.is_file():
													cls.changed_path.rename(cls.config_path  )

			def 							UpperCAmelCase__       ( self			:   str  )						->   Optional[Any]:
								__magic_name__							:	Dict    							=							self.base_cmd
								if torch.cuda.is_available() and (torch.cuda.device_count() > 1):
													cmd += ["--multi_gpu"]
								execute_subprocess_async(cmd + [self.test_file_path]       ,						env=os.environ.copy()  )

			def 							UpperCAmelCase__       ( self			:   int  )						->   Optional[int]:
								for config in sorted(self.test_config_path.glob('''**/*.yaml'''  )  ):
													with self.subTest(config_file=lowerCamelCase_  ):
																		execute_subprocess_async(
																		    self.base_cmd + ['''--config_file''', str(lowerCamelCase_  ), self.test_file_path]       ,						env=os.environ.copy()  )


			def 							UpperCAmelCase__       ( self			:   int  )						->   Union[str, Any]:
								execute_subprocess_async(['''accelerate''', '''test''']       ,						env=os.environ.copy()  )






class    lowerCamelCase    (		unittest.TestCase						):






			'''simple docstring'''





			UpperCamelCase__								='''test-tpu'''
			UpperCamelCase__								='''us-central1-a'''
			UpperCamelCase__								='''ls'''
			UpperCamelCase__								=['''accelerate''', '''tpu-config''']
			UpperCamelCase__								='''cd /usr/share'''
			UpperCamelCase__								='''tests/test_samples/test_command_file.sh'''
			UpperCamelCase__								='''Running gcloud compute tpus tpu-vm ssh'''

			def 							UpperCAmelCase__       ( self			:   Optional[Any]  )						->   Optional[Any]:
								__magic_name__							:	str    							=							run_command(
								    self.cmd
								    + ['''--command''', self.command, '''--tpu_zone''', self.tpu_zone, '''--tpu_name''', self.tpu_name, '''--debug''']       ,						return_stdout=lowerCamelCase_       ,						)
								self.assertIn(
								    F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all'''       ,						lowerCamelCase_       ,						)

			def 							UpperCAmelCase__       ( self			:   Union[str, Any]  )						->   int:
								__magic_name__							:	int    							=							run_command(
								    self.cmd
								    + [
								        '''--config_file''',
								        '''tests/test_configs/0_12_0.yaml''',
								        '''--command''',
								        self.command,
								        '''--tpu_zone''',
								        self.tpu_zone,
								        '''--tpu_name''',
								        self.tpu_name,
								        '''--debug''',
								    ]       ,						return_stdout=lowerCamelCase_       ,						)
								self.assertIn(
								    F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all'''       ,						lowerCamelCase_       ,						)

			def 							UpperCAmelCase__       ( self			:   List[Any]  )						->   Tuple:
								__magic_name__							:	Any    							=							run_command(
								    self.cmd + ['''--config_file''', '''tests/test_configs/latest.yaml''', '''--debug''']       ,						return_stdout=lowerCamelCase_  )
								self.assertIn(
								    F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo "hello world"; echo "this is a second command" --worker all'''       ,						lowerCamelCase_       ,						)

			def 							UpperCAmelCase__       ( self			:   Optional[int]  )						->   Tuple:
								__magic_name__							:	Tuple    							=							run_command(
								    self.cmd + ['''--config_file''', '''tests/test_configs/latest.yaml''', '''--command''', self.command, '''--debug''']       ,						return_stdout=lowerCamelCase_       ,						)
								self.assertIn(
								    F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all'''       ,						lowerCamelCase_       ,						)

			def 							UpperCAmelCase__       ( self			:   List[Any]  )						->   Dict:
								__magic_name__							:	str    							=							run_command(
								    self.cmd
								    + [
								        '''--config_file''',
								        '''tests/test_configs/latest.yaml''',
								        '''--command''',
								        self.command,
								        '''--command''',
								        '''echo "Hello World"''',
								        '''--debug''',
								    ]       ,						return_stdout=lowerCamelCase_       ,						)
								self.assertIn(
								    F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls; echo "Hello World" --worker all'''       ,						lowerCamelCase_       ,						)

			def 							UpperCAmelCase__       ( self			:   Optional[Any]  )						->   Optional[int]:
								__magic_name__							:	Union[str, Any]    							=							run_command(
								    self.cmd
								    + ['''--config_file''', '''tests/test_configs/latest.yaml''', '''--command_file''', self.command_file, '''--debug''']       ,						return_stdout=lowerCamelCase_       ,						)
								self.assertIn(
								    F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo "hello world"; echo "this is a second command" --worker all'''       ,						lowerCamelCase_       ,						)

			def 							UpperCAmelCase__       ( self			:   int  )						->   List[Any]:
								__magic_name__							:	List[Any]    							=							run_command(
								    self.cmd
								    + [
								        '''--config_file''',
								        '''tests/test_configs/0_12_0.yaml''',
								        '''--command_file''',
								        self.command_file,
								        '''--tpu_zone''',
								        self.tpu_zone,
								        '''--tpu_name''',
								        self.tpu_name,
								        '''--debug''',
								    ]       ,						return_stdout=lowerCamelCase_       ,						)
								self.assertIn(
								    F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo "hello world"; echo "this is a second command" --worker all'''       ,						lowerCamelCase_       ,						)

			def 							UpperCAmelCase__       ( self			:   str  )						->   Dict:
								__magic_name__							:	Tuple    							=							run_command(
								    self.cmd + ['''--config_file''', '''tests/test_configs/latest.yaml''', '''--install_accelerate''', '''--debug''']       ,						return_stdout=lowerCamelCase_       ,						)
								self.assertIn(
								    F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; pip install accelerate -U; echo "hello world"; echo "this is a second command" --worker all'''       ,						lowerCamelCase_       ,						)


			def 							UpperCAmelCase__       ( self			:   Any  )						->   Tuple:
								__magic_name__							:	Tuple    							=							run_command(
								    self.cmd
								    + [
								        '''--config_file''',
								        '''tests/test_configs/latest.yaml''',
								        '''--install_accelerate''',
								        '''--accelerate_version''',
								        '''12.0.0''',
								        '''--debug''',
								    ]       ,						return_stdout=lowerCamelCase_       ,						)
								self.assertIn(
								    F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; pip install accelerate==12.0.0; echo "hello world"; echo "this is a second command" --worker all'''       ,						lowerCamelCase_       ,						)



