

"""simple docstring"""






import os
import re
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_lowerCAmelCase						=		logging.get_logger(__name__)

_lowerCAmelCase						=		{
    """vocab_file""": """vocab.txt""",
    """merges_file""": """bpe.codes""",
}

_lowerCAmelCase						=		{
    """vocab_file""": {
        """vinai/phobert-base""": """https://huggingface.co/vinai/phobert-base/resolve/main/vocab.txt""",
        """vinai/phobert-large""": """https://huggingface.co/vinai/phobert-large/resolve/main/vocab.txt""",
    },
    """merges_file""": {
        """vinai/phobert-base""": """https://huggingface.co/vinai/phobert-base/resolve/main/bpe.codes""",
        """vinai/phobert-large""": """https://huggingface.co/vinai/phobert-large/resolve/main/bpe.codes""",
    },
}

_lowerCAmelCase						=		{
    """vinai/phobert-base""": 2_5_6,
    """vinai/phobert-large""": 2_5_6,
}

def 		lowerCamelCase__		(					_lowerCamelCase   ):

    '''simple docstring'''



    _lowerCAmelCase       :   List[str]  =    set()
    _lowerCAmelCase       :   List[Any]  =    word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)   )
        _lowerCAmelCase       :   Optional[int]  =    char

    _lowerCAmelCase       :   Any  =    set(_lowerCamelCase   )
    return pairs







class 	__UpperCamelCase    (    a__						):
  _UpperCAmelCase          =		VOCAB_FILES_NAMES
  _UpperCAmelCase          =		PRETRAINED_VOCAB_FILES_MAP
  _UpperCAmelCase          =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES

  def __init__(		self    ,_A    ,_A    ,_A="<s>"    ,_A="</s>"    ,_A="</s>"    ,_A="<s>"    ,_A="<unk>"    ,_A="<pad>"    ,_A="<mask>"    ,**_A    ,):
      '''simple docstring'''

      super().__init__(
          bos_token=_A    ,eos_token=_A    ,unk_token=_A    ,sep_token=_A    ,cls_token=_A    ,pad_token=_A    ,mask_token=_A    ,**_A    ,)

      _lowerCAmelCase       :   List[Any]  =    vocab_file
      _lowerCAmelCase       :   Dict  =    merges_file

      _lowerCAmelCase       :   Optional[Any]  =    {}
      _lowerCAmelCase       :   Tuple  =    0
      _lowerCAmelCase       :   Tuple  =    1
      _lowerCAmelCase       :   List[Any]  =    2
      _lowerCAmelCase       :   Any  =    3

      self.add_from_file(_A      )

      _lowerCAmelCase       :   Union[str, Any]  =    {v: k for k, v in self.encoder.items()}

      with open(_A    ,encoding='utf-8'      ) as merges_handle:
          _lowerCAmelCase       :   Tuple  =    merges_handle.read().split('\n'      )[:-1]
      _lowerCAmelCase       :   Optional[Any]  =    [tuple(merge.split()[:-1]      ) for merge in merges]
      _lowerCAmelCase       :   Union[str, Any]  =    dict(zip(_A    ,range(len(_A      )      )      )      )
      _lowerCAmelCase       :   Union[str, Any]  =    {}

  def __lowerCamelCase		(		self    ,_A    ,_A = None      ):
      '''simple docstring'''

      if token_ids_a is None:
          return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
      _lowerCAmelCase       :   Tuple  =    [self.cls_token_id]
      _lowerCAmelCase       :   Tuple  =    [self.sep_token_id]
      return cls + token_ids_a + sep + sep + token_ids_a + sep

  def __lowerCamelCase		(		self    ,_A    ,_A = None    ,_A = False      ):
      '''simple docstring'''

      if already_has_special_tokens:
          return super().get_special_tokens_mask(
              token_ids_a=_A    ,token_ids_a=_A    ,already_has_special_tokens=_A      )

      if token_ids_a is None:
          return [1] + ([0] * len(_A      )) + [1]
      return [1] + ([0] * len(_A      )) + [1, 1] + ([0] * len(_A      )) + [1]

  def __lowerCamelCase		(		self    ,_A    ,_A = None      ):
      '''simple docstring'''

      _lowerCAmelCase       :   Tuple  =    [self.sep_token_id]
      _lowerCAmelCase       :   Any  =    [self.cls_token_id]

      if token_ids_a is None:
          return len(cls + token_ids_a + sep      ) * [0]
      return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]

  @property
  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      return len(self.encoder      )

  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      return dict(self.encoder    ,**self.added_tokens_encoder      )

  def __lowerCamelCase		(		self    ,_A      ):
      '''simple docstring'''

      if token in self.cache:
          return self.cache[token]
      _lowerCAmelCase       :   str  =    tuple(_A      )
      _lowerCAmelCase       :   int  =    tuple(list(word[:-1]      ) + [word[-1] + '</w>']      )
      _lowerCAmelCase       :   Dict  =    get_pairs(_A      )

      if not pairs:
          return token

      while True:
          _lowerCAmelCase       :   str  =    min(_A    ,key=lambda _A      : self.bpe_ranks.get(_A    ,float('inf'      )      )      )
          if bigram not in self.bpe_ranks:
              break
          _lowerCAmelCase,					_lowerCAmelCase       :   int  =    bigram
          _lowerCAmelCase       :   int  =    []
          _lowerCAmelCase       :   List[Any]  =    0
          while i < len(_A      ):
              try:
                  _lowerCAmelCase       :   int  =    word.index(_A    ,_A      )
              except ValueError:
                  new_word.extend(word[i:]      )
                  break
              else:
                  new_word.extend(word[i:j]      )
                  _lowerCAmelCase       :   Optional[int]  =    j

              if word[i] == first and i < len(_A      ) - 1 and word[i + 1] == second:
                  new_word.append(first + second      )
                  i += 2
              else:
                  new_word.append(word[i]      )
                  i += 1
          _lowerCAmelCase       :   int  =    tuple(_A      )
          _lowerCAmelCase       :   Optional[int]  =    new_word
          if len(_A      ) == 1:
              break
          else:
              _lowerCAmelCase       :   List[str]  =    get_pairs(_A      )
      _lowerCAmelCase       :   str  =    '@@ '.join(_A      )
      _lowerCAmelCase       :   Union[str, Any]  =    word[:-4]
      _lowerCAmelCase       :   Dict  =    word
      return word

  def __lowerCamelCase		(		self    ,_A      ):
      '''simple docstring'''

      _lowerCAmelCase       :   List[Any]  =    []

      _lowerCAmelCase       :   List[str]  =    re.findall(r'\S+\n?'    ,_A      )

      for token in words:
          split_tokens.extend(list(self.bpe(_A      ).split(' '      )      )      )
      return split_tokens

  def __lowerCamelCase		(		self    ,_A      ):
      '''simple docstring'''

      return self.encoder.get(_A    ,self.encoder.get(self.unk_token      )      )

  def __lowerCamelCase		(		self    ,_A      ):
      '''simple docstring'''

      return self.decoder.get(_A    ,self.unk_token      )

  def __lowerCamelCase		(		self    ,_A      ):
      '''simple docstring'''

      _lowerCAmelCase       :   List[str]  =    ' '.join(_A      ).replace('@@ '    ,''      ).strip()
      return out_string

  def __lowerCamelCase		(		self    ,_A    ,_A = None      ):
      '''simple docstring'''

      if not os.path.isdir(_A      ):
          logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""      )
          return
      _lowerCAmelCase       :   Any  =    os.path.join(
          _A    ,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']      )
      _lowerCAmelCase       :   List[Any]  =    os.path.join(
          _A    ,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']      )

      if os.path.abspath(self.vocab_file      ) != os.path.abspath(_A      ):
          copyfile(self.vocab_file    ,_A      )

      if os.path.abspath(self.merges_file      ) != os.path.abspath(_A      ):
          copyfile(self.merges_file    ,_A      )

      return out_vocab_file, out_merge_file







  def __lowerCamelCase		(		self    ,_A      ):
      '''simple docstring'''

      if isinstance(_A    ,_A      ):
          try:
              with open(_A    ,'r'    ,encoding='utf-8'      ) as fd:
                  self.add_from_file(_A      )
          except FileNotFoundError as fnfe:
              raise fnfe
          except UnicodeError:
              raise Exception(F"""Incorrect encoding detected in {f}, please rebuild the dataset"""      )
          return

      _lowerCAmelCase       :   List[str]  =    f.readlines()
      for lineTmp in lines:
          _lowerCAmelCase       :   List[str]  =    lineTmp.strip()
          _lowerCAmelCase       :   Optional[Any]  =    line.rfind(' '      )
          if idx == -1:
              raise ValueError('Incorrect dictionary format, expected \'<token> <cnt>\''      )
          _lowerCAmelCase       :   Optional[Any]  =    line[:idx]
          _lowerCAmelCase       :   Optional[Any]  =    len(self.encoder      )








"""simple docstring"""






import inspect
import os
import unittest

import torch

import accelerate
from accelerate import Accelerator
from accelerate.test_utils import execute_subprocess_async, require_multi_gpu
from accelerate.utils import patch_environment







class 	__UpperCamelCase    (    unittest.TestCase						):

  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      _lowerCAmelCase       :   Dict  =    inspect.getfile(accelerate.test_utils      )
      _lowerCAmelCase       :   Union[str, Any]  =    os.path.sep.join(mod_file.split(os.path.sep      )[:-1] + ['scripts', 'test_script.py']      )
      _lowerCAmelCase       :   int  =    os.path.sep.join(
          mod_file.split(os.path.sep      )[:-1] + ['scripts', 'test_distributed_data_loop.py']      )
      _lowerCAmelCase       :   Optional[Any]  =    os.path.sep.join(mod_file.split(os.path.sep      )[:-1] + ['scripts', 'test_ops.py']      )

  @require_multi_gpu
  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      print(F"""Found {torch.cuda.device_count()} devices."""      )
      _lowerCAmelCase       :   int  =    ['torchrun', F"""--nproc_per_node={torch.cuda.device_count()}""", self.test_file_path]
      with patch_environment(omp_num_threads=1      ):
          execute_subprocess_async(_A    ,env=os.environ.copy()      )

  @require_multi_gpu
  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      print(F"""Found {torch.cuda.device_count()} devices."""      )
      _lowerCAmelCase       :   str  =    ['torchrun', F"""--nproc_per_node={torch.cuda.device_count()}""", self.operation_file_path]
      print(F"""Command: {cmd}"""      )
      with patch_environment(omp_num_threads=1      ):
          execute_subprocess_async(_A    ,env=os.environ.copy()      )

  @require_multi_gpu
  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      _lowerCAmelCase       :   List[Any]  =    ['torchrun', F"""--nproc_per_node={torch.cuda.device_count()}""", inspect.getfile(self.__class__      )]
      with patch_environment(omp_num_threads=1      ):
          execute_subprocess_async(_A    ,env=os.environ.copy()      )







  @require_multi_gpu
  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      print(F"""Found {torch.cuda.device_count()} devices, using 2 devices only"""      )
      _lowerCAmelCase       :   Tuple  =    ['torchrun', F"""--nproc_per_node={torch.cuda.device_count()}""", self.data_loop_file_path]
      with patch_environment(omp_num_threads=1    ,cuda_visible_devices='0,1'      ):
          execute_subprocess_async(_A    ,env=os.environ.copy()      )


if __name__ == "__main__":
 _lowerCAmelCase						=		Accelerator()
 _lowerCAmelCase						=		(accelerator.state.process_index + 2, 1_0)
 _lowerCAmelCase						=		torch.randint(0, 1_0, shape).to(accelerator.device)

 _lowerCAmelCase						=		""""""

 _lowerCAmelCase						=		accelerator.pad_across_processes(tensor)
 if tensora.shape[0] != accelerator.state.num_processes + 1:
  error_msg += F"Found shape {tensora.shape} but should have {accelerator.state.num_processes + 1} at dim 0."
 if not torch.equal(tensora[: accelerator.state.process_index + 2], tensor):
  error_msg += "Tensors have different values."
 if not torch.all(tensora[accelerator.state.process_index + 2 :] == 0):
  error_msg += "Padding was not done with the right value (0)."

 _lowerCAmelCase						=		accelerator.pad_across_processes(tensor, pad_first=True)
 if tensora.shape[0] != accelerator.state.num_processes + 1:
  error_msg += F"Found shape {tensora.shape} but should have {accelerator.state.num_processes + 1} at dim 0."
 _lowerCAmelCase						=		accelerator.state.num_processes - accelerator.state.process_index - 1
 if not torch.equal(tensora[index:], tensor):
  error_msg += "Tensors have different values."
 if not torch.all(tensora[:index] == 0):
  error_msg += "Padding was not done with the right value (0)."

 # Raise error at the end to make sure we don't stop at the first failure.
 if len(error_msg) > 0:
  raise ValueError(error_msg)





