'''simple docstring'''
from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase__  :      List[str]        =					logging.get_logger(__name__)

lowercase__  :      Optional[int]        =					{
    "s-JoL/Open-Llama-V1": "https://huggingface.co/s-JoL/Open-Llama-V1/blob/main/config.json",
}







class lowerCamelCase							(	lowerCamelCase					):




  '''simple docstring'''



  lowerCAmelCase__										=	'''open-llama'''






  def __init__(      self			:       List[Any]  ,							UpperCAmelCase__			:       Any=10_0000  ,							UpperCAmelCase__			:       Union[str, Any]=4096  ,							UpperCAmelCase__			:       int=1_1008  ,							UpperCAmelCase__			:       Any=32  ,							UpperCAmelCase__			:       int=32  ,							UpperCAmelCase__			:       Union[str, Any]="silu"  ,							UpperCAmelCase__			:       List[str]=2048  ,							UpperCAmelCase__			:       List[Any]=0.02  ,							UpperCAmelCase__			:       Dict=1e-6  ,							UpperCAmelCase__			:       int=True  ,							UpperCAmelCase__			:       Tuple=0  ,							UpperCAmelCase__			:       Tuple=1  ,							UpperCAmelCase__			:       Tuple=2  ,							UpperCAmelCase__			:       List[str]=False  ,							UpperCAmelCase__			:       Optional[Any]=True  ,							UpperCAmelCase__			:       int=0.1  ,							UpperCAmelCase__			:       str=0.1  ,							UpperCAmelCase__			:       Optional[int]=True  ,							UpperCAmelCase__			:       Optional[int]=True  ,							UpperCAmelCase__			:       int=None  ,							**UpperCAmelCase__			:       str  ,							)				->int:
        UpperCAmelCase_            =  vocab_size
        UpperCAmelCase_            =  max_position_embeddings
        UpperCAmelCase_            =  hidden_size
        UpperCAmelCase_            =  intermediate_size
        UpperCAmelCase_            =  num_hidden_layers
        UpperCAmelCase_            =  num_attention_heads
        UpperCAmelCase_            =  hidden_act
        UpperCAmelCase_            =  initializer_range
        UpperCAmelCase_            =  rms_norm_eps
        UpperCAmelCase_            =  use_cache
        UpperCAmelCase_            =  kwargs.pop(
            '''use_memorry_efficient_attention'''  ,							UpperCAmelCase__  )
        UpperCAmelCase_            =  hidden_dropout_prob
        UpperCAmelCase_            =  attention_dropout_prob
        UpperCAmelCase_            =  use_stable_embedding
        UpperCAmelCase_            =  shared_input_output_embedding
        UpperCAmelCase_            =  rope_scaling
        self._rope_scaling_validation()

        super().__init__(
            pad_token_id=UpperCAmelCase__  ,							bos_token_id=UpperCAmelCase__  ,							eos_token_id=UpperCAmelCase__  ,							tie_word_embeddings=UpperCAmelCase__  ,							**UpperCAmelCase__  ,							)






  def  lowerCAmelCase__ (      self			:       str  )				->Tuple:
        if self.rope_scaling is None:
              return

        if not isinstance(self.rope_scaling  ,							UpperCAmelCase__  ) or len(self.rope_scaling  ) != 2:
              raise ValueError(
                  '''`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '''
                  f"""got {self.rope_scaling}"""  )
        UpperCAmelCase_            =  self.rope_scaling.get('''type'''  ,							UpperCAmelCase__  )
        UpperCAmelCase_            =  self.rope_scaling.get('''factor'''  ,							UpperCAmelCase__  )
        if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
              raise ValueError(
                  f"""`rope_scaling`'s name field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"""  )
        if rope_scaling_factor is None or not isinstance(UpperCAmelCase__  ,							UpperCAmelCase__  ) or rope_scaling_factor <= 1.0:
              raise ValueError(f"""`rope_scaling`'s factor field must be an float > 1, got {rope_scaling_factor}"""  )






'''simple docstring'''
import json
import os
import unittest

from transformers.models.xlm.tokenization_xlm import VOCAB_FILES_NAMES, XLMTokenizer
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin







class lowerCamelCase							(	lowerCamelCase     ,		unittest.TestCase					):




  '''simple docstring'''



  lowerCAmelCase__										=	XLMTokenizer
  lowerCAmelCase__										=	False






  def  lowerCAmelCase__ (      self			:       int  )				->Union[str, Any]:
        super().setUp()

        # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
        UpperCAmelCase_            =  [
            '''l''',
            '''o''',
            '''w''',
            '''e''',
            '''r''',
            '''s''',
            '''t''',
            '''i''',
            '''d''',
            '''n''',
            '''w</w>''',
            '''r</w>''',
            '''t</w>''',
            '''lo''',
            '''low''',
            '''er</w>''',
            '''low</w>''',
            '''lowest</w>''',
            '''newer</w>''',
            '''wider</w>''',
            '''<unk>''',
        ]
        UpperCAmelCase_            =  dict(zip(UpperCAmelCase__  ,							range(len(UpperCAmelCase__  )  )  )  )
        UpperCAmelCase_            =  ['''l o 123''', '''lo w 1456''', '''e r</w> 1789''', '''''']

        UpperCAmelCase_            =  os.path.join(self.tmpdirname  ,							VOCAB_FILES_NAMES['''vocab_file''']  )
        UpperCAmelCase_            =  os.path.join(self.tmpdirname  ,							VOCAB_FILES_NAMES['''merges_file''']  )
        with open(self.vocab_file  ,							'''w'''  ) as fp:
              fp.write(json.dumps(UpperCAmelCase__  )  )
        with open(self.merges_file  ,							'''w'''  ) as fp:
              fp.write('''\n'''.join(UpperCAmelCase__  )  )






  def  lowerCAmelCase__ (      self			:       Optional[int]  ,							UpperCAmelCase__			:       Any  )				->List[Any]:
        UpperCAmelCase_            =  '''lower newer'''
        UpperCAmelCase_            =  '''lower newer'''
        return input_text, output_text






  def  lowerCAmelCase__ (      self			:       Union[str, Any]  )				->Tuple:
        UpperCAmelCase_            =  XLMTokenizer(self.vocab_file  ,							self.merges_file  )

        UpperCAmelCase_            =  '''lower'''
        UpperCAmelCase_            =  ['''low''', '''er</w>''']
        UpperCAmelCase_            =  tokenizer.tokenize(UpperCAmelCase__  )
        self.assertListEqual(UpperCAmelCase__  ,							UpperCAmelCase__  )

        UpperCAmelCase_            =  tokens + ['''<unk>''']
        UpperCAmelCase_            =  [14, 15, 20]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase__  )  ,							UpperCAmelCase__  )






  @slow
  def  lowerCAmelCase__ (      self			:       Any  )				->str:
        UpperCAmelCase_            =  XLMTokenizer.from_pretrained('''xlm-mlm-en-2048'''  )

        UpperCAmelCase_            =  tokenizer.encode('''sequence builders'''  ,							add_special_tokens=UpperCAmelCase__  )
        UpperCAmelCase_            =  tokenizer.encode('''multi-sequence build'''  ,							add_special_tokens=UpperCAmelCase__  )

        UpperCAmelCase_            =  tokenizer.build_inputs_with_special_tokens(UpperCAmelCase__  )
        UpperCAmelCase_            =  tokenizer.build_inputs_with_special_tokens(UpperCAmelCase__  ,							UpperCAmelCase__  )

        assert encoded_sentence == [0] + text + [1]
        assert encoded_pair == [0] + text + [1] + text_a + [1]





