




import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__lowerCamelCase   :						Optional[Any]				=			logging.get_logger(__name__)

__lowerCamelCase   :						Optional[int]				=			{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

__lowerCamelCase   :						Union[str, Any]				=			{
    '''vocab_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json'''},
    '''merges_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt'''},
}

__lowerCamelCase   :						Optional[int]				=			{
    '''ctrl''': 2_56,
}

__lowerCamelCase   :						int				=			{
    '''Pregnancy''': 16_86_29,
    '''Christianity''': 76_75,
    '''Explain''': 10_64_23,
    '''Fitness''': 6_34_40,
    '''Saving''': 6_31_63,
    '''Ask''': 2_71_71,
    '''Ass''': 9_59_85,
    '''Joke''': 16_35_09,
    '''Questions''': 4_56_22,
    '''Thoughts''': 4_96_05,
    '''Retail''': 5_23_42,
    '''Feminism''': 16_43_38,
    '''Writing''': 1_19_92,
    '''Atheism''': 19_22_63,
    '''Netflix''': 4_86_16,
    '''Computing''': 3_96_39,
    '''Opinion''': 4_32_13,
    '''Alone''': 4_49_67,
    '''Funny''': 5_89_17,
    '''Gaming''': 4_03_58,
    '''Human''': 40_88,
    '''India''': 13_31,
    '''Joker''': 7_71_38,
    '''Diet''': 3_62_06,
    '''Legal''': 1_18_59,
    '''Norman''': 49_39,
    '''Tip''': 7_26_89,
    '''Weight''': 5_23_43,
    '''Movies''': 4_62_73,
    '''Running''': 2_34_25,
    '''Science''': 20_90,
    '''Horror''': 3_77_93,
    '''Confession''': 6_05_72,
    '''Finance''': 1_22_50,
    '''Politics''': 1_63_60,
    '''Scary''': 19_19_85,
    '''Support''': 1_26_54,
    '''Technologies''': 3_25_16,
    '''Teenage''': 6_61_60,
    '''Event''': 3_27_69,
    '''Learned''': 6_74_60,
    '''Notion''': 18_27_70,
    '''Wikipedia''': 3_75_83,
    '''Books''': 66_65,
    '''Extract''': 7_60_50,
    '''Confessions''': 10_27_01,
    '''Conspiracy''': 7_59_32,
    '''Links''': 6_36_74,
    '''Narcissus''': 15_04_25,
    '''Relationship''': 5_47_66,
    '''Relationships''': 13_47_96,
    '''Reviews''': 4_16_71,
    '''News''': 42_56,
    '''Translation''': 2_68_20,
    '''multilingual''': 12_84_06,
}







def 		lowercase__   (		__A:	Optional[Any]				):





     '''simple docstring'''






     __magic_name__							:	Optional[int]    							=							set()
     __magic_name__							:	Tuple    							=							word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)				)
          __magic_name__							:	List[Any]    							=							char

     __magic_name__							:	Optional[Any]    							=							set(__A				)
     return pairs


class    lowerCamelCase    (		lowercase_						):






   '''simple docstring'''





   UpperCamelCase__								=VOCAB_FILES_NAMES
   UpperCamelCase__								=PRETRAINED_VOCAB_FILES_MAP
   UpperCamelCase__								=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   UpperCamelCase__								=CONTROL_CODES

   def __init__( self			:   Optional[int]       ,						lowerCamelCase_			:   List[Any]       ,						lowerCamelCase_			:   Optional[Any]       ,						lowerCamelCase_			:   Tuple="<unk>"       ,						**lowerCamelCase_			:   int  )						->   Tuple:
        super().__init__(unk_token=UpperCamelCase__       ,						**UpperCamelCase__  )

        with open(UpperCamelCase__       ,						encoding='''utf-8'''  ) as vocab_handle:
             __magic_name__							:	Dict    							=							json.load(UpperCamelCase__  )
        __magic_name__							:	Any    							=							{v: k for k, v in self.encoder.items()}
        with open(UpperCamelCase__       ,						encoding='''utf-8'''  ) as merges_handle:
             __magic_name__							:	Tuple    							=							merges_handle.read().split('''\n'''  )[1:-1]
        __magic_name__							:	int    							=							[tuple(merge.split()  ) for merge in merges]
        __magic_name__							:	Optional[Any]    							=							dict(zip(UpperCamelCase__       ,						range(len(UpperCamelCase__  )  )  )  )
        __magic_name__							:	Optional[Any]    							=							{}

   @property
   def 							UpperCAmelCase__       ( self			:   Optional[Any]  )						->   Union[str, Any]:
        return len(self.encoder  )

   def 							UpperCAmelCase__       ( self			:   List[Any]  )						->   int:
        return dict(self.encoder       ,						**self.added_tokens_encoder  )

   def 							UpperCAmelCase__       ( self			:   Tuple       ,						lowerCamelCase_			:   str  )						->   Any:







        if token in self.cache:
             return self.cache[token]
        __magic_name__							:	Optional[Any]    							=							tuple(UpperCamelCase__  )
        __magic_name__							:	Union[str, Any]    							=							tuple(list(word[:-1]  ) + [word[-1] + '''</w>''']  )
        __magic_name__							:	Any    							=							get_pairs(UpperCamelCase__  )

        if not pairs:
             return token

        while True:
             __magic_name__							:	Optional[Any]    							=							min(UpperCamelCase__       ,						key=lambda lowerCamelCase_  : self.bpe_ranks.get(UpperCamelCase__       ,						float('''inf'''  )  )  )
             if bigram not in self.bpe_ranks:
                  break
             __magic_name__							,    __magic_name__							:	Optional[Any]    							=							bigram
             __magic_name__							:	int    							=							[]
             __magic_name__							:	List[str]    							=							0
             while i < len(UpperCamelCase__  ):
                  try:
                       __magic_name__							:	Tuple    							=							word.index(UpperCamelCase__       ,						UpperCamelCase__  )
                  except ValueError:
                       new_word.extend(word[i:]  )
                       break
                  else:
                       new_word.extend(word[i:j]  )
                       __magic_name__							:	int    							=							j

                  if word[i] == first and i < len(UpperCamelCase__  ) - 1 and word[i + 1] == second:
                       new_word.append(first + second  )
                       i += 2
                  else:
                       new_word.append(word[i]  )
                       i += 1
             __magic_name__							:	Tuple    							=							tuple(UpperCamelCase__  )
             __magic_name__							:	Optional[int]    							=							new_word
             if len(UpperCamelCase__  ) == 1:
                  break
             else:
                  __magic_name__							:	str    							=							get_pairs(UpperCamelCase__  )
        __magic_name__							:	int    							=							'''@@ '''.join(UpperCamelCase__  )
        __magic_name__							:	Tuple    							=							word[:-4]
        __magic_name__							:	Union[str, Any]    							=							word
        return word

   def 							UpperCAmelCase__       ( self			:   Any       ,						lowerCamelCase_			:   Union[str, Any]  )						->   Optional[int]:
        __magic_name__							:	Union[str, Any]    							=							[]

        __magic_name__							:	Tuple    							=							re.findall(R'''\S+\n?'''       ,						UpperCamelCase__  )

        for token in words:
             split_tokens.extend(list(self.bpe(UpperCamelCase__  ).split(''' '''  )  )  )
        return split_tokens

   def 							UpperCAmelCase__       ( self			:   Dict       ,						lowerCamelCase_			:   Any  )						->   List[Any]:
        return self.encoder.get(UpperCamelCase__       ,						self.encoder.get(self.unk_token  )  )

   def 							UpperCAmelCase__       ( self			:   str       ,						lowerCamelCase_			:   Any  )						->   List[Any]:
        return self.decoder.get(UpperCamelCase__       ,						self.unk_token  )

   def 							UpperCAmelCase__       ( self			:   int       ,						lowerCamelCase_			:   int  )						->   Optional[int]:
        __magic_name__							:	Union[str, Any]    							=							''' '''.join(UpperCamelCase__  ).replace('''@@ '''       ,						''''''  ).strip()
        return out_string


   def 							UpperCAmelCase__       ( self			:   List[str]       ,						lowerCamelCase_			:   str       ,						lowerCamelCase_			:   Optional[str] = None  )						->   Dict:







        if not os.path.isdir(UpperCamelCase__  ):
             logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''  )
             return
        __magic_name__							:	str    							=							os.path.join(
            UpperCamelCase__       ,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']  )
        __magic_name__							:	str    							=							os.path.join(
            UpperCamelCase__       ,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']  )

        with open(UpperCamelCase__       ,						'''w'''       ,						encoding='''utf-8'''  ) as f:
             f.write(json.dumps(self.encoder       ,						indent=2       ,						sort_keys=UpperCamelCase__       ,						ensure_ascii=UpperCamelCase__  ) + '''\n'''  )

        __magic_name__							:	Optional[Any]    							=							0
        with open(UpperCamelCase__       ,						'''w'''       ,						encoding='''utf-8'''  ) as writer:
             writer.write('''#version: 0.2\n'''  )
             for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,						key=lambda lowerCamelCase_  : kv[1]  ):
                  if index != token_index:
                       logger.warning(
                           F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                           ''' Please check that the tokenizer is not corrupted!'''  )
                       __magic_name__							:	Any    							=							token_index
                  writer.write(''' '''.join(UpperCamelCase__  ) + '''\n'''  )
                  index += 1

        return vocab_file, merge_file

   # def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
   #     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
   #     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
   #     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
   #     return ''.join(tokens_generated_so_far)









from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowerCamelCase   :						Optional[int]				=			logging.get_logger(__name__)

__lowerCamelCase   :						Dict				=			{}


class    lowerCamelCase    (		_lowerCamelCase						):






			'''simple docstring'''





			UpperCamelCase__								='''llama'''
			UpperCamelCase__								=['''past_key_values''']

			def __init__( self			:   Dict       ,						lowerCamelCase_			:   List[str]=32000       ,						lowerCamelCase_			:   List[Any]=4096       ,						lowerCamelCase_			:   Any=11008       ,						lowerCamelCase_			:   Optional[Any]=32       ,						lowerCamelCase_			:   Dict=32       ,						lowerCamelCase_			:   Optional[Any]=None       ,						lowerCamelCase_			:   Dict="silu"       ,						lowerCamelCase_			:   Optional[int]=2048       ,						lowerCamelCase_			:   int=0.0_2       ,						lowerCamelCase_			:   Tuple=1E-6       ,						lowerCamelCase_			:   Tuple=True       ,						lowerCamelCase_			:   List[str]=0       ,						lowerCamelCase_			:   List[str]=1       ,						lowerCamelCase_			:   Optional[int]=2       ,						lowerCamelCase_			:   List[str]=1       ,						lowerCamelCase_			:   Dict=False       ,						lowerCamelCase_			:   int=None       ,						**lowerCamelCase_			:   Tuple       ,						)						->   Tuple:
								__magic_name__							:	Any    							=							vocab_size
								__magic_name__							:	Optional[int]    							=							max_position_embeddings
								__magic_name__							:	Union[str, Any]    							=							hidden_size
								__magic_name__							:	Dict    							=							intermediate_size
								__magic_name__							:	Union[str, Any]    							=							num_hidden_layers
								__magic_name__							:	Tuple    							=							num_attention_heads

								# for backward compatibility
								if num_key_value_heads is None:
													__magic_name__							:	Optional[int]    							=							num_attention_heads

								__magic_name__							:	Optional[int]    							=							num_key_value_heads
								__magic_name__							:	List[str]    							=							hidden_act
								__magic_name__							:	List[str]    							=							initializer_range
								__magic_name__							:	Tuple    							=							rms_norm_eps
								__magic_name__							:	Optional[int]    							=							pretraining_tp
								__magic_name__							:	Any    							=							use_cache
								__magic_name__							:	int    							=							rope_scaling
								self._rope_scaling_validation()

								super().__init__(
								    pad_token_id=lowerCamelCase_       ,						bos_token_id=lowerCamelCase_       ,						eos_token_id=lowerCamelCase_       ,						tie_word_embeddings=lowerCamelCase_       ,						**lowerCamelCase_       ,						)


			def 							UpperCAmelCase__       ( self			:   str  )						->   Tuple:
								if self.rope_scaling is None:
													return

								if not isinstance(self.rope_scaling       ,						lowerCamelCase_  ) or len(self.rope_scaling  ) != 2:
													raise ValueError(
													    '''`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '''
													    F'''got {self.rope_scaling}'''  )
								__magic_name__							:	List[str]    							=							self.rope_scaling.get('''type'''       ,						lowerCamelCase_  )
								__magic_name__							:	str    							=							self.rope_scaling.get('''factor'''       ,						lowerCamelCase_  )
								if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
													raise ValueError(
													    F'''`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}'''  )
								if rope_scaling_factor is None or not isinstance(lowerCamelCase_       ,						lowerCamelCase_  ) or rope_scaling_factor <= 1.0:
													raise ValueError(F'''`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}'''  )



