


import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__lowercase											=      logging.get_logger(__name__)

__lowercase											=      {
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

__lowercase											=      {
    'vocab_file': {'ctrl': 'https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json'},
    'merges_file': {'ctrl': 'https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt'},
}

__lowercase											=      {
    'ctrl': 256,
}

__lowercase											=      {
    'Pregnancy': 16_8629,
    'Christianity': 7675,
    'Explain': 10_6423,
    'Fitness': 6_3440,
    'Saving': 6_3163,
    'Ask': 2_7171,
    'Ass': 9_5985,
    'Joke': 16_3509,
    'Questions': 4_5622,
    'Thoughts': 4_9605,
    'Retail': 5_2342,
    'Feminism': 16_4338,
    'Writing': 1_1992,
    'Atheism': 19_2263,
    'Netflix': 4_8616,
    'Computing': 3_9639,
    'Opinion': 4_3213,
    'Alone': 4_4967,
    'Funny': 5_8917,
    'Gaming': 4_0358,
    'Human': 4088,
    'India': 1331,
    'Joker': 7_7138,
    'Diet': 3_6206,
    'Legal': 1_1859,
    'Norman': 4939,
    'Tip': 7_2689,
    'Weight': 5_2343,
    'Movies': 4_6273,
    'Running': 2_3425,
    'Science': 2090,
    'Horror': 3_7793,
    'Confession': 6_0572,
    'Finance': 1_2250,
    'Politics': 1_6360,
    'Scary': 19_1985,
    'Support': 1_2654,
    'Technologies': 3_2516,
    'Teenage': 6_6160,
    'Event': 3_2769,
    'Learned': 6_7460,
    'Notion': 18_2770,
    'Wikipedia': 3_7583,
    'Books': 6665,
    'Extract': 7_6050,
    'Confessions': 10_2701,
    'Conspiracy': 7_5932,
    'Links': 6_3674,
    'Narcissus': 15_0425,
    'Relationship': 5_4766,
    'Relationships': 13_4796,
    'Reviews': 4_1671,
    'News': 4256,
    'Translation': 2_6820,
    'multilingual': 12_8406,
}



def lowerCamelCase						(     SCREAMING_SNAKE_CASE			):
    '''simple docstring'''







    __UpperCamelCase  :Optional[int]    						=	set()
    __UpperCamelCase  :List[Any]    						=	word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)			)
        __UpperCamelCase  :Tuple    						=	char

    __UpperCamelCase  :Any    						=	set(A_			)
    return pairs


class 			lowerCamelCase_  (     _A				):






    '''simple docstring'''






    a__			:		Tuple      			=   VOCAB_FILES_NAMES
    a__			:		List[str]      			=   PRETRAINED_VOCAB_FILES_MAP
    a__			:		Any      			=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    a__			:		List[Any]      			=   CONTROL_CODES







    def __init__(			self				,				__lowercase				,				__lowercase				,				__lowercase="<unk>"				,				**__lowercase) ->			List[Any]:
        super().__init__(unk_token=UpperCamelCase__				,				**UpperCamelCase__)

        with open(UpperCamelCase__				,				encoding='''utf-8''') as vocab_handle:
            __UpperCamelCase  :List[Any]    						=	json.load(UpperCamelCase__)
        __UpperCamelCase  :Optional[Any]    						=	{v: k for k, v in self.encoder.items()}
        with open(UpperCamelCase__				,				encoding='''utf-8''') as merges_handle:
            __UpperCamelCase  :int    						=	merges_handle.read().split('''\n''')[1:-1]
        __UpperCamelCase  :int    						=	[tuple(merge.split()) for merge in merges]
        __UpperCamelCase  :Any    						=	dict(zip(UpperCamelCase__				,				range(len(UpperCamelCase__))))
        __UpperCamelCase  :Union[str, Any]    						=	{}







    @property
    def     UpperCamelCase__     (			self) ->			int:
        return len(self.encoder)







    def     UpperCamelCase__     (			self) ->			Optional[int]:
        return dict(self.encoder				,				**self.added_tokens_encoder)







    def     UpperCamelCase__     (			self				,				__lowercase) ->			Any:






        if token in self.cache:
            return self.cache[token]
        __UpperCamelCase  :int    						=	tuple(UpperCamelCase__)
        __UpperCamelCase  :Optional[Any]    						=	tuple(list(word[:-1]) + [word[-1] + '''</w>'''])
        __UpperCamelCase  :str    						=	get_pairs(UpperCamelCase__)

        if not pairs:
            return token

        while True:
            __UpperCamelCase  :Any    						=	min(UpperCamelCase__				,				key=lambda __lowercase: self.bpe_ranks.get(UpperCamelCase__				,				float('''inf''')))
            if bigram not in self.bpe_ranks:
                break
            __UpperCamelCase						, __UpperCamelCase  :Optional[int]    						=	bigram
            __UpperCamelCase  :List[Any]    						=	[]
            __UpperCamelCase  :Optional[Any]    						=	0
            while i < len(UpperCamelCase__):
                try:
                    __UpperCamelCase  :Optional[Any]    						=	word.index(UpperCamelCase__				,				UpperCamelCase__)
                except ValueError:
                    new_word.extend(word[i:])
                    break
                else:
                    new_word.extend(word[i:j])
                    __UpperCamelCase  :str    						=	j

                if word[i] == first and i < len(UpperCamelCase__) - 1 and word[i + 1] == second:
                    new_word.append(first + second)
                    i += 2
                else:
                    new_word.append(word[i])
                    i += 1
            __UpperCamelCase  :Tuple    						=	tuple(UpperCamelCase__)
            __UpperCamelCase  :Tuple    						=	new_word
            if len(UpperCamelCase__) == 1:
                break
            else:
                __UpperCamelCase  :Any    						=	get_pairs(UpperCamelCase__)
        __UpperCamelCase  :Union[str, Any]    						=	'''@@ '''.join(UpperCamelCase__)
        __UpperCamelCase  :Dict    						=	word[:-4]
        __UpperCamelCase  :Optional[int]    						=	word
        return word







    def     UpperCamelCase__     (			self				,				__lowercase) ->			Any:
        __UpperCamelCase  :Any    						=	[]

        __UpperCamelCase  :List[Any]    						=	re.findall(r'''\S+\n?'''				,				UpperCamelCase__)

        for token in words:
            split_tokens.extend(list(self.bpe(UpperCamelCase__).split(''' ''')))
        return split_tokens







    def     UpperCamelCase__     (			self				,				__lowercase) ->			Dict:
        return self.encoder.get(UpperCamelCase__				,				self.encoder.get(self.unk_token))







    def     UpperCamelCase__     (			self				,				__lowercase) ->			Optional[Any]:
        return self.decoder.get(UpperCamelCase__				,				self.unk_token)







    def     UpperCamelCase__     (			self				,				__lowercase) ->			Dict:
        __UpperCamelCase  :Dict    						=	''' '''.join(UpperCamelCase__).replace('''@@ '''				,				'''''').strip()
        return out_string







    def     UpperCamelCase__     (			self				,				__lowercase				,				__lowercase = None) ->			Tuple[str]:






        if not os.path.isdir(UpperCamelCase__):
            logger.error(f"""Vocabulary path ({save_directory}) should be a directory""")
            return
        __UpperCamelCase  :Optional[int]    						=	os.path.join(
            UpperCamelCase__				,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])
        __UpperCamelCase  :Any    						=	os.path.join(
            UpperCamelCase__				,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file'''])

        with open(UpperCamelCase__				,				'''w'''				,				encoding='''utf-8''') as f:
            f.write(json.dumps(self.encoder				,				indent=2				,				sort_keys=UpperCamelCase__				,				ensure_ascii=UpperCamelCase__) + '''\n''')

        __UpperCamelCase  :Dict    						=	0
        with open(UpperCamelCase__				,				'''w'''				,				encoding='''utf-8''') as writer:
            writer.write('''#version: 0.2\n''')
            for bpe_tokens, token_index in sorted(self.bpe_ranks.items()				,				key=lambda __lowercase: kv[1]):
                if index != token_index:
                    logger.warning(
                        f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                        ''' Please check that the tokenizer is not corrupted!''')
                    __UpperCamelCase  :Any    						=	token_index
                writer.write(''' '''.join(UpperCamelCase__) + '''\n''')
                index += 1

        return vocab_file, merge_file

    # def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
    #     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
    #     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
    #     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
    #     return ''.join(tokens_generated_so_far)






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_speech_available, is_torch_available


__lowercase											=      {
    '''configuration_audio_spectrogram_transformer''': [
        '''AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''ASTConfig''',
    ]
}

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    __lowercase											=      [
        '''AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST''',
        '''ASTForAudioClassification''',
        '''ASTModel''',
        '''ASTPreTrainedModel''',
    ]

try:
    if not is_speech_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    __lowercase											=      ['''ASTFeatureExtractor''']

if TYPE_CHECKING:
    from .configuration_audio_spectrogram_transformer import (
        AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP,
        ASTConfig,
    )

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_audio_spectrogram_transformer import (
            AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
            ASTForAudioClassification,
            ASTModel,
            ASTPreTrainedModel,
        )

    try:
        if not is_speech_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .feature_extraction_audio_spectrogram_transformer import ASTFeatureExtractor


else:
    import sys

    __lowercase											=      _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)





