'''simple docstring'''




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


_A								=   logging.get_logger(__name__)

_A								=   {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

_A								=   [
    'small',
    'small-base',
    'medium',
    'medium-base',
    'intermediate',
    'intermediate-base',
    'large',
    'large-base',
    'xlarge',
    'xlarge-base',
]

_A								=   {
    'vocab_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt',
        'funnel-transformer/small-base': 'https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt',
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt',
        'funnel-transformer/large-base': 'https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt',
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json',
        'funnel-transformer/small-base': (
            'https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json',
        'funnel-transformer/large-base': (
            'https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'
        ),
    },
}
_A								=   {F"""funnel-transformer/{name}""": 5_1_2 for name in _model_names}
_A								=   {F"""funnel-transformer/{name}""": {'do_lower_case': True} for name in _model_names}


class       UpperCAmelCase__  (       _snake_case			):






   """simple docstring"""
   A							:       Tuple     =    VOCAB_FILES_NAMES
   A							:       Optional[int]     =    PRETRAINED_VOCAB_FILES_MAP
   A							:       Optional[Any]     =    PRETRAINED_INIT_CONFIGURATION
   A							:       str     =    FunnelTokenizer
   A							:       Optional[int]     =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   A							:       int     =    2
   def __init__(self       ,				_a=None       ,				_a=None       ,				_a=True       ,				_a="<unk>"       ,				_a="<sep>"       ,				_a="<pad>"       ,				_a="<cls>"       ,				_a="<mask>"       ,				_a="<s>"       ,				_a="</s>"       ,				_a=True       ,				_a=True       ,				_a=None       ,				_a="##"       ,				**_a       ,				)    ->     List[Any]:
        super().__init__(
            _a       ,				tokenizer_file=_a       ,				do_lower_case=_a       ,				unk_token=_a       ,				sep_token=_a       ,				pad_token=_a       ,				cls_token=_a       ,				mask_token=_a       ,				bos_token=_a       ,				eos_token=_a       ,				clean_text=_a       ,				tokenize_chinese_chars=_a       ,				strip_accents=_a       ,				wordpieces_prefix=_a       ,				**_a       ,				)

        lowercase_					:       str     				=					json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
        if (
            normalizer_state.get('lowercase'       ,				_a				) != do_lower_case
            or normalizer_state.get('strip_accents'       ,				_a				) != strip_accents
            or normalizer_state.get('handle_chinese_chars'       ,				_a				) != tokenize_chinese_chars
        ):
             lowercase_					:       Dict     				=					getattr(_a       ,				normalizer_state.pop('type'				)				)
             lowercase_					:       int     				=					do_lower_case
             lowercase_					:       Any     				=					strip_accents
             lowercase_					:       Optional[Any]     				=					tokenize_chinese_chars
             lowercase_					:       Optional[int]     				=					normalizer_class(**_a				)

        lowercase_					:       Dict     				=					do_lower_case
   def 		_lowerCamelCase (self       ,				_a       ,				_a=None				)    ->     Optional[int]:
        lowercase_					:       Optional[Any]     				=					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
             output += token_ids_a + [self.sep_token_id]

        return output
   def 		_lowerCamelCase (self       ,				_a       ,				_a = None				)    ->     List[int]:
        lowercase_					:       Union[str, Any]     				=					[self.sep_token_id]
        lowercase_					:       int     				=					[self.cls_token_id]
        if token_ids_a is None:
             return len(cls				) * [self.cls_token_type_id] + len(token_ids_a + sep				) * [0]
        return len(cls				) * [self.cls_token_type_id] + len(token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]





   def 		_lowerCamelCase (self       ,				_a       ,				_a = None				)    ->     Tuple[str]:
        lowercase_					:       List[str]     				=					self._tokenizer.model.save(_a       ,				name=_a				)
        return tuple(_a				)





'''simple docstring'''




import math
import os
import sys

def _UpperCamelCase   (       SCREAMING_SNAKE_CASE_							):
     lowercase_					:       List[str]     				=					''
     try:
          with open(SCREAMING_SNAKE_CASE_  ,       'rb'							) as binary_file:
               lowercase_					:       Dict     				=					binary_file.read()
          for dat in data:
               lowercase_					:       Union[str, Any]     				=					f'''{dat:08b}'''
               result += curr_byte
          return result
     except OSError:
          print('File not accessible'							)
          sys.exit()

def _UpperCamelCase   (       SCREAMING_SNAKE_CASE_  ,       SCREAMING_SNAKE_CASE_  ,       SCREAMING_SNAKE_CASE_  ,       SCREAMING_SNAKE_CASE_							):
     lexicon.pop(SCREAMING_SNAKE_CASE_							)
     lowercase_					:       Any     				=					last_match_id

     if math.loga(SCREAMING_SNAKE_CASE_							).is_integer():
          for curr_key in lexicon:
               lowercase_					:       Tuple     				=					'0' + lexicon[curr_key]

     lowercase_					:       Any     				=					bin(SCREAMING_SNAKE_CASE_							)[2:]

def _UpperCamelCase   (       SCREAMING_SNAKE_CASE_							):
     lowercase_					:       Dict     				=					{'0': '0', '1': '1'}
     lowercase_				,lowercase_					:       Any     				=					'', ''
     lowercase_					:       List[Any]     				=					len(SCREAMING_SNAKE_CASE_							)

     for i in range(len(SCREAMING_SNAKE_CASE_							)							):
          curr_string += data_bits[i]
          if curr_string not in lexicon:
               continue

          lowercase_					:       str     				=					lexicon[curr_string]
          result += last_match_id
          add_key_to_lexicon(SCREAMING_SNAKE_CASE_  ,       SCREAMING_SNAKE_CASE_  ,       SCREAMING_SNAKE_CASE_  ,       SCREAMING_SNAKE_CASE_							)
          index += 1
          lowercase_					:       List[Any]     				=					''

     while curr_string != "" and curr_string not in lexicon:
          curr_string += "0"

     if curr_string != "":
          lowercase_					:       Optional[Any]     				=					lexicon[curr_string]
          result += last_match_id

     return result

def _UpperCamelCase   (       SCREAMING_SNAKE_CASE_  ,       SCREAMING_SNAKE_CASE_							):
     lowercase_					:       List[str]     				=					os.path.getsize(SCREAMING_SNAKE_CASE_							)
     lowercase_					:       Any     				=					bin(SCREAMING_SNAKE_CASE_							)[2:]
     lowercase_					:       Optional[Any]     				=					len(SCREAMING_SNAKE_CASE_							)

     return "0" * (length_length - 1) + file_length_binary + compressed

def _UpperCamelCase   (       SCREAMING_SNAKE_CASE_  ,       SCREAMING_SNAKE_CASE_							):
     lowercase_					:       List[str]     				=					8
     try:
          with open(SCREAMING_SNAKE_CASE_  ,       'wb'							) as opened_file:
               lowercase_					:       List[Any]     				=					[
                   to_write[i : i + byte_length]
                   for i in range(0  ,       len(SCREAMING_SNAKE_CASE_							)  ,       SCREAMING_SNAKE_CASE_							)
               ]

               if len(result_byte_array[-1]							) % byte_length == 0:
                    result_byte_array.append('10000000'							)
               else:
                    result_byte_array[-1] += "1" + "0" * (
                        byte_length - len(result_byte_array[-1]							) - 1
                    )

               for elem in result_byte_array:
                    opened_file.write(int(SCREAMING_SNAKE_CASE_  ,       2							).to_bytes(1  ,       byteorder='big'							)							)
     except OSError:
          print('File not accessible'							)
          sys.exit()

def _UpperCamelCase   (       SCREAMING_SNAKE_CASE_  ,       SCREAMING_SNAKE_CASE_							):
     lowercase_					:       List[str]     				=					read_file_binary(SCREAMING_SNAKE_CASE_							)
     lowercase_					:       List[str]     				=					compress_data(SCREAMING_SNAKE_CASE_							)
     lowercase_					:       Union[str, Any]     				=					add_file_length(SCREAMING_SNAKE_CASE_  ,       SCREAMING_SNAKE_CASE_							)
     write_file_binary(SCREAMING_SNAKE_CASE_  ,       SCREAMING_SNAKE_CASE_							)


if __name__ == "__main__":
 compress(sys.argv[1], sys.argv[2])




