





import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase_					:   Union[str, Any] 							=						logging.get_logger(__name__)


UpperCAmelCase_					:   List[str] 							=						{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

UpperCAmelCase_					:   List[Any] 							=						{
    """vocab_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
    },
}

UpperCAmelCase_					:   List[str] 							=						{
    """allenai/longformer-base-4096""": 40_96,
    """allenai/longformer-large-4096""": 40_96,
    """allenai/longformer-large-4096-finetuned-triviaqa""": 40_96,
    """allenai/longformer-base-4096-extra.pos.embd.only""": 40_96,
    """allenai/longformer-large-4096-extra.pos.embd.only""": 40_96,
}






@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def 							_lowerCAmelCase (  )	->			Union[str, Any]:
     lowerCAmelCase_					:       List[str]    =		(
         list(range(ord("""!"""			)					,   ord("""~"""			) + 1			)			) + list(range(ord("""¡"""			)					,   ord("""¬"""			) + 1			)			) + list(range(ord("""®"""			)					,   ord("""ÿ"""			) + 1			)			)
     )
     lowerCAmelCase_					:       Union[str, Any]    =		bs[:]
     lowerCAmelCase_					:       Dict    =		0
     for b in range(2**8			):
          if b not in bs:
               bs.append(_a			)
               cs.append(2**8 + n			)
               n += 1
     lowerCAmelCase_					:       Optional[Any]    =		[chr(_a			) for n in cs]
     return dict(zip(_a					,   _a			)			)






def 							_lowerCAmelCase (  _a   :					int			)	->			List[Any]:
     lowerCAmelCase_					:       Optional[Any]    =		set()
     lowerCAmelCase_					:       List[str]    =		word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)			)
          lowerCAmelCase_					:       str    =		char
     return pairs







class      lowercase__    (							__A  ):
  __UpperCamelCase	     =  VOCAB_FILES_NAMES
  __UpperCamelCase	     =  PRETRAINED_VOCAB_FILES_MAP
  __UpperCamelCase	     =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  __UpperCamelCase	     =  ["""input_ids""", """attention_mask"""]







  def __init__(	self    ,						_lowercase    ,						_lowercase    ,						_lowercase="replace"    ,						_lowercase="<s>"    ,						_lowercase="</s>"    ,						_lowercase="</s>"    ,						_lowercase="<s>"    ,						_lowercase="<unk>"    ,						_lowercase="<pad>"    ,						_lowercase="<mask>"    ,						_lowercase=False    ,						**_lowercase    ,						):
       lowerCAmelCase_					:       Dict    =		AddedToken(_lowercase    ,						lstrip=_lowercase    ,						rstrip=_lowercase						) if isinstance(_lowercase    ,						_lowercase						) else bos_token
       lowerCAmelCase_					:       str    =		AddedToken(_lowercase    ,						lstrip=_lowercase    ,						rstrip=_lowercase						) if isinstance(_lowercase    ,						_lowercase						) else eos_token
       lowerCAmelCase_					:       Optional[Any]    =		AddedToken(_lowercase    ,						lstrip=_lowercase    ,						rstrip=_lowercase						) if isinstance(_lowercase    ,						_lowercase						) else sep_token
       lowerCAmelCase_					:       Optional[Any]    =		AddedToken(_lowercase    ,						lstrip=_lowercase    ,						rstrip=_lowercase						) if isinstance(_lowercase    ,						_lowercase						) else cls_token
       lowerCAmelCase_					:       Union[str, Any]    =		AddedToken(_lowercase    ,						lstrip=_lowercase    ,						rstrip=_lowercase						) if isinstance(_lowercase    ,						_lowercase						) else unk_token
       lowerCAmelCase_					:       List[str]    =		AddedToken(_lowercase    ,						lstrip=_lowercase    ,						rstrip=_lowercase						) if isinstance(_lowercase    ,						_lowercase						) else pad_token

       # Mask token behave like a normal word, i.e. include the space before it
       lowerCAmelCase_					:       Optional[Any]    =		AddedToken(_lowercase    ,						lstrip=_lowercase    ,						rstrip=_lowercase						) if isinstance(_lowercase    ,						_lowercase						) else mask_token

       super().__init__(
           errors=_lowercase    ,						bos_token=_lowercase    ,						eos_token=_lowercase    ,						unk_token=_lowercase    ,						sep_token=_lowercase    ,						cls_token=_lowercase    ,						pad_token=_lowercase    ,						mask_token=_lowercase    ,						add_prefix_space=_lowercase    ,						**_lowercase    ,						)

       with open(_lowercase    ,						encoding="""utf-8"""						) as vocab_handle:
            lowerCAmelCase_					:       Optional[Any]    =		json.load(_lowercase						)
       lowerCAmelCase_					:       str    =		{v: k for k, v in self.encoder.items()}
       lowerCAmelCase_					:       Any    =		errors  # how to handle errors in decoding
       lowerCAmelCase_					:       str    =		bytes_to_unicode()
       lowerCAmelCase_					:       Optional[Any]    =		{v: k for k, v in self.byte_encoder.items()}
       with open(_lowercase    ,						encoding="""utf-8"""						) as merges_handle:
            lowerCAmelCase_					:       Optional[int]    =		merges_handle.read().split("""\n"""						)[1:-1]
       lowerCAmelCase_					:       Optional[Any]    =		[tuple(merge.split()						) for merge in bpe_merges]
       lowerCAmelCase_					:       List[str]    =		dict(zip(_lowercase    ,						range(len(_lowercase						)						)						)						)
       lowerCAmelCase_					:       Optional[int]    =		{}
       lowerCAmelCase_					:       int    =		add_prefix_space

       # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
       lowerCAmelCase_					:       Optional[Any]    =		re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""						)







  @property
  def 	UpperCAmelCase__		(	self						):
       return len(self.encoder						)







  def 	UpperCAmelCase__		(	self						):
       return dict(self.encoder    ,						**self.added_tokens_encoder						)







  def 	UpperCAmelCase__		(	self    ,						_lowercase						):
       if token in self.cache:
            return self.cache[token]
       lowerCAmelCase_					:       List[str]    =		tuple(_lowercase						)
       lowerCAmelCase_					:       Any    =		get_pairs(_lowercase						)

       if not pairs:
            return token

       while True:
            lowerCAmelCase_					:       Optional[int]    =		min(_lowercase    ,						key=lambda _lowercase						: self.bpe_ranks.get(_lowercase    ,						float("""inf"""						)						)						)
            if bigram not in self.bpe_ranks:
                 break
            lowerCAmelCase_   ,  lowerCAmelCase_					:       Tuple    =		bigram
            lowerCAmelCase_					:       str    =		[]
            lowerCAmelCase_					:       Any    =		0
            while i < len(_lowercase						):
                 try:
                      lowerCAmelCase_					:       str    =		word.index(_lowercase    ,						_lowercase						)
                 except ValueError:
                      new_word.extend(word[i:]						)
                      break
                 else:
                      new_word.extend(word[i:j]						)
                      lowerCAmelCase_					:       List[Any]    =		j

                 if word[i] == first and i < len(_lowercase						) - 1 and word[i + 1] == second:
                      new_word.append(first + second						)
                      i += 2
                 else:
                      new_word.append(word[i]						)
                      i += 1
            lowerCAmelCase_					:       Optional[Any]    =		tuple(_lowercase						)
            lowerCAmelCase_					:       str    =		new_word
            if len(_lowercase						) == 1:
                 break
            else:
                 lowerCAmelCase_					:       Dict    =		get_pairs(_lowercase						)
       lowerCAmelCase_					:       Optional[int]    =		""" """.join(_lowercase						)
       lowerCAmelCase_					:       Tuple    =		word
       return word







  def 	UpperCAmelCase__		(	self    ,						_lowercase						):
       lowerCAmelCase_					:       int    =		[]
       for token in re.findall(self.pat    ,						_lowercase						):
            lowerCAmelCase_					:       Union[str, Any]    =		"""""".join(
                self.byte_encoder[b] for b in token.encode("""utf-8"""						)						)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
            bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_lowercase						).split(""" """						)						)
       return bpe_tokens







  def 	UpperCAmelCase__		(	self    ,						_lowercase						):
       return self.encoder.get(_lowercase    ,						self.encoder.get(self.unk_token						)						)







  def 	UpperCAmelCase__		(	self    ,						_lowercase						):
       return self.decoder.get(_lowercase						)







  def 	UpperCAmelCase__		(	self    ,						_lowercase						):
       lowerCAmelCase_					:       List[str]    =		"""""".join(_lowercase						)
       lowerCAmelCase_					:       Any    =		bytearray([self.byte_decoder[c] for c in text]						).decode("""utf-8"""    ,						errors=self.errors						)
       return text







  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase = None						):
       if not os.path.isdir(_lowercase						):
            logger.error(F'Vocabulary path ({save_directory}) should be a directory'						)
            return
       lowerCAmelCase_					:       Optional[Any]    =		os.path.join(
           _lowercase    ,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)
       lowerCAmelCase_					:       Optional[int]    =		os.path.join(
           _lowercase    ,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]						)

       with open(_lowercase    ,						"""w"""    ,						encoding="""utf-8"""						) as f:
            f.write(json.dumps(self.encoder    ,						indent=2    ,						sort_keys=_lowercase    ,						ensure_ascii=_lowercase						) + """\n"""						)

       lowerCAmelCase_					:       int    =		0
       with open(_lowercase    ,						"""w"""    ,						encoding="""utf-8"""						) as writer:
            writer.write("""#version: 0.2\n"""						)
            for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,						key=lambda _lowercase						: kv[1]						):
                 if index != token_index:
                      logger.warning(
                          F'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                          """ Please check that the tokenizer is not corrupted!"""						)
                      lowerCAmelCase_					:       Dict    =		token_index
                 writer.write(""" """.join(_lowercase						) + """\n"""						)
                 index += 1

       return vocab_file, merge_file







  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase = None						):
       if token_ids_a is None:
            return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
       lowerCAmelCase_					:       str    =		[self.cls_token_id]
       lowerCAmelCase_					:       Optional[Any]    =		[self.sep_token_id]
       return cls + token_ids_a + sep + sep + token_ids_a + sep







  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase = None    ,						_lowercase = False						):
       if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_a=_lowercase    ,						token_ids_a=_lowercase    ,						already_has_special_tokens=_lowercase						)

       if token_ids_a is None:
            return [1] + ([0] * len(_lowercase						)) + [1]
       return [1] + ([0] * len(_lowercase						)) + [1, 1] + ([0] * len(_lowercase						)) + [1]







  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase = None						):
       lowerCAmelCase_					:       str    =		[self.sep_token_id]
       lowerCAmelCase_					:       Optional[int]    =		[self.cls_token_id]

       if token_ids_a is None:
            return len(cls + token_ids_a + sep						) * [0]
       return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]







  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase=False    ,						**_lowercase						):
       lowerCAmelCase_					:       str    =		kwargs.pop("""add_prefix_space"""    ,						self.add_prefix_space						)
       if (is_split_into_words or add_prefix_space) and (len(_lowercase						) > 0 and not text[0].isspace()):
            lowerCAmelCase_					:       str    =		""" """ + text
       return (text, kwargs)







# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING

# rely on isort to merge the imports
from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


UpperCAmelCase_					:   Tuple 							=						{
    """configuration_vivit""": ["""VIVIT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """VivitConfig"""],
}
try:
      if not is_vision_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      UpperCAmelCase_					:   Tuple 							=						["""VivitImageProcessor"""]


try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      UpperCAmelCase_					:   Any 							=						[
          """VIVIT_PRETRAINED_MODEL_ARCHIVE_LIST""",
          """VivitModel""",
          """VivitPreTrainedModel""",
          """VivitForVideoClassification""",
      ]


if TYPE_CHECKING:
      from .configuration_vivit import VIVIT_PRETRAINED_CONFIG_ARCHIVE_MAP, VivitConfig

      try:
            if not is_vision_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .image_processing_vivit import VivitImageProcessor

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_vivit import (
                VIVIT_PRETRAINED_MODEL_ARCHIVE_LIST,
                VivitForVideoClassification,
                VivitModel,
                VivitPreTrainedModel,
            )


else:
      import sys

      UpperCAmelCase_					:   Dict 							=						_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)
