
import json
import os
from typing import Dict, List, Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCamelCase__    :		List[Any]          =   logging.get_logger(__name__)


lowerCamelCase__    :		Dict          =   {
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
    """tokenizer_config_file""": """tokenizer_config.json""",
}

lowerCamelCase__    :		Optional[int]          =   {
    """vocab_file""": {
        """facebook/blenderbot_small-90M""": """https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/vocab.json"""
    },
    """merges_file""": {
        """facebook/blenderbot_small-90M""": """https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/merges.txt"""
    },
    """tokenizer_config_file""": {
        """facebook/blenderbot_small-90M""": (
            """https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/tokenizer_config.json"""
        )
    },
}

lowerCamelCase__    :		Any          =   {"""facebook/blenderbot_small-90M""": 5_1_2}

def 		UpperCamelCase	(      lowercase_   )		->	Optional[int]:


  '''simple docstring'''






  lowercase__  :   List[str]    		=      set()
  lowercase__  :   int    		=      word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)   )
    lowercase__  :   str    		=      char

  lowercase__  :   Tuple    		=      set(lowercase_   )
  return pairs



class       _snake_case   (	UpperCAmelCase_							):
 __lowerCAmelCase  :     Any        =			VOCAB_FILES_NAMES
 __lowerCAmelCase  :     int        =			PRETRAINED_VOCAB_FILES_MAP
 __lowerCAmelCase  :     List[str]        =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __lowerCAmelCase  :     Union[str, Any]        =			['input_ids', 'attention_mask']


 def __init__(					self					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_="__start__"					,  SCREAMING_SNAKE_CASE_="__end__"					,  SCREAMING_SNAKE_CASE_="__unk__"					,  SCREAMING_SNAKE_CASE_="__null__"					,  **SCREAMING_SNAKE_CASE_					,  ):


   '''simple docstring'''




   super().__init__(unk_token=SCREAMING_SNAKE_CASE_					,  bos_token=SCREAMING_SNAKE_CASE_					,  eos_token=SCREAMING_SNAKE_CASE_					,  pad_token=SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_)

   with open(SCREAMING_SNAKE_CASE_					,  encoding="""utf-8""") as vocab_handle:
     lowercase__  :   Optional[int]    		=      json.load(SCREAMING_SNAKE_CASE_)
   lowercase__  :   Tuple    		=      {v: k for k, v in self.encoder.items()}
   with open(SCREAMING_SNAKE_CASE_					,  encoding="""utf-8""") as merges_handle:
     lowercase__  :   Union[str, Any]    		=      merges_handle.read().split("""\n""")[1:-1]
   lowercase__  :   List[str]    		=      [tuple(merge.split()) for merge in merges]
   lowercase__  :   str    		=      dict(zip(SCREAMING_SNAKE_CASE_					,  range(len(SCREAMING_SNAKE_CASE_))))
   lowercase__  :   int    		=      {}


 @property
 def      lowercase__     (					self):


   '''simple docstring'''




   return len(self.encoder)


 def      lowercase__     (					self):


   '''simple docstring'''




   return dict(self.encoder					,  **self.added_tokens_encoder)


 def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   if token in self.cache:
     return self.cache[token]
   lowercase__  :   Optional[int]    		=      re.sub("""([.,!?()])"""					,  R""" \1"""					,  SCREAMING_SNAKE_CASE_)
   lowercase__  :   str    		=      re.sub("""(')"""					,  R""" \1 """					,  SCREAMING_SNAKE_CASE_)
   lowercase__  :   Tuple    		=      re.sub(R"""\s{2,}"""					,  """ """					,  SCREAMING_SNAKE_CASE_)
   if "\n" in token:
     lowercase__  :   Optional[Any]    		=      token.replace("""\n"""					,  """ __newln__""")

   lowercase__  :   List[str]    		=      token.split(""" """)
   lowercase__  :   List[Any]    		=      []
   for token in tokens:
     if not len(SCREAMING_SNAKE_CASE_):
       continue

     lowercase__  :   Union[str, Any]    		=      token.lower()
     lowercase__  :   Tuple    		=      tuple(SCREAMING_SNAKE_CASE_)
     lowercase__  :   Optional[int]    		=      tuple(list(word[:-1]) + [word[-1] + """</w>"""])
     lowercase__  :   Dict    		=      get_pairs(SCREAMING_SNAKE_CASE_)

     if not pairs:
       words.append(SCREAMING_SNAKE_CASE_)
       continue

     while True:
       lowercase__  :   str    		=      min(SCREAMING_SNAKE_CASE_					,  key=lambda SCREAMING_SNAKE_CASE_: self.bpe_ranks.get(SCREAMING_SNAKE_CASE_					,  float("""inf""")))
       if bigram not in self.bpe_ranks:
         break
       lowercase__		,    lowercase__  :   str    		=      bigram
       lowercase__  :   List[str]    		=      []
       lowercase__  :   List[Any]    		=      0

       while i < len(SCREAMING_SNAKE_CASE_):
         try:
           lowercase__  :   Optional[Any]    		=      word.index(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_)
           new_word.extend(word[i:j])
           lowercase__  :   Any    		=      j
         except ValueError:
           new_word.extend(word[i:])
           break

         if word[i] == first and i < len(SCREAMING_SNAKE_CASE_) - 1 and word[i + 1] == second:
           new_word.append(first + second)
           i += 2
         else:
           new_word.append(word[i])
           i += 1
       lowercase__  :   Any    		=      tuple(SCREAMING_SNAKE_CASE_)
       lowercase__  :   Tuple    		=      new_word
       if len(SCREAMING_SNAKE_CASE_) == 1:
         break
       else:
         lowercase__  :   int    		=      get_pairs(SCREAMING_SNAKE_CASE_)
     lowercase__  :   str    		=      """@@ """.join(SCREAMING_SNAKE_CASE_)
     lowercase__  :   List[str]    		=      word[:-4]

     lowercase__  :   Optional[int]    		=      word
     words.append(SCREAMING_SNAKE_CASE_)
   return " ".join(SCREAMING_SNAKE_CASE_)


 def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   lowercase__  :   Any    		=      []

   lowercase__  :   Union[str, Any]    		=      re.findall(R"""\S+\n?"""					,  SCREAMING_SNAKE_CASE_)

   for token in words:
     split_tokens.extend(list(self.bpe(SCREAMING_SNAKE_CASE_).split(""" """)))
   return split_tokens


 def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   lowercase__  :   List[Any]    		=      token.lower()
   return self.encoder.get(SCREAMING_SNAKE_CASE_					,  self.encoder.get(self.unk_token))


 def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   return self.decoder.get(SCREAMING_SNAKE_CASE_					,  self.unk_token)


 def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   lowercase__  :   str    		=      """ """.join(SCREAMING_SNAKE_CASE_).replace("""@@ """					,  """""").strip()
   return out_string



 def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_ = None):


   '''simple docstring'''




   if not os.path.isdir(SCREAMING_SNAKE_CASE_):
     logger.error(f'Vocabulary path ({save_directory}) should be a directory')
     return
   lowercase__  :   List[Any]    		=      os.path.join(
       SCREAMING_SNAKE_CASE_					,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""])
   lowercase__  :   str    		=      os.path.join(
       SCREAMING_SNAKE_CASE_					,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""])

   with open(SCREAMING_SNAKE_CASE_					,  """w"""					,  encoding="""utf-8""") as f:
     f.write(json.dumps(self.encoder					,  indent=2					,  sort_keys=SCREAMING_SNAKE_CASE_					,  ensure_ascii=SCREAMING_SNAKE_CASE_) + """\n""")

   lowercase__  :   Optional[int]    		=      0
   with open(SCREAMING_SNAKE_CASE_					,  """w"""					,  encoding="""utf-8""") as writer:
     writer.write("""#version: 0.2\n""")
     for bpe_tokens, token_index in sorted(self.bpe_ranks.items()					,  key=lambda SCREAMING_SNAKE_CASE_: kv[1]):
       if index != token_index:
         logger.warning(
             f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
             """ Please check that the tokenizer is not corrupted!""")
         lowercase__  :   Dict    		=      token_index
       writer.write(""" """.join(SCREAMING_SNAKE_CASE_) + """\n""")
       index += 1

   return vocab_file, merge_file








lowerCamelCase__    :		Any          =   """0.18.2"""

from .configuration_utils import ConfigMixin
from .utils import (
    OptionalDependencyNotAvailable,
    is_flax_available,
    is_inflect_available,
    is_invisible_watermark_available,
    is_k_diffusion_available,
    is_k_diffusion_version,
    is_librosa_available,
    is_note_seq_available,
    is_onnx_available,
    is_scipy_available,
    is_torch_available,
    is_torchsde_available,
    is_transformers_available,
    is_transformers_version,
    is_unidecode_available,
    logging,
)


try:
     if not is_onnx_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     from .utils.dummy_onnx_objects import *  # noqa F403
else:
     from .pipelines import OnnxRuntimeModel

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     from .utils.dummy_pt_objects import *  # noqa F403
else:
     from .models import (
         AutoencoderKL,
         ControlNetModel,
         ModelMixin,
         PriorTransformer,
         TaFilmDecoder,
         TransformeraDModel,
         UNetaDModel,
         UNetaDConditionModel,
         UNetaDModel,
         UNetaDConditionModel,
         VQModel,
     )
     from .optimization import (
         get_constant_schedule,
         get_constant_schedule_with_warmup,
         get_cosine_schedule_with_warmup,
         get_cosine_with_hard_restarts_schedule_with_warmup,
         get_linear_schedule_with_warmup,
         get_polynomial_decay_schedule_with_warmup,
         get_scheduler,
     )
     from .pipelines import (
         AudioPipelineOutput,
         ConsistencyModelPipeline,
         DanceDiffusionPipeline,
         DDIMPipeline,
         DDPMPipeline,
         DiffusionPipeline,
         DiTPipeline,
         ImagePipelineOutput,
         KarrasVePipeline,
         LDMPipeline,
         LDMSuperResolutionPipeline,
         PNDMPipeline,
         RePaintPipeline,
         ScoreSdeVePipeline,
     )
     from .schedulers import (
         CMStochasticIterativeScheduler,
         DDIMInverseScheduler,
         DDIMParallelScheduler,
         DDIMScheduler,
         DDPMParallelScheduler,
         DDPMScheduler,
         DEISMultistepScheduler,
         DPMSolverMultistepInverseScheduler,
         DPMSolverMultistepScheduler,
         DPMSolverSinglestepScheduler,
         EulerAncestralDiscreteScheduler,
         EulerDiscreteScheduler,
         HeunDiscreteScheduler,
         IPNDMScheduler,
         KarrasVeScheduler,
         KDPMaAncestralDiscreteScheduler,
         KDPMaDiscreteScheduler,
         PNDMScheduler,
         RePaintScheduler,
         SchedulerMixin,
         ScoreSdeVeScheduler,
         UnCLIPScheduler,
         UniPCMultistepScheduler,
         VQDiffusionScheduler,
     )
     from .training_utils import EMAModel

try:
     if not (is_torch_available() and is_scipy_available()):
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     from .utils.dummy_torch_and_scipy_objects import *  # noqa F403
else:
     from .schedulers import LMSDiscreteScheduler

try:
     if not (is_torch_available() and is_torchsde_available()):
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     from .utils.dummy_torch_and_torchsde_objects import *  # noqa F403
else:
     from .schedulers import DPMSolverSDEScheduler

try:
     if not (is_torch_available() and is_transformers_available()):
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     from .utils.dummy_torch_and_transformers_objects import *  # noqa F403
else:
     from .pipelines import (
         AltDiffusionImgaImgPipeline,
         AltDiffusionPipeline,
         AudioLDMPipeline,
         CycleDiffusionPipeline,
         IFImgaImgPipeline,
         IFImgaImgSuperResolutionPipeline,
         IFInpaintingPipeline,
         IFInpaintingSuperResolutionPipeline,
         IFPipeline,
         IFSuperResolutionPipeline,
         ImageTextPipelineOutput,
         KandinskyImgaImgPipeline,
         KandinskyInpaintPipeline,
         KandinskyPipeline,
         KandinskyPriorPipeline,
         KandinskyVaaControlnetImgaImgPipeline,
         KandinskyVaaControlnetPipeline,
         KandinskyVaaImgaImgPipeline,
         KandinskyVaaInpaintPipeline,
         KandinskyVaaPipeline,
         KandinskyVaaPriorEmbaEmbPipeline,
         KandinskyVaaPriorPipeline,
         LDMTextToImagePipeline,
         PaintByExamplePipeline,
         SemanticStableDiffusionPipeline,
         ShapEImgaImgPipeline,
         ShapEPipeline,
         StableDiffusionAttendAndExcitePipeline,
         StableDiffusionControlNetImgaImgPipeline,
         StableDiffusionControlNetInpaintPipeline,
         StableDiffusionControlNetPipeline,
         StableDiffusionDepthaImgPipeline,
         StableDiffusionDiffEditPipeline,
         StableDiffusionImageVariationPipeline,
         StableDiffusionImgaImgPipeline,
         StableDiffusionInpaintPipeline,
         StableDiffusionInpaintPipelineLegacy,
         StableDiffusionInstructPixaPixPipeline,
         StableDiffusionLatentUpscalePipeline,
         StableDiffusionLDMaDPipeline,
         StableDiffusionModelEditingPipeline,
         StableDiffusionPanoramaPipeline,
         StableDiffusionParadigmsPipeline,
         StableDiffusionPipeline,
         StableDiffusionPipelineSafe,
         StableDiffusionPixaPixZeroPipeline,
         StableDiffusionSAGPipeline,
         StableDiffusionUpscalePipeline,
         StableUnCLIPImgaImgPipeline,
         StableUnCLIPPipeline,
         TextToVideoSDPipeline,
         TextToVideoZeroPipeline,
         UnCLIPImageVariationPipeline,
         UnCLIPPipeline,
         UniDiffuserModel,
         UniDiffuserPipeline,
         UniDiffuserTextDecoder,
         VersatileDiffusionDualGuidedPipeline,
         VersatileDiffusionImageVariationPipeline,
         VersatileDiffusionPipeline,
         VersatileDiffusionTextToImagePipeline,
         VideoToVideoSDPipeline,
         VQDiffusionPipeline,
     )

try:
     if not (is_torch_available() and is_transformers_available() and is_invisible_watermark_available()):
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     from .utils.dummy_torch_and_transformers_and_invisible_watermark_objects import *  # noqa F403
else:
     from .pipelines import StableDiffusionXLImgaImgPipeline, StableDiffusionXLPipeline

try:
     if not (is_torch_available() and is_transformers_available() and is_k_diffusion_available()):
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     from .utils.dummy_torch_and_transformers_and_k_diffusion_objects import *  # noqa F403
else:
     from .pipelines import StableDiffusionKDiffusionPipeline

try:
     if not (is_torch_available() and is_transformers_available() and is_onnx_available()):
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     from .utils.dummy_torch_and_transformers_and_onnx_objects import *  # noqa F403
else:
     from .pipelines import (
         OnnxStableDiffusionImgaImgPipeline,
         OnnxStableDiffusionInpaintPipeline,
         OnnxStableDiffusionInpaintPipelineLegacy,
         OnnxStableDiffusionPipeline,
         OnnxStableDiffusionUpscalePipeline,
         StableDiffusionOnnxPipeline,
     )

try:
     if not (is_torch_available() and is_librosa_available()):
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     from .utils.dummy_torch_and_librosa_objects import *  # noqa F403
else:
     from .pipelines import AudioDiffusionPipeline, Mel

try:
     if not (is_transformers_available() and is_torch_available() and is_note_seq_available()):
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     from .utils.dummy_transformers_and_torch_and_note_seq_objects import *  # noqa F403
else:
     from .pipelines import SpectrogramDiffusionPipeline

try:
     if not is_flax_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     from .utils.dummy_flax_objects import *  # noqa F403
else:
     from .models.controlnet_flax import FlaxControlNetModel
     from .models.modeling_flax_utils import FlaxModelMixin
     from .models.unet_ad_condition_flax import FlaxUNetaDConditionModel
     from .models.vae_flax import FlaxAutoencoderKL
     from .pipelines import FlaxDiffusionPipeline
     from .schedulers import (
         FlaxDDIMScheduler,
         FlaxDDPMScheduler,
         FlaxDPMSolverMultistepScheduler,
         FlaxKarrasVeScheduler,
         FlaxLMSDiscreteScheduler,
         FlaxPNDMScheduler,
         FlaxSchedulerMixin,
         FlaxScoreSdeVeScheduler,
     )


try:
     if not (is_flax_available() and is_transformers_available()):
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     from .utils.dummy_flax_and_transformers_objects import *  # noqa F403
else:
     from .pipelines import (
         FlaxStableDiffusionControlNetPipeline,
         FlaxStableDiffusionImgaImgPipeline,
         FlaxStableDiffusionInpaintPipeline,
         FlaxStableDiffusionPipeline,
     )

try:
     if not (is_note_seq_available()):
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     from .utils.dummy_note_seq_objects import *  # noqa F403
else:
     from .pipelines import MidiProcessor






