






'''simple docstring'''



import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE						=	logging.get_logger(__name__)

SCREAMING_SNAKE_CASE						=	{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

SCREAMING_SNAKE_CASE						=	{
    'vocab_file': {'ctrl': 'https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json'},
    'merges_file': {'ctrl': 'https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt'},
}

SCREAMING_SNAKE_CASE						=	{
    'ctrl': 256,
}

SCREAMING_SNAKE_CASE						=	{
    'Pregnancy': 168_629,
    'Christianity': 7_675,
    'Explain': 106_423,
    'Fitness': 63_440,
    'Saving': 63_163,
    'Ask': 27_171,
    'Ass': 95_985,
    'Joke': 163_509,
    'Questions': 45_622,
    'Thoughts': 49_605,
    'Retail': 52_342,
    'Feminism': 164_338,
    'Writing': 11_992,
    'Atheism': 192_263,
    'Netflix': 48_616,
    'Computing': 39_639,
    'Opinion': 43_213,
    'Alone': 44_967,
    'Funny': 58_917,
    'Gaming': 40_358,
    'Human': 4_088,
    'India': 1_331,
    'Joker': 77_138,
    'Diet': 36_206,
    'Legal': 11_859,
    'Norman': 4_939,
    'Tip': 72_689,
    'Weight': 52_343,
    'Movies': 46_273,
    'Running': 23_425,
    'Science': 2_090,
    'Horror': 37_793,
    'Confession': 60_572,
    'Finance': 12_250,
    'Politics': 16_360,
    'Scary': 191_985,
    'Support': 12_654,
    'Technologies': 32_516,
    'Teenage': 66_160,
    'Event': 32_769,
    'Learned': 67_460,
    'Notion': 182_770,
    'Wikipedia': 37_583,
    'Books': 6_665,
    'Extract': 76_050,
    'Confessions': 102_701,
    'Conspiracy': 75_932,
    'Links': 63_674,
    'Narcissus': 150_425,
    'Relationship': 54_766,
    'Relationships': 134_796,
    'Reviews': 41_671,
    'News': 4_256,
    'Translation': 26_820,
    'multilingual': 128_406,
}


def 							lowercase_				(    __A :       Optional[int]			)       ->      Optional[Any]:



 """simple docstring"""




 lowercase					:   Tuple				  =set()
 lowercase					:   Tuple				  =word[0]
 for char in word[1:]:
  pairs.add((prev_char, char)			)
  lowercase					:   Any				  =char

 lowercase					:   Union[str, Any]				  =set(__A			)
 return pairs





class UpperCAmelCase_					(    __A				):




      """simple docstring"""




      UpperCamelCase_								=			VOCAB_FILES_NAMES
      UpperCamelCase_								=			PRETRAINED_VOCAB_FILES_MAP
      UpperCamelCase_								=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCamelCase_								=			CONTROL_CODES






      def __init__( self			:		Dict     ,      UpperCAmelCase			:		Tuple     ,      UpperCAmelCase			:		int     ,      UpperCAmelCase			:		Dict="<unk>"     ,      **UpperCAmelCase			:		Tuple		)  ->  List[str]:


       '''simple docstring'''

       super().__init__(unk_token=UpperCAmelCase     ,      **UpperCAmelCase		)

       with open(UpperCAmelCase     ,      encoding='''utf-8'''		) as vocab_handle:
        lowercase					:   List[Any]				  =json.load(UpperCAmelCase		)
       lowercase					:   Union[str, Any]				  ={v: k for k, v in self.encoder.items()}
       with open(UpperCAmelCase     ,      encoding='''utf-8'''		) as merges_handle:
        lowercase					:   Tuple				  =merges_handle.read().split('''\n'''		)[1:-1]
       lowercase					:   List[str]				  =[tuple(merge.split()		) for merge in merges]
       lowercase					:   Tuple				  =dict(zip(UpperCAmelCase     ,      range(len(UpperCAmelCase		)		)		)		)
       lowercase					:   Dict				  ={}






      @property
      def 				A__      ( self			:		Optional[Any]		)  ->  Dict:


       '''simple docstring'''

       return len(self.encoder		)






      def 				A__      ( self			:		Optional[int]		)  ->  List[str]:


       '''simple docstring'''

       return dict(self.encoder     ,      **self.added_tokens_encoder		)






      def 				A__      ( self			:		str     ,      UpperCAmelCase			:		Optional[int]		)  ->  Union[str, Any]:


       '''simple docstring'''

       if token in self.cache:
        return self.cache[token]
       lowercase					:   Dict				  =tuple(UpperCAmelCase		)
       lowercase					:   Any				  =tuple(list(word[:-1]		) + [word[-1] + '''</w>''']		)
       lowercase					:   Dict				  =get_pairs(UpperCAmelCase		)

       if not pairs:
        return token

       while True:
        lowercase					:   List[Any]				  =min(UpperCAmelCase     ,      key=lambda UpperCAmelCase		: self.bpe_ranks.get(UpperCAmelCase     ,      float('''inf'''		)		)		)
        if bigram not in self.bpe_ranks:
         break
        lowercase  ,		lowercase					:   Tuple				  =bigram
        lowercase					:   Union[str, Any]				  =[]
        lowercase					:   Dict				  =0
        while i < len(UpperCAmelCase		):
         try:
          lowercase					:   List[Any]				  =word.index(UpperCAmelCase     ,      UpperCAmelCase		)
         except ValueError:
          new_word.extend(word[i:]		)
          break
         else:
          new_word.extend(word[i:j]		)
          lowercase					:   int				  =j

         if word[i] == first and i < len(UpperCAmelCase		) - 1 and word[i + 1] == second:
          new_word.append(first + second		)
          i += 2
         else:
          new_word.append(word[i]		)
          i += 1
        lowercase					:   int				  =tuple(UpperCAmelCase		)
        lowercase					:   Optional[Any]				  =new_word
        if len(UpperCAmelCase		) == 1:
         break
        else:
         lowercase					:   int				  =get_pairs(UpperCAmelCase		)
       lowercase					:   List[str]				  ='''@@ '''.join(UpperCAmelCase		)
       lowercase					:   Optional[Any]				  =word[:-4]
       lowercase					:   Union[str, Any]				  =word
       return word






      def 				A__      ( self			:		int     ,      UpperCAmelCase			:		List[str]		)  ->  Any:


       '''simple docstring'''

       lowercase					:   int				  =[]

       lowercase					:   Any				  =re.findall(R'''\S+\n?'''     ,      UpperCAmelCase		)

       for token in words:
        split_tokens.extend(list(self.bpe(UpperCAmelCase		).split(''' '''		)		)		)
       return split_tokens






      def 				A__      ( self			:		int     ,      UpperCAmelCase			:		str		)  ->  Optional[int]:


       '''simple docstring'''

       return self.encoder.get(UpperCAmelCase     ,      self.encoder.get(self.unk_token		)		)






      def 				A__      ( self			:		Tuple     ,      UpperCAmelCase			:		List[str]		)  ->  Optional[Any]:


       '''simple docstring'''

       return self.decoder.get(UpperCAmelCase     ,      self.unk_token		)






      def 				A__      ( self			:		Optional[Any]     ,      UpperCAmelCase			:		List[str]		)  ->  Dict:


       '''simple docstring'''

       lowercase					:   Optional[Any]				  =''' '''.join(UpperCAmelCase		).replace('''@@ '''     ,      ''''''		).strip()
       return out_string






      def 				A__      ( self			:		List[Any]     ,      UpperCAmelCase			:		str     ,      UpperCAmelCase			:		Optional[str] = None		)  ->  Tuple[str]:


       '''simple docstring'''

       if not os.path.isdir(UpperCAmelCase		):
        logger.error(f'Vocabulary path ({save_directory}) should be a directory'		)
        return
       lowercase					:   Union[str, Any]				  =os.path.join(
           UpperCAmelCase     ,      (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)
       lowercase					:   Union[str, Any]				  =os.path.join(
           UpperCAmelCase     ,      (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']		)

       with open(UpperCAmelCase     ,      '''w'''     ,      encoding='''utf-8'''		) as f:
        f.write(json.dumps(self.encoder     ,      indent=2     ,      sort_keys=UpperCAmelCase     ,      ensure_ascii=UpperCAmelCase		) + '''\n'''		)

       lowercase					:   Dict				  =0
       with open(UpperCAmelCase     ,      '''w'''     ,      encoding='''utf-8'''		) as writer:
        writer.write('''#version: 0.2\n'''		)
        for bpe_tokens, token_index in sorted(self.bpe_ranks.items()     ,      key=lambda UpperCAmelCase		: kv[1]		):
         if index != token_index:
          logger.warning(
              f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
              ''' Please check that the tokenizer is not corrupted!'''		)
          lowercase					:   List[str]				  =token_index
         writer.write(''' '''.join(UpperCAmelCase		) + '''\n'''		)
         index += 1

       return vocab_file, merge_file

      # def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
      #     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
      #     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
      #     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
      #     return ''.join(tokens_generated_so_far)











'''simple docstring'''



from typing import Dict, Iterable, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import normalize, rescale, resize, to_channel_dimension_format, to_pil_image
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_pytesseract_available, is_vision_available, logging, requires_backends


if is_vision_available():
      import PIL

# soft dependency
if is_pytesseract_available():
      import pytesseract

SCREAMING_SNAKE_CASE						=	logging.get_logger(__name__)


def 							lowercase_				(    __A :       List[Any]					,      __A :       int					,      __A :       int			)       ->      Optional[int]:



 """simple docstring"""




 return [
     int(1_0_0_0 * (box[0] / width)			),
     int(1_0_0_0 * (box[1] / height)			),
     int(1_0_0_0 * (box[2] / width)			),
     int(1_0_0_0 * (box[3] / height)			),
 ]


def 							lowercase_				(    __A :       np.ndarray					,      __A :       Optional[str]					,      __A :       Optional[str]			)       ->      Optional[Any]:



 """simple docstring"""




 lowercase					:   int				  =to_pil_image(__A			)
 lowercase  ,		lowercase					:   Tuple				  =pil_image.size
 lowercase					:   Optional[Any]				  =pytesseract.image_to_data(__A					,      lang=__A					,      output_type='''dict'''					,      config=__A			)
 lowercase  ,		lowercase  ,		lowercase  ,		lowercase  ,		lowercase					:   Optional[Any]				  =data['''text'''], data['''left'''], data['''top'''], data['''width'''], data['''height''']

 # filter empty words and corresponding coordinates
 lowercase					:   Dict				  =[idx for idx, word in enumerate(__A			) if not word.strip()]
 lowercase					:   str				  =[word for idx, word in enumerate(__A			) if idx not in irrelevant_indices]
 lowercase					:   Optional[int]				  =[coord for idx, coord in enumerate(__A			) if idx not in irrelevant_indices]
 lowercase					:   List[Any]				  =[coord for idx, coord in enumerate(__A			) if idx not in irrelevant_indices]
 lowercase					:   str				  =[coord for idx, coord in enumerate(__A			) if idx not in irrelevant_indices]
 lowercase					:   int				  =[coord for idx, coord in enumerate(__A			) if idx not in irrelevant_indices]

 # turn coordinates into (left, top, left+width, top+height) format
 lowercase					:   Tuple				  =[]
 for x, y, w, h in zip(__A					,      __A					,      __A					,      __A			):
  lowercase					:   str				  =[x, y, x + w, y + h]
  actual_boxes.append(__A			)

 # finally, normalize the bounding boxes
 lowercase					:   List[str]				  =[]
 for box in actual_boxes:
  normalized_boxes.append(normalize_box(__A					,      __A					,      __A			)			)

 assert len(__A			) == len(__A			), "Not as many words as there are bounding boxes"

 return words, normalized_boxes





class UpperCAmelCase_					(    __A				):




      """simple docstring"""




      UpperCamelCase_								=			['''pixel_values''']






      def __init__( self			:		List[Any]     ,      UpperCAmelCase			:		bool = True     ,      UpperCAmelCase			:		Dict[str, int] = None     ,      UpperCAmelCase			:		PILImageResampling = PILImageResampling.BILINEAR     ,      UpperCAmelCase			:		bool = True     ,      UpperCAmelCase			:		float = 1 / 255     ,      UpperCAmelCase			:		bool = True     ,      UpperCAmelCase			:		Union[float, Iterable[float]] = None     ,      UpperCAmelCase			:		Union[float, Iterable[float]] = None     ,      UpperCAmelCase			:		bool = True     ,      UpperCAmelCase			:		Optional[str] = None     ,      UpperCAmelCase			:		Optional[str] = ""     ,      **UpperCAmelCase			:		Tuple     ,      )  ->  None:


       '''simple docstring'''

       super().__init__(**UpperCAmelCase		)
       lowercase					:   Tuple				  =size if size is not None else {'''height''': 224, '''width''': 224}
       lowercase					:   Optional[Any]				  =get_size_dict(UpperCAmelCase		)

       lowercase					:   Optional[Any]				  =do_resize
       lowercase					:   List[Any]				  =size
       lowercase					:   List[str]				  =resample
       lowercase					:   Dict				  =do_rescale
       lowercase					:   str				  =rescale_value
       lowercase					:   Optional[int]				  =do_normalize
       lowercase					:   Any				  =image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
       lowercase					:   Union[str, Any]				  =image_std if image_std is not None else IMAGENET_STANDARD_STD
       lowercase					:   List[Any]				  =apply_ocr
       lowercase					:   Union[str, Any]				  =ocr_lang
       lowercase					:   str				  =tesseract_config






      def 				A__      ( self			:		Dict     ,      UpperCAmelCase			:		np.ndarray     ,      UpperCAmelCase			:		Dict[str, int]     ,      UpperCAmelCase			:		PILImageResampling = PILImageResampling.BILINEAR     ,      UpperCAmelCase			:		Optional[Union[str, ChannelDimension]] = None     ,      **UpperCAmelCase			:		Optional[Any]     ,      )  ->  np.ndarray:


       '''simple docstring'''

       lowercase					:   Tuple				  =get_size_dict(UpperCAmelCase		)
       if "height" not in size or "width" not in size:
        raise ValueError(f'The size dictionary must contain the keys \'height\' and \'width\'. Got {size.keys()}'		)
       lowercase					:   Optional[Any]				  =(size['''height'''], size['''width'''])
       return resize(UpperCAmelCase     ,      size=UpperCAmelCase     ,      resample=UpperCAmelCase     ,      data_format=UpperCAmelCase     ,      **UpperCAmelCase		)






      def 				A__      ( self			:		Union[str, Any]     ,      UpperCAmelCase			:		np.ndarray     ,      UpperCAmelCase			:		Union[int, float]     ,      UpperCAmelCase			:		Optional[Union[str, ChannelDimension]] = None     ,      **UpperCAmelCase			:		Optional[Any]     ,      )  ->  np.ndarray:


       '''simple docstring'''

       return rescale(UpperCAmelCase     ,      scale=UpperCAmelCase     ,      data_format=UpperCAmelCase     ,      **UpperCAmelCase		)






      def 				A__      ( self			:		Union[str, Any]     ,      UpperCAmelCase			:		np.ndarray     ,      UpperCAmelCase			:		Union[float, Iterable[float]]     ,      UpperCAmelCase			:		Union[float, Iterable[float]]     ,      UpperCAmelCase			:		Optional[Union[str, ChannelDimension]] = None     ,      **UpperCAmelCase			:		Optional[int]     ,      )  ->  np.ndarray:


       '''simple docstring'''

       return normalize(UpperCAmelCase     ,      mean=UpperCAmelCase     ,      std=UpperCAmelCase     ,      data_format=UpperCAmelCase     ,      **UpperCAmelCase		)






      def 				A__      ( self			:		Any     ,      UpperCAmelCase			:		ImageInput     ,      UpperCAmelCase			:		bool = None     ,      UpperCAmelCase			:		Dict[str, int] = None     ,      UpperCAmelCase			:		Union[str, Any]=None     ,      UpperCAmelCase			:		bool = None     ,      UpperCAmelCase			:		float = None     ,      UpperCAmelCase			:		bool = None     ,      UpperCAmelCase			:		Union[float, Iterable[float]] = None     ,      UpperCAmelCase			:		Union[float, Iterable[float]] = None     ,      UpperCAmelCase			:		bool = None     ,      UpperCAmelCase			:		Optional[str] = None     ,      UpperCAmelCase			:		Optional[str] = None     ,      UpperCAmelCase			:		Optional[Union[str, TensorType]] = None     ,      UpperCAmelCase			:		ChannelDimension = ChannelDimension.FIRST     ,      **UpperCAmelCase			:		List[str]     ,      )  ->  PIL.Image.Image:


       '''simple docstring'''

       lowercase					:   Optional[int]				  =do_resize if do_resize is not None else self.do_resize
       lowercase					:   Tuple				  =size if size is not None else self.size
       lowercase					:   Optional[int]				  =get_size_dict(UpperCAmelCase		)
       lowercase					:   List[str]				  =resample if resample is not None else self.resample
       lowercase					:   List[Any]				  =do_rescale if do_rescale is not None else self.do_rescale
       lowercase					:   List[Any]				  =rescale_factor if rescale_factor is not None else self.rescale_factor
       lowercase					:   Optional[int]				  =do_normalize if do_normalize is not None else self.do_normalize
       lowercase					:   List[Any]				  =image_mean if image_mean is not None else self.image_mean
       lowercase					:   Optional[int]				  =image_std if image_std is not None else self.image_std
       lowercase					:   Any				  =apply_ocr if apply_ocr is not None else self.apply_ocr
       lowercase					:   Any				  =ocr_lang if ocr_lang is not None else self.ocr_lang
       lowercase					:   Dict				  =tesseract_config if tesseract_config is not None else self.tesseract_config

       lowercase					:   str				  =make_list_of_images(UpperCAmelCase		)

       if not valid_images(UpperCAmelCase		):
        raise ValueError(
            '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
            '''torch.Tensor, tf.Tensor or jax.ndarray.'''		)

       if do_resize and size is None:
        raise ValueError('''Size must be specified if do_resize is True.'''		)

       if do_rescale and rescale_factor is None:
        raise ValueError('''Rescale factor must be specified if do_rescale is True.'''		)

       if do_normalize and (image_mean is None or image_std is None):
        raise ValueError('''If do_normalize is True, image_mean and image_std must be specified.'''		)

       # All transformations expect numpy arrays.
       lowercase					:   Tuple				  =[to_numpy_array(UpperCAmelCase		) for image in images]

       # Tesseract OCR to get words + normalized bounding boxes
       if apply_ocr:
        requires_backends(self     ,      '''pytesseract'''		)
        lowercase					:   int				  =[]
        lowercase					:   Tuple				  =[]
        for image in images:
         lowercase  ,		lowercase					:   Dict				  =apply_tesseract(UpperCAmelCase     ,      UpperCAmelCase     ,      UpperCAmelCase		)
         words_batch.append(UpperCAmelCase		)
         boxes_batch.append(UpperCAmelCase		)

       if do_resize:
        lowercase					:   int				  =[self.resize(image=UpperCAmelCase     ,      size=UpperCAmelCase     ,      resample=UpperCAmelCase		) for image in images]

       if do_rescale:
        lowercase					:   Tuple				  =[self.rescale(image=UpperCAmelCase     ,      scale=UpperCAmelCase		) for image in images]

       if do_normalize:
        lowercase					:   str				  =[self.normalize(image=UpperCAmelCase     ,      mean=UpperCAmelCase     ,      std=UpperCAmelCase		) for image in images]

       lowercase					:   Optional[Any]				  =[to_channel_dimension_format(UpperCAmelCase     ,      UpperCAmelCase		) for image in images]

       lowercase					:   Dict				  =BatchFeature(data={'''pixel_values''': images}     ,      tensor_type=UpperCAmelCase		)

       if apply_ocr:
        lowercase					:   int				  =words_batch
        lowercase					:   List[str]				  =boxes_batch
       return data



