import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


UpperCAmelCase_								=							datasets.utils.logging.get_logger(__name__)





@dataclass
class      lowerCAmelCase						(					datasets.BuilderConfig			):
     _SCREAMING_SNAKE_CASE     :     Optional[datasets.Features]		=None
     _SCREAMING_SNAKE_CASE     :     str		="utf-8"
     _SCREAMING_SNAKE_CASE     :     Optional[str]		=None
     _SCREAMING_SNAKE_CASE     :     Optional[str]		=None
     _SCREAMING_SNAKE_CASE     :     bool		=True  # deprecated
     _SCREAMING_SNAKE_CASE     :     Optional[int]		=None  # deprecated
     _SCREAMING_SNAKE_CASE     :     int		=10 << 20  # 10MB
     _SCREAMING_SNAKE_CASE     :     Optional[bool]		=None





class      lowerCAmelCase						(					datasets.ArrowBasedBuilder			):
     _SCREAMING_SNAKE_CASE     :     int		=JsonConfig



     def 			a__  (					self       ):
            if self.config.block_size is not None:
                   logger.warning('The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead'       )
                   _A=	self.config.block_size
            if self.config.use_threads is not True:
                   logger.warning(
                       'The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.'       )
            if self.config.newlines_in_values is not None:
                   raise ValueError('The JSON loader parameter `newlines_in_values` is no longer supported'       )
            return datasets.DatasetInfo(features=self.config.features       )



     def 			a__  (					self				,      lowerCAmelCase__       ):
            if not self.config.data_files:
                   raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}"       )
            _A=	dl_manager.download_and_extract(self.config.data_files       )
            if isinstance(lowerCAmelCase__				,      (str, list, tuple)       ):
                   _A=	data_files
                   if isinstance(lowerCAmelCase__				,      lowerCAmelCase__       ):
                          _A=	[files]
                   _A=	[dl_manager.iter_files(lowerCAmelCase__       ) for file in files]
                   return [datasets.SplitGenerator(name=datasets.Split.TRAIN				,      gen_kwargs={'files': files}       )]
            _A=	[]
            for split_name, files in data_files.items():
                   if isinstance(lowerCAmelCase__				,      lowerCAmelCase__       ):
                          _A=	[files]
                   _A=	[dl_manager.iter_files(lowerCAmelCase__       ) for file in files]
                   splits.append(datasets.SplitGenerator(name=lowerCAmelCase__				,      gen_kwargs={'files': files}       )       )
            return splits



     def 			a__  (					self				,      lowerCAmelCase__       ):
            if self.config.features is not None:
                   # adding missing columns
                   for column_name in set(self.config.features       ) - set(pa_table.column_names       ):
                          _A=	self.config.features.arrow_schema.field(lowerCAmelCase__       ).type
                          _A=	pa_table.append_column(lowerCAmelCase__				,      pa.array([None] * len(lowerCAmelCase__       )				,      type=lowerCAmelCase__       )       )
                   # more expensive cast to support nested structures with keys in a different order
                   # allows str <-> int/float or str to Audio for example
                   _A=	table_cast(lowerCAmelCase__				,      self.config.features.arrow_schema       )
            return pa_table





     def 			a__  (					self				,      lowerCAmelCase__       ):
            for file_idx, file in enumerate(itertools.chain.from_iterable(lowerCAmelCase__       )       ):
                   # If the file is one json object and if we need to look at the list of items in one specific field
                   if self.config.field is not None:
                          with open(lowerCAmelCase__				,      encoding=self.config.encoding				,      errors=self.config.encoding_errors       ) as f:
                                 _A=	json.load(lowerCAmelCase__       )

                          # We keep only the field we are interested in
                          _A=	dataset[self.config.field]

                          # We accept two format: a list of dicts or a dict of lists
                          if isinstance(lowerCAmelCase__				,      (list, tuple)       ):
                                 _A=	set().union(*[row.keys() for row in dataset]       )
                                 _A=	{col: [row.get(lowerCAmelCase__       ) for row in dataset] for col in keys}
                          else:
                                 _A=	dataset
                          _A=	pa.Table.from_pydict(lowerCAmelCase__       )
                          yield file_idx, self._cast_table(lowerCAmelCase__       )

                   # If the file has one json object per line
                   else:
                          with open(lowerCAmelCase__				,      'rb'       ) as f:
                                 _A=	0
                                 # Use block_size equal to the chunk size divided by 32 to leverage multithreading
                                 # Set a default minimum value of 16kB if the chunk size is really small
                                 _A=	max(self.config.chunksize // 32				,      16 << 10       )
                                 _A=	(
                                     self.config.encoding_errors if self.config.encoding_errors is not None else 'strict'
                                 )
                                 while True:
                                        _A=	f.read(self.config.chunksize       )
                                        if not batch:
                                               break
                                        # Finish current line
                                        try:
                                               batch += f.readline()
                                        except (AttributeError, io.UnsupportedOperation):
                                               batch += readline(lowerCAmelCase__       )
                                        # PyArrow only accepts utf-8 encoded bytes
                                        if self.config.encoding != "utf-8":
                                               _A=	batch.decode(self.config.encoding				,      errors=lowerCAmelCase__       ).encode('utf-8'       )
                                        try:
                                               while True:
                                                      try:
                                                             _A=	paj.read_json(
                                                                 io.BytesIO(lowerCAmelCase__       )				,      read_options=paj.ReadOptions(block_size=lowerCAmelCase__       )       )
                                                             break
                                                      except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
                                                             if (
                                                                 isinstance(lowerCAmelCase__				,      pa.ArrowInvalid       )
                                                                 and "straddling" not in str(lowerCAmelCase__       )
                                                                 or block_size > len(lowerCAmelCase__       )
                                                             ):
                                                                    raise
                                                             else:
                                                                    # Increase the block size in case it was too small.
                                                                    # The block size will be reset for the next file.
                                                                    logger.debug(
                                                                        f"Batch of {len(lowerCAmelCase__       )} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."       )
                                                                    block_size *= 2
                                        except pa.ArrowInvalid as e:
                                               try:
                                                      with open(
                                                          lowerCAmelCase__				,      encoding=self.config.encoding				,      errors=self.config.encoding_errors       ) as f:
                                                             _A=	json.load(lowerCAmelCase__       )
                                               except json.JSONDecodeError:
                                                      logger.error(f"Failed to read file '{file}' with error {type(lowerCAmelCase__       )}: {e}"       )
                                                      raise e
                                               # If possible, parse the file as a list of json objects and exit the loop
                                               if isinstance(lowerCAmelCase__				,      lowerCAmelCase__       ):  # list is the only sequence type supported in JSON
                                                      try:
                                                             _A=	set().union(*[row.keys() for row in dataset]       )
                                                             _A=	{col: [row.get(lowerCAmelCase__       ) for row in dataset] for col in keys}
                                                             _A=	pa.Table.from_pydict(lowerCAmelCase__       )
                                                      except (pa.ArrowInvalid, AttributeError) as e:
                                                             logger.error(f"Failed to read file '{file}' with error {type(lowerCAmelCase__       )}: {e}"       )
                                                             raise ValueError(f"Not able to read records in the JSON file at {file}."       ) from None
                                                      yield file_idx, self._cast_table(lowerCAmelCase__       )
                                                      break
                                               else:
                                                      logger.error(f"Failed to read file '{file}' with error {type(lowerCAmelCase__       )}: {e}"       )
                                                      raise ValueError(
                                                          f"Not able to read records in the JSON file at {file}. "
                                                          f"You should probably indicate the field of the JSON file containing your records. "
                                                          f"This JSON file contain the following fields: {str(list(dataset.keys()       )       )}. "
                                                          f"Select the correct one and provide it as `field='XXX'` to the dataset loading method. "       ) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                        yield (file_idx, batch_idx), self._cast_table(lowerCAmelCase__       )
                                        batch_idx += 1
import copy
import os
from typing import Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase_								=							logging.get_logger(__name__)

UpperCAmelCase_								=							{
    '''BridgeTower/bridgetower-base''': '''https://huggingface.co/BridgeTower/bridgetower-base/blob/main/config.json''',
    '''BridgeTower/bridgetower-base-itm-mlm''': (
        '''https://huggingface.co/BridgeTower/bridgetower-base-itm-mlm/blob/main/config.json'''
    ),
}





class      lowerCAmelCase						(					_a			):
     _SCREAMING_SNAKE_CASE     :     List[Any]		="""bridgetower_vision_model"""



     def __init__(					self				,      lowerCAmelCase__=768				,      lowerCAmelCase__=12				,      lowerCAmelCase__=3				,      lowerCAmelCase__=16				,      lowerCAmelCase__=288				,      lowerCAmelCase__=1				,      lowerCAmelCase__=1E-05				,      lowerCAmelCase__=False				,      lowerCAmelCase__=True				,      lowerCAmelCase__=False				,      **lowerCAmelCase__				,      ):
            super().__init__(**lowerCAmelCase__       )
            _A=	hidden_size
            _A=	num_hidden_layers
            _A=	num_channels
            _A=	patch_size
            _A=	image_size
            _A=	initializer_factor
            _A=	layer_norm_eps
            _A=	stop_gradient
            _A=	share_layernorm
            _A=	remove_last_layer





     @classmethod
     def 			a__  (					cls				,      lowerCAmelCase__				,      **lowerCAmelCase__       ):
            _A,			_A=	cls.get_config_dict(lowerCAmelCase__				,      **lowerCAmelCase__       )

            if config_dict.get('model_type'       ) == "bridgetower":
                   _A=	config_dict['text_config']

            if "model_type" in config_dict and hasattr(cls				,      'model_type'       ) and config_dict["model_type"] != cls.model_type:
                   logger.warning(
                       f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
                       f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."       )

            return cls.from_dict(lowerCAmelCase__				,      **lowerCAmelCase__       )





class      lowerCAmelCase						(					_a			):
     _SCREAMING_SNAKE_CASE     :     List[Any]		="""bridgetower_text_model"""



     def __init__(					self				,      lowerCAmelCase__=50265				,      lowerCAmelCase__=768				,      lowerCAmelCase__=12				,      lowerCAmelCase__=12				,      lowerCAmelCase__=1				,      lowerCAmelCase__=3072				,      lowerCAmelCase__="gelu"				,      lowerCAmelCase__=0.1				,      lowerCAmelCase__=0.1				,      lowerCAmelCase__=514				,      lowerCAmelCase__=1				,      lowerCAmelCase__=1E-05				,      lowerCAmelCase__=1				,      lowerCAmelCase__=0				,      lowerCAmelCase__=2				,      lowerCAmelCase__="absolute"				,      lowerCAmelCase__=True				,      **lowerCAmelCase__				,      ):
            super().__init__(**lowerCAmelCase__       )

            _A=	vocab_size
            _A=	hidden_size
            _A=	num_hidden_layers
            _A=	num_attention_heads
            _A=	hidden_act
            _A=	initializer_factor
            _A=	intermediate_size
            _A=	hidden_dropout_prob
            _A=	attention_probs_dropout_prob
            _A=	max_position_embeddings
            _A=	type_vocab_size
            _A=	layer_norm_eps
            _A=	position_embedding_type
            _A=	use_cache
            _A=	pad_token_id
            _A=	bos_token_id
            _A=	eos_token_id





     @classmethod
     def 			a__  (					cls				,      lowerCAmelCase__				,      **lowerCAmelCase__       ):
            _A,			_A=	cls.get_config_dict(lowerCAmelCase__				,      **lowerCAmelCase__       )

            if config_dict.get('model_type'       ) == "bridgetower":
                   _A=	config_dict['text_config']

            if "model_type" in config_dict and hasattr(cls				,      'model_type'       ) and config_dict["model_type"] != cls.model_type:
                   logger.warning(
                       f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
                       f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."       )

            return cls.from_dict(lowerCAmelCase__				,      **lowerCAmelCase__       )





class      lowerCAmelCase						(					_a			):
     _SCREAMING_SNAKE_CASE     :     Dict		="""bridgetower"""



     def __init__(					self				,      lowerCAmelCase__=True				,      lowerCAmelCase__="gelu"				,      lowerCAmelCase__=768				,      lowerCAmelCase__=1				,      lowerCAmelCase__=1E-05				,      lowerCAmelCase__=False				,      lowerCAmelCase__="add"				,      lowerCAmelCase__=12				,      lowerCAmelCase__=6				,      lowerCAmelCase__=False				,      lowerCAmelCase__=False				,      lowerCAmelCase__=None				,      lowerCAmelCase__=None				,      **lowerCAmelCase__				,      ):
            # TODO: remove this once the Hub files are updated.
            _A=	kwargs.pop('text_config_dict'				,      lowerCAmelCase__       )
            _A=	kwargs.pop('vision_config_dict'				,      lowerCAmelCase__       )

            super().__init__(**lowerCAmelCase__       )
            _A=	share_cross_modal_transformer_layers
            _A=	hidden_act
            _A=	hidden_size
            _A=	initializer_factor
            _A=	layer_norm_eps
            _A=	share_link_tower_layers
            _A=	link_tower_type
            _A=	num_attention_heads
            _A=	num_hidden_layers
            _A=	tie_word_embeddings
            _A=	init_layernorm_from_vision_encoder

            if text_config is None:
                   _A=	{}
                   logger.info('`text_config` is `None`. Initializing the `BridgeTowerTextConfig` with default values.'       )

            if vision_config is None:
                   _A=	{}
                   logger.info('`vision_config` is `None`. Initializing the `BridgeTowerVisionConfig` with default values.'       )

            _A=	BridgeTowerTextConfig(**lowerCAmelCase__       )
            _A=	BridgeTowerVisionConfig(**lowerCAmelCase__       )



     @classmethod
     def 			a__  (					cls				,      lowerCAmelCase__				,      lowerCAmelCase__				,      **lowerCAmelCase__       ):
            return cls(text_config=text_config.to_dict()				,      vision_config=vision_config.to_dict()				,      **lowerCAmelCase__       )





     def 			a__  (					self       ):
            _A=	copy.deepcopy(self.__dict__       )
            _A=	self.text_config.to_dict()
            _A=	self.vision_config.to_dict()
            _A=	self.__class__.model_type
            return output