
import os
import socket
from contextlib import contextmanager

import torch

from ..commands.config.default import write_basic_config  # noqa: F401
from ..state import PartialState
from .dataclasses import DistributedType
from .imports import is_deepspeed_available, is_tpu_available
from .transformer_engine import convert_model
from .versions import is_torch_version


if is_deepspeed_available():
    from deepspeed import DeepSpeedEngine

if is_tpu_available(check_device=False):
    import torch_xla.core.xla_model as xm


def 							SCREAMING_SNAKE_CASE_      (			UpperCamelCase__		):
      if is_torch_version('''<''' ,		'''2.0.0'''		) or not hasattr(UpperCamelCase__ ,		'''_dynamo'''		):
            return False
      return isinstance(UpperCamelCase__ ,		torch._dynamo.eval_frame.OptimizedModule		)


def 							SCREAMING_SNAKE_CASE_      (			UpperCamelCase__ ,		UpperCamelCase__ = True		):
      UpperCamelCase__    :   Optional[Any]					=				(torch.nn.parallel.DistributedDataParallel, torch.nn.DataParallel)

      UpperCamelCase__    :   Optional[Any]					=				is_compiled_module(UpperCamelCase__		)
      if is_compiled:
            UpperCamelCase__    :   Any					=				model
            UpperCamelCase__    :   Dict					=				model._orig_mod

      if is_deepspeed_available():
            options += (DeepSpeedEngine,)

      while isinstance(UpperCamelCase__ ,		UpperCamelCase__		):
            UpperCamelCase__    :   Union[str, Any]					=				model.module

      if not keep_fpaa_wrapper:
            UpperCamelCase__    :   str					=				getattr(UpperCamelCase__ ,		'''forward'''		)
            UpperCamelCase__    :   int					=				model.__dict__.pop('''_original_forward''' ,		UpperCamelCase__		)
            if original_forward is not None:
                  while hasattr(UpperCamelCase__ ,		'''__wrapped__'''		):
                        UpperCamelCase__    :   List[str]					=				forward.__wrapped__
                        if forward == original_forward:
                              break
                  UpperCamelCase__    :   Tuple					=				forward
            if getattr(UpperCamelCase__ ,		'''_converted_to_transformer_engine''' ,		UpperCamelCase__		):
                  convert_model(UpperCamelCase__ ,		to_transformer_engine=UpperCamelCase__		)

      if is_compiled:
            UpperCamelCase__    :   Union[str, Any]					=				model
            UpperCamelCase__    :   str					=				compiled_model

      return model


def 							SCREAMING_SNAKE_CASE_      (			):
      PartialState().wait_for_everyone()


def 							SCREAMING_SNAKE_CASE_      (			UpperCamelCase__ ,		UpperCamelCase__		):
      if PartialState().distributed_type == DistributedType.TPU:
            xm.save(UpperCamelCase__ ,		UpperCamelCase__		)
      elif PartialState().local_process_index == 0:
            torch.save(UpperCamelCase__ ,		UpperCamelCase__		)


@contextmanager
def 							SCREAMING_SNAKE_CASE_      (			**UpperCamelCase__		):
      for key, value in kwargs.items():
            UpperCamelCase__    :   List[Any]					=				str(UpperCamelCase__		)

      yield

      for key in kwargs:
            if key.upper() in os.environ:
                  del os.environ[key.upper()]


def 							SCREAMING_SNAKE_CASE_      (			UpperCamelCase__		):
      if not hasattr(UpperCamelCase__ ,		'''__qualname__'''		) and not hasattr(UpperCamelCase__ ,		'''__name__'''		):
            UpperCamelCase__    :   Tuple					=				getattr(UpperCamelCase__ ,		'''__class__''' ,		UpperCamelCase__		)
      if hasattr(UpperCamelCase__ ,		'''__qualname__'''		):
            return obj.__qualname__
      if hasattr(UpperCamelCase__ ,		'''__name__'''		):
            return obj.__name__
      return str(UpperCamelCase__		)


def 							SCREAMING_SNAKE_CASE_      (			UpperCamelCase__ ,		UpperCamelCase__		):
      for key, value in source.items():
            if isinstance(UpperCamelCase__ ,		UpperCamelCase__		):
                  UpperCamelCase__    :   Optional[Any]					=				destination.setdefault(UpperCamelCase__ ,		{}		)
                  merge_dicts(UpperCamelCase__ ,		UpperCamelCase__		)
            else:
                  UpperCamelCase__    :   int					=				value

      return destination


def 							SCREAMING_SNAKE_CASE_      (			UpperCamelCase__ = None		):
      if port is None:
            UpperCamelCase__    :   List[Any]					=				2_9_5_0_0
      with socket.socket(socket.AF_INET ,		socket.SOCK_STREAM		) as s:
            return s.connect_ex(('''localhost''', port)		) == 0




from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase      =logging.get_logger(__name__)

lowerCamelCase      ={
    "google/realm-cc-news-pretrained-embedder": (
        "https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/config.json"
    ),
    "google/realm-cc-news-pretrained-encoder": (
        "https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/config.json"
    ),
    "google/realm-cc-news-pretrained-scorer": (
        "https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/config.json"
    ),
    "google/realm-cc-news-pretrained-openqa": (
        "https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/config.json"
    ),
    "google/realm-orqa-nq-openqa": "https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/config.json",
    "google/realm-orqa-nq-reader": "https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/config.json",
    "google/realm-orqa-wq-openqa": "https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/config.json",
    "google/realm-orqa-wq-reader": "https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/config.json",
    # See all REALM models at https://huggingface.co/models?filter=realm
}




class 				_lowerCamelCase (      UpperCamelCase_ ):





     """simple docstring"""







     SCREAMING_SNAKE_CASE_		   =  '''realm'''






     def __init__(							self						,						__SCREAMING_SNAKE_CASE=3_0_5_2_2						,						__SCREAMING_SNAKE_CASE=7_6_8						,						__SCREAMING_SNAKE_CASE=1_2_8						,						__SCREAMING_SNAKE_CASE=1_2						,						__SCREAMING_SNAKE_CASE=1_2						,						__SCREAMING_SNAKE_CASE=8						,						__SCREAMING_SNAKE_CASE=3_0_7_2						,						__SCREAMING_SNAKE_CASE="gelu_new"						,						__SCREAMING_SNAKE_CASE=0.1						,						__SCREAMING_SNAKE_CASE=0.1						,						__SCREAMING_SNAKE_CASE=5_1_2						,						__SCREAMING_SNAKE_CASE=2						,						__SCREAMING_SNAKE_CASE=0.02						,						__SCREAMING_SNAKE_CASE=1e-12						,						__SCREAMING_SNAKE_CASE=2_5_6						,						__SCREAMING_SNAKE_CASE=1_0						,						__SCREAMING_SNAKE_CASE=1e-3						,						__SCREAMING_SNAKE_CASE=5						,						__SCREAMING_SNAKE_CASE=3_2_0						,						__SCREAMING_SNAKE_CASE=1_3_3_5_3_7_1_8						,						__SCREAMING_SNAKE_CASE=5_0_0_0						,						__SCREAMING_SNAKE_CASE=1						,						__SCREAMING_SNAKE_CASE=0						,						__SCREAMING_SNAKE_CASE=2						,						**__SCREAMING_SNAKE_CASE						,						)  ->   List[str]:




           """simple docstring"""







           super().__init__(pad_token_id=__SCREAMING_SNAKE_CASE						,						bos_token_id=__SCREAMING_SNAKE_CASE						,						eos_token_id=__SCREAMING_SNAKE_CASE						,						**__SCREAMING_SNAKE_CASE  )

           # Common config
           UpperCamelCase__    :   int					=				vocab_size
           UpperCamelCase__    :   Any					=				max_position_embeddings
           UpperCamelCase__    :   List[str]					=				hidden_size
           UpperCamelCase__    :   Union[str, Any]					=				retriever_proj_size
           UpperCamelCase__    :   int					=				num_hidden_layers
           UpperCamelCase__    :   Union[str, Any]					=				num_attention_heads
           UpperCamelCase__    :   str					=				num_candidates
           UpperCamelCase__    :   int					=				intermediate_size
           UpperCamelCase__    :   Dict					=				hidden_act
           UpperCamelCase__    :   Any					=				hidden_dropout_prob
           UpperCamelCase__    :   Any					=				attention_probs_dropout_prob
           UpperCamelCase__    :   Any					=				initializer_range
           UpperCamelCase__    :   Optional[Any]					=				type_vocab_size
           UpperCamelCase__    :   List[Any]					=				layer_norm_eps

           # Reader config
           UpperCamelCase__    :   List[Any]					=				span_hidden_size
           UpperCamelCase__    :   List[Any]					=				max_span_width
           UpperCamelCase__    :   Optional[Any]					=				reader_layer_norm_eps
           UpperCamelCase__    :   Optional[Any]					=				reader_beam_size
           UpperCamelCase__    :   int					=				reader_seq_len

           # Retrieval config
           UpperCamelCase__    :   List[str]					=				num_block_records
           UpperCamelCase__    :   Union[str, Any]					=				searcher_beam_size


