


import os
from pickle import UnpicklingError
from typing import Dict, Tuple

import jax
import jax.numpy as jnp
import numpy as np
from flax.serialization import from_bytes
from flax.traverse_util import flatten_dict, unflatten_dict

import transformers

from .utils import logging


A   :					List[Any]	=      logging.get_logger(__name__)

def    lowercase_	(	_A    :					List[Any]					,      _A    :					List[Any]					,      _A    :					Optional[Any]					,      _A    :					Optional[int]=False		):

      """simple docstring"""




      try:
            import torch  # noqa: F401
      except ImportError:
            logger.error(
                "Loading a PyTorch model in Flax, requires both PyTorch and Flax to be installed. Please see"
                " https://pytorch.org/ and https://flax.readthedocs.io/en/latest/installation.html for installation"
                " instructions."		)
            raise

      if not is_sharded:
            lowerCamelCase__	:       Optional[Any]					=						os.path.abspath(_A		)
            logger.info(F"Loading PyTorch weights from {pt_path}"		)

            lowerCamelCase__	:       Union[str, Any]					=						torch.load(_A					,      map_location="cpu"		)
            logger.info(F"PyTorch checkpoint contains {sum(t.numel() for t in pt_state_dict.values()		):,} parameters."		)

            lowerCamelCase__	:       str					=						convert_pytorch_state_dict_to_flax(_A					,      _A		)
      else:
            # model is sharded and pytorch_checkpoint_path already contains the list of .pt shard files
            lowerCamelCase__	:       List[str]					=						convert_pytorch_sharded_state_dict_to_flax(_A					,      _A		)
      return flax_state_dict

def    lowercase_	(	_A    :					Tuple[str]					,      _A    :					np.ndarray					,      _A    :					Dict[str, jnp.ndarray]					,      _A    :					str					,      ):

      """simple docstring"""




      def is_key_or_prefix_key_in_dict(_A    :					Tuple[str]		) -> bool:
            return len(set(_A		) & {key, (model_prefix,) + key}		) > 0

      # layer norm
      lowerCamelCase__	:       Union[str, Any]					=						pt_tuple_key[:-1] + ("scale",)
      if pt_tuple_key[-1] in ["weight", "gamma"] and is_key_or_prefix_key_in_dict(_A		):
            return renamed_pt_tuple_key, pt_tensor

      # batch norm layer mean
      lowerCamelCase__	:       Optional[Any]					=						pt_tuple_key[:-1] + ("mean",)
      if pt_tuple_key[-1] == "running_mean" and not is_key_or_prefix_key_in_dict(_A		):
            return renamed_pt_tuple_key, pt_tensor

      # batch norm layer var
      lowerCamelCase__	:       int					=						pt_tuple_key[:-1] + ("var",)
      if pt_tuple_key[-1] == "running_var" and not is_key_or_prefix_key_in_dict(_A		):
            return renamed_pt_tuple_key, pt_tensor

      # embedding
      lowerCamelCase__	:       str					=						pt_tuple_key[:-1] + ("embedding",)
      if pt_tuple_key[-1] == "weight" and is_key_or_prefix_key_in_dict(_A		):
            return renamed_pt_tuple_key, pt_tensor

      # conv layer
      lowerCamelCase__	:       Tuple					=						pt_tuple_key[:-1] + ("kernel",)
      if pt_tuple_key[-1] == "weight" and pt_tensor.ndim == 4 and not is_key_or_prefix_key_in_dict(_A		):
            lowerCamelCase__	:       Optional[int]					=						pt_tensor.transpose(2					,      3					,      1					,      0		)
            return renamed_pt_tuple_key, pt_tensor

      # linear layer
      lowerCamelCase__	:       Dict					=						pt_tuple_key[:-1] + ("kernel",)
      if pt_tuple_key[-1] == "weight" and not is_key_or_prefix_key_in_dict(_A		):
            lowerCamelCase__	:       int					=						pt_tensor.T
            return renamed_pt_tuple_key, pt_tensor

      # old PyTorch layer norm weight
      lowerCamelCase__	:       Optional[int]					=						pt_tuple_key[:-1] + ("weight",)
      if pt_tuple_key[-1] == "gamma":
            return renamed_pt_tuple_key, pt_tensor

      # old PyTorch layer norm bias
      lowerCamelCase__	:       Dict					=						pt_tuple_key[:-1] + ("bias",)
      if pt_tuple_key[-1] == "beta":
            return renamed_pt_tuple_key, pt_tensor

      # New `weight_norm` from https://github.com/huggingface/transformers/pull/24030
      lowerCamelCase__	:       Optional[int]					=						None
      if pt_tuple_key[-3::2] == ("parametrizations", "original0"):
            lowerCamelCase__	:       Optional[Any]					=						pt_tuple_key[-2] + "_g"
      elif pt_tuple_key[-3::2] == ("parametrizations", "original1"):
            lowerCamelCase__	:       Optional[int]					=						pt_tuple_key[-2] + "_v"
      if name is not None:
            lowerCamelCase__	:       Optional[Any]					=						pt_tuple_key[:-3] + (name,)
            return renamed_pt_tuple_key, pt_tensor

      return pt_tuple_key, pt_tensor

def    lowercase_	(	_A    :					Tuple					,      _A    :					Optional[int]		):

      """simple docstring"""




      lowerCamelCase__	:       List[str]					=						{k: v.numpy() for k, v in pt_state_dict.items()}

      lowerCamelCase__	:       List[Any]					=						flax_model.base_model_prefix

      # use params dict if the model contains batch norm layers
      if "params" in flax_model.params:
            lowerCamelCase__	:       List[Any]					=						flax_model.params["params"]
      else:
            lowerCamelCase__	:       List[str]					=						flax_model.params
      lowerCamelCase__	:       int					=						flatten_dict(_A		)

      # add batch_stats keys,values to dict
      if "batch_stats" in flax_model.params:
            lowerCamelCase__	:       Optional[int]					=						flatten_dict(flax_model.params["batch_stats"]		)
            random_flax_state_dict.update(_A		)

      lowerCamelCase__	:       Any					=						{}

      lowerCamelCase__	:       List[str]					=						(model_prefix not in flax_model_params) and (
          model_prefix in {k.split("."		)[0] for k in pt_state_dict.keys()}
      )
      lowerCamelCase__	:       int					=						(model_prefix in flax_model_params) and (
          model_prefix not in {k.split("."		)[0] for k in pt_state_dict.keys()}
      )

      # Need to change some parameters name to match Flax names
      for pt_key, pt_tensor in pt_state_dict.items():
            lowerCamelCase__	:       Any					=						tuple(pt_key.split("."		)		)

            # remove base model prefix if necessary
            lowerCamelCase__	:       Tuple					=						pt_tuple_key[0] == model_prefix
            if load_model_with_head_into_base_model and has_base_model_prefix:
                  lowerCamelCase__	:       List[str]					=						pt_tuple_key[1:]

            # Correctly rename weight parameters
            lowerCamelCase__					,     lowerCamelCase__	:       List[Any]					=						rename_key_and_reshape_tensor(
                _A					,      _A					,      _A					,      _A		)

            # add model prefix if necessary
            lowerCamelCase__	:       int					=						(model_prefix,) + flax_key in random_flax_state_dict
            if load_base_model_into_model_with_head and require_base_model_prefix:
                  lowerCamelCase__	:       List[str]					=						(model_prefix,) + flax_key

            if flax_key in random_flax_state_dict:
                  if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
                        raise ValueError(
                            F"PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape "
                            F"{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}."		)

        # add batch stats if the model contains batchnorm layers
            if "batch_stats" in flax_model.params:
                  if "mean" in flax_key[-1] or "var" in flax_key[-1]:
                        lowerCamelCase__	:       str					=						jnp.asarray(_A		)
                        continue
                  # remove num_batches_tracked key
                  if "num_batches_tracked" in flax_key[-1]:
                        flax_state_dict.pop(_A					,      _A		)
                        continue

                  # also add unexpected weight so that warning is thrown
                  lowerCamelCase__	:       Optional[int]					=						jnp.asarray(_A		)

            else:
                  # also add unexpected weight so that warning is thrown
                  lowerCamelCase__	:       Tuple					=						jnp.asarray(_A		)

      return unflatten_dict(_A		)

def    lowercase_	(	_A    :					List[Any]					,      _A    :					Dict		):

      """simple docstring"""




      import torch

      # Load the index
      lowerCamelCase__	:       Union[str, Any]					=						{}
      for shard_file in shard_filenames:
            # load using msgpack utils
            lowerCamelCase__	:       int					=						torch.load(_A		)
            lowerCamelCase__	:       Tuple					=						{k: v.numpy() for k, v in pt_state_dict.items()}

            lowerCamelCase__	:       int					=						flax_model.base_model_prefix

            # use params dict if the model contains batch norm layers and then add batch_stats keys,values to dict
            if "batch_stats" in flax_model.params:
                  lowerCamelCase__	:       int					=						flax_model.params["params"]

                  lowerCamelCase__	:       int					=						flatten_dict(_A		)
                  random_flax_state_dict.update(flatten_dict(flax_model.params["batch_stats"]		)		)
            else:
                  lowerCamelCase__	:       Union[str, Any]					=						flax_model.params
                  lowerCamelCase__	:       int					=						flatten_dict(_A		)

            lowerCamelCase__	:       Optional[Any]					=						(model_prefix not in flax_model_params) and (
                model_prefix in {k.split("."		)[0] for k in pt_state_dict.keys()}
            )
            lowerCamelCase__	:       Union[str, Any]					=						(model_prefix in flax_model_params) and (
                model_prefix not in {k.split("."		)[0] for k in pt_state_dict.keys()}
            )
            # Need to change some parameters name to match Flax names
            for pt_key, pt_tensor in pt_state_dict.items():
                  lowerCamelCase__	:       Optional[Any]					=						tuple(pt_key.split("."		)		)

                  # remove base model prefix if necessary
                  lowerCamelCase__	:       Dict					=						pt_tuple_key[0] == model_prefix
                  if load_model_with_head_into_base_model and has_base_model_prefix:
                        lowerCamelCase__	:       str					=						pt_tuple_key[1:]

                  # Correctly rename weight parameters
                  lowerCamelCase__					,     lowerCamelCase__	:       Union[str, Any]					=						rename_key_and_reshape_tensor(
                      _A					,      _A					,      _A					,      _A		)
                  # add model prefix if necessary
                  lowerCamelCase__	:       Optional[Any]					=						(model_prefix,) + flax_key in random_flax_state_dict
                  if load_base_model_into_model_with_head and require_base_model_prefix:
                        lowerCamelCase__	:       Optional[int]					=						(model_prefix,) + flax_key

                  if flax_key in random_flax_state_dict:
                        if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
                              raise ValueError(
                                  F"PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape "
                                  F"{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}."		)

            # add batch stats if the model contains batchnorm layers
                  if "batch_stats" in flax_model.params:
                        if "mean" in flax_key[-1]:
                              lowerCamelCase__	:       Dict					=						jnp.asarray(_A		)
                              continue
                        if "var" in flax_key[-1]:
                              lowerCamelCase__	:       int					=						jnp.asarray(_A		)
                              continue
                        # remove num_batches_tracked key
                        if "num_batches_tracked" in flax_key[-1]:
                              flax_state_dict.pop(_A					,      _A		)
                              continue

                        # also add unexpected weight so that warning is thrown
                        lowerCamelCase__	:       Tuple					=						jnp.asarray(_A		)

                  else:
                        # also add unexpected weight so that warning is thrown
                        lowerCamelCase__	:       List[str]					=						jnp.asarray(_A		)
      return unflatten_dict(_A		)

def    lowercase_	(	_A    :					Union[str, Any]					,      _A    :					str		):

      """simple docstring"""




      lowerCamelCase__	:       List[str]					=						os.path.abspath(_A		)
      logger.info(F"Loading Flax weights from {flax_checkpoint_path}"		)

      # import correct flax class
      lowerCamelCase__	:       Tuple					=						getattr(_A					,      "Flax" + model.__class__.__name__		)

      # load flax weight dict
      with open(_A					,      "rb"		) as state_f:
            try:
                  lowerCamelCase__	:       List[Any]					=						from_bytes(_A					,      state_f.read()		)
            except UnpicklingError:
                  raise EnvironmentError(F"Unable to convert {flax_checkpoint_path} to Flax deserializable object. "		)

      return load_flax_weights_in_pytorch_model(_A					,      _A		)


def    lowercase_	(	_A    :					Optional[int]					,      _A    :					Union[str, Any]		):

      """simple docstring"""




      try:
            import torch  # noqa: F401
      except ImportError:
            logger.error(
                "Loading a Flax weights in PyTorch, requires both PyTorch and Flax to be installed. Please see"
                " https://pytorch.org/ and https://flax.readthedocs.io/en/latest/installation.html for installation"
                " instructions."		)
            raise

      # check if we have bf16 weights
      lowerCamelCase__	:       List[Any]					=						flatten_dict(jax.tree_util.tree_map(lambda _A		: x.dtype == jnp.bfloataa					,      _A		)		).values()
      if any(_A		):
            # convert all weights to fp32 if the are bf16 since torch.from_numpy can-not handle bf16
            # and bf16 is not fully supported in PT yet.
            logger.warning(
                "Found ``bfloat16`` weights in Flax model. Casting all ``bfloat16`` weights to ``float32`` "
                "before loading those in PyTorch model."		)
            lowerCamelCase__	:       Any					=						jax.tree_util.tree_map(
                lambda _A		: params.astype(np.floataa		) if params.dtype == jnp.bfloataa else params					,      _A		)

      lowerCamelCase__	:       int					=						flatten_dict(_A		)
      lowerCamelCase__	:       Tuple					=						pt_model.state_dict()

      lowerCamelCase__	:       Dict					=						(pt_model.base_model_prefix in flax_state) and (
          pt_model.base_model_prefix not in {k.split("."		)[0] for k in pt_model_dict.keys()}
      )
      lowerCamelCase__	:       str					=						(pt_model.base_model_prefix not in flax_state) and (
          pt_model.base_model_prefix in {k.split("."		)[0] for k in pt_model_dict.keys()}
      )

      # keep track of unexpected & missing keys
      lowerCamelCase__	:       Union[str, Any]					=						[]
      lowerCamelCase__	:       List[str]					=						set(pt_model_dict.keys()		)

      for flax_key_tuple, flax_tensor in flax_state_dict.items():
            lowerCamelCase__	:       Union[str, Any]					=						flax_key_tuple[0] == pt_model.base_model_prefix
            lowerCamelCase__	:       List[Any]					=						".".join((pt_model.base_model_prefix,) + flax_key_tuple		) in pt_model_dict

            # adapt flax_key to prepare for loading from/to base model only
            if load_model_with_head_into_base_model and has_base_model_prefix:
                  lowerCamelCase__	:       Dict					=						flax_key_tuple[1:]
            elif load_base_model_into_model_with_head and require_base_model_prefix:
                  lowerCamelCase__	:       List[str]					=						(pt_model.base_model_prefix,) + flax_key_tuple

            # rename flax weights to PyTorch format
            if flax_key_tuple[-1] == "kernel" and flax_tensor.ndim == 4 and ".".join(_A		) not in pt_model_dict:
                  # conv layer
                  lowerCamelCase__	:       Dict					=						flax_key_tuple[:-1] + ("weight",)
                  lowerCamelCase__	:       str					=						jnp.transpose(_A					,      (3, 2, 0, 1)		)
            elif flax_key_tuple[-1] == "kernel" and ".".join(_A		) not in pt_model_dict:
                  # linear layer
                  lowerCamelCase__	:       Any					=						flax_key_tuple[:-1] + ("weight",)
                  lowerCamelCase__	:       Optional[Any]					=						flax_tensor.T
            elif flax_key_tuple[-1] in ["scale", "embedding"]:
                  lowerCamelCase__	:       Optional[Any]					=						flax_key_tuple[:-1] + ("weight",)

            # adding batch stats from flax batch norm to pt
            elif "mean" in flax_key_tuple[-1]:
                  lowerCamelCase__	:       Union[str, Any]					=						flax_key_tuple[:-1] + ("running_mean",)
            elif "var" in flax_key_tuple[-1]:
                  lowerCamelCase__	:       Union[str, Any]					=						flax_key_tuple[:-1] + ("running_var",)

            if "batch_stats" in flax_state:
                  lowerCamelCase__	:       Optional[Any]					=						".".join(flax_key_tuple[1:]		)  # Remove the params/batch_stats header
            else:
                  lowerCamelCase__	:       int					=						".".join(_A		)

            # We also need to look at `pt_model_dict` and see if there are keys requiring further transformation.
            lowerCamelCase__	:       List[Any]					=						{}
            # New `weight_norm` from https://github.com/huggingface/transformers/pull/24030
            for key in pt_model_dict:
                  lowerCamelCase__	:       Optional[Any]					=						key.split("."		)
                  lowerCamelCase__	:       Optional[Any]					=						None
                  if key_components[-3::2] == ["parametrizations", "original0"]:
                        lowerCamelCase__	:       str					=						key_components[-2] + "_g"
                  elif key_components[-3::2] == ["parametrizations", "original1"]:
                        lowerCamelCase__	:       int					=						key_components[-2] + "_v"
                  if name is not None:
                        lowerCamelCase__	:       Tuple					=						key_components[:-3] + [name]
                        lowerCamelCase__	:       str					=						".".join(_A		)
                        lowerCamelCase__	:       Dict					=						key

            if flax_key in special_pt_names:
                  lowerCamelCase__	:       str					=						special_pt_names[flax_key]

            if flax_key in pt_model_dict:
                  if flax_tensor.shape != pt_model_dict[flax_key].shape:
                        raise ValueError(
                            F"Flax checkpoint seems to be incorrect. Weight {flax_key_tuple} was expected "
                            F"to be of shape {pt_model_dict[flax_key].shape}, but is {flax_tensor.shape}."		)
                  else:
                        # add weight to pytorch dict
                        lowerCamelCase__	:       Dict					=						np.asarray(_A		) if not isinstance(_A					,      np.ndarray		) else flax_tensor
                        lowerCamelCase__	:       Tuple					=						torch.from_numpy(_A		)
                        # remove from missing keys
                        missing_keys.remove(_A		)
            else:
                  # weight is not expected by PyTorch model
                  unexpected_keys.append(_A		)

      pt_model.load_state_dict(_A		)

      # re-transform missing_keys to list
      lowerCamelCase__	:       Optional[int]					=						list(_A		)

      if len(_A		) > 0:
            logger.warning(
                "Some weights of the Flax model were not used when initializing the PyTorch model"
                F" {pt_model.__class__.__name__}: {unexpected_keys}\n- This IS expected if you are initializing"
                F" {pt_model.__class__.__name__} from a Flax model trained on another task or with another architecture"
                " (e.g. initializing a BertForSequenceClassification model from a FlaxBertForPreTraining model).\n- This"
                F" IS NOT expected if you are initializing {pt_model.__class__.__name__} from a Flax model that you expect"
                " to be exactly identical (e.g. initializing a BertForSequenceClassification model from a"
                " FlaxBertForSequenceClassification model)."		)
      else:
            logger.warning(F"All Flax model weights were used when initializing {pt_model.__class__.__name__}.\n"		)
      if len(_A		) > 0:
            logger.warning(
                F"Some weights of {pt_model.__class__.__name__} were not initialized from the Flax model and are newly"
                F" initialized: {missing_keys}\nYou should probably TRAIN this model on a down-stream task to be able to"
                " use it for predictions and inference."		)
      else:
            logger.warning(
                F"All the weights of {pt_model.__class__.__name__} were initialized from the Flax model.\n"
                "If your task is similar to the task the model of the checkpoint was trained on, "
                F"you can already use {pt_model.__class__.__name__} for predictions without further training."		)

      return pt_model






import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import (
        OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
        OpenAIGPTConfig,
        OpenAIGPTDoubleHeadsModel,
        OpenAIGPTForSequenceClassification,
        OpenAIGPTLMHeadModel,
        OpenAIGPTModel,
    )







class     _lowercase				:


       """simple docstring"""




       def __init__(     self							:    Dict      ,   __lowerCamelCase							:    str      ,   __lowerCamelCase							:    Optional[int]=13      ,   __lowerCamelCase							:    List[str]=7      ,   __lowerCamelCase							:    Tuple=True      ,   __lowerCamelCase							:    Optional[int]=True      ,   __lowerCamelCase							:    List[str]=True      ,   __lowerCamelCase							:    Union[str, Any]=99      ,   __lowerCamelCase							:    List[Any]=32      ,   __lowerCamelCase							:    List[Any]=5      ,   __lowerCamelCase							:    Optional[Any]=4      ,   __lowerCamelCase							:    Optional[int]=37      ,   __lowerCamelCase							:    List[str]="gelu"      ,   __lowerCamelCase							:    List[str]=0.1      ,   __lowerCamelCase							:    int=0.1      ,   __lowerCamelCase							:    List[str]=512      ,   __lowerCamelCase							:    Optional[Any]=16      ,   __lowerCamelCase							:    Optional[Any]=2      ,   __lowerCamelCase							:    str=0.0_2      ,   __lowerCamelCase							:    List[str]=3      ,   __lowerCamelCase							:    Tuple=4      ,   __lowerCamelCase							:    Optional[int]=None      ,   ):


             '''simple docstring'''



             lowerCamelCase__	:       Tuple					=						parent
             lowerCamelCase__	:       int					=						batch_size
             lowerCamelCase__	:       List[Any]					=						seq_length
             lowerCamelCase__	:       Union[str, Any]					=						is_training
             lowerCamelCase__	:       Any					=						use_token_type_ids
             lowerCamelCase__	:       Union[str, Any]					=						use_labels
             lowerCamelCase__	:       List[str]					=						vocab_size
             lowerCamelCase__	:       Union[str, Any]					=						hidden_size
             lowerCamelCase__	:       List[Any]					=						num_hidden_layers
             lowerCamelCase__	:       Optional[Any]					=						num_attention_heads
             lowerCamelCase__	:       Any					=						intermediate_size
             lowerCamelCase__	:       str					=						hidden_act
             lowerCamelCase__	:       str					=						hidden_dropout_prob
             lowerCamelCase__	:       Any					=						attention_probs_dropout_prob
             lowerCamelCase__	:       List[str]					=						max_position_embeddings
             lowerCamelCase__	:       Optional[int]					=						type_vocab_size
             lowerCamelCase__	:       List[Any]					=						type_sequence_label_size
             lowerCamelCase__	:       List[str]					=						initializer_range
             lowerCamelCase__	:       List[str]					=						num_labels
             lowerCamelCase__	:       List[Any]					=						num_choices
             lowerCamelCase__	:       Optional[Any]					=						scope
             lowerCamelCase__	:       List[Any]					=						self.vocab_size - 1





       def 		lowerCAmelCase  (     self							:    List[Any] ):


             '''simple docstring'''



             lowerCamelCase__	:       Optional[int]					=						ids_tensor([self.batch_size, self.seq_length]      ,   self.vocab_size )

             lowerCamelCase__	:       Optional[Any]					=						None
             if self.use_token_type_ids:
                   lowerCamelCase__	:       Optional[Any]					=						ids_tensor([self.batch_size, self.seq_length]      ,   self.type_vocab_size )

             lowerCamelCase__	:       Any					=						None
             lowerCamelCase__	:       str					=						None
             lowerCamelCase__	:       str					=						None
             if self.use_labels:
                   lowerCamelCase__	:       Dict					=						ids_tensor([self.batch_size]      ,   self.type_sequence_label_size )
                   lowerCamelCase__	:       Dict					=						ids_tensor([self.batch_size, self.seq_length]      ,   self.num_labels )
                   lowerCamelCase__	:       Dict					=						ids_tensor([self.batch_size]      ,   self.num_choices )

             lowerCamelCase__	:       Union[str, Any]					=						OpenAIGPTConfig(
                 vocab_size=self.vocab_size      ,   n_embd=self.hidden_size      ,   n_layer=self.num_hidden_layers      ,   n_head=self.num_attention_heads      ,   n_positions=self.max_position_embeddings      ,   pad_token_id=self.pad_token_id      ,   )

             lowerCamelCase__	:       Optional[int]					=						ids_tensor([self.num_hidden_layers, self.num_attention_heads]      ,   2 )

             return (
                 config,
                 input_ids,
                 head_mask,
                 token_type_ids,
                 sequence_labels,
                 token_labels,
                 choice_labels,
             )





       def 		lowerCAmelCase  (     self							:    str      ,   __lowerCamelCase							:    int      ,   __lowerCamelCase							:    int      ,   __lowerCamelCase							:    Optional[int]      ,   __lowerCamelCase							:    int      ,   *__lowerCamelCase							:    List[Any] ):


             '''simple docstring'''



             lowerCamelCase__	:       Optional[int]					=						OpenAIGPTModel(config=__lowerCamelCase )
             model.to(__lowerCamelCase )
             model.eval()

             lowerCamelCase__	:       Tuple					=						model(__lowerCamelCase      ,   token_type_ids=__lowerCamelCase      ,   head_mask=__lowerCamelCase )
             lowerCamelCase__	:       str					=						model(__lowerCamelCase      ,   token_type_ids=__lowerCamelCase )
             lowerCamelCase__	:       Optional[int]					=						model(__lowerCamelCase )

             self.parent.assertEqual(result.last_hidden_state.shape      ,   (self.batch_size, self.seq_length, self.hidden_size) )





       def 		lowerCAmelCase  (     self							:    str      ,   __lowerCamelCase							:    List[Any]      ,   __lowerCamelCase							:    Union[str, Any]      ,   __lowerCamelCase							:    List[str]      ,   __lowerCamelCase							:    Any      ,   *__lowerCamelCase							:    Optional[int] ):


             '''simple docstring'''



             lowerCamelCase__	:       Tuple					=						OpenAIGPTLMHeadModel(__lowerCamelCase )
             model.to(__lowerCamelCase )
             model.eval()

             lowerCamelCase__	:       List[str]					=						model(__lowerCamelCase      ,   token_type_ids=__lowerCamelCase      ,   labels=__lowerCamelCase )
             self.parent.assertEqual(result.loss.shape      ,   () )
             self.parent.assertEqual(result.logits.shape      ,   (self.batch_size, self.seq_length, self.vocab_size) )





       def 		lowerCAmelCase  (     self							:    Dict      ,   __lowerCamelCase							:    Any      ,   __lowerCamelCase							:    Union[str, Any]      ,   __lowerCamelCase							:    Optional[int]      ,   __lowerCamelCase							:    Optional[int]      ,   *__lowerCamelCase							:    Tuple ):


             '''simple docstring'''



             lowerCamelCase__	:       List[Any]					=						OpenAIGPTDoubleHeadsModel(__lowerCamelCase )
             model.to(__lowerCamelCase )
             model.eval()

             lowerCamelCase__	:       Optional[Any]					=						model(__lowerCamelCase      ,   token_type_ids=__lowerCamelCase      ,   labels=__lowerCamelCase )
             self.parent.assertEqual(result.loss.shape      ,   () )
             self.parent.assertEqual(result.logits.shape      ,   (self.batch_size, self.seq_length, self.vocab_size) )





       def 		lowerCAmelCase  (     self							:    Tuple      ,   __lowerCamelCase							:    Union[str, Any]      ,   __lowerCamelCase							:    Union[str, Any]      ,   __lowerCamelCase							:    Optional[Any]      ,   __lowerCamelCase							:    List[Any]      ,   *__lowerCamelCase							:    Optional[int] ):


             '''simple docstring'''



             lowerCamelCase__	:       Dict					=						self.num_labels
             lowerCamelCase__	:       Tuple					=						OpenAIGPTForSequenceClassification(__lowerCamelCase )
             model.to(__lowerCamelCase )
             model.eval()

             lowerCamelCase__	:       Dict					=						ids_tensor([self.batch_size]      ,   self.type_sequence_label_size )
             lowerCamelCase__	:       List[str]					=						model(__lowerCamelCase      ,   token_type_ids=__lowerCamelCase      ,   labels=__lowerCamelCase )
             self.parent.assertEqual(result.logits.shape      ,   (self.batch_size, self.num_labels) )





       def 		lowerCAmelCase  (     self							:    List[str] ):


             '''simple docstring'''



             lowerCamelCase__	:       str					=						self.prepare_config_and_inputs()
             (
                 (
                 lowerCamelCase__
             )					,     (
                 lowerCamelCase__
             )					,     (
                 lowerCamelCase__
             )					,     (
                 lowerCamelCase__
             )					,     (
                 lowerCamelCase__
             )					,     (
                 lowerCamelCase__
             )					,     (
                 lowerCamelCase__
             )					,     
             )	:       Any					=						config_and_inputs
             lowerCamelCase__	:       Union[str, Any]					=						{
                 "input_ids": input_ids,
                 "token_type_ids": token_type_ids,
                 "head_mask": head_mask,
             }

             return config, inputs_dict







@require_torch
class     _lowercase				(							lowercase__							,						lowercase__							,						lowercase__							,						unittest.TestCase):


       """simple docstring"""




       A__      		=    (
           (OpenAIGPTModel, OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel, OpenAIGPTForSequenceClassification)
           if is_torch_available()
           else ()
       )
       A__      		=    (
           (OpenAIGPTLMHeadModel,) if is_torch_available() else ()
       )  # TODO (PVP): Add Double HeadsModel when generate() function is changed accordingly
       A__      		=    (
           {
               "feature-extraction": OpenAIGPTModel,
               "text-classification": OpenAIGPTForSequenceClassification,
               "text-generation": OpenAIGPTLMHeadModel,
               "zero-shot": OpenAIGPTForSequenceClassification,
           }
           if is_torch_available()
           else {}
       )





       def 		lowerCAmelCase  (     self							:    List[str]      ,   __lowerCamelCase							:    str      ,   __lowerCamelCase							:    Tuple      ,   __lowerCamelCase							:    Any      ,   __lowerCamelCase							:    List[Any]      ,   __lowerCamelCase							:    Union[str, Any] ):


             '''simple docstring'''



             if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
                   # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
                   # `OpenAIGPTConfig` was never used in pipeline tests, either because of a missing checkpoint or because a
                   # tiny config could not be created.
                   return True

             return False





       def 		lowerCAmelCase  (     self							:    Union[str, Any]      ,   __lowerCamelCase							:    Optional[int]      ,   __lowerCamelCase							:    Tuple      ,   __lowerCamelCase							:    Tuple=False ):


             '''simple docstring'''



             lowerCamelCase__	:       Tuple					=						super()._prepare_for_class(__lowerCamelCase      ,   __lowerCamelCase      ,   return_labels=__lowerCamelCase )

             if return_labels:
                   if model_class.__name__ == "OpenAIGPTDoubleHeadsModel":
                         lowerCamelCase__	:       Optional[Any]					=						torch.zeros(
                             (self.model_tester.batch_size, self.model_tester.num_choices, self.model_tester.seq_length)      ,   dtype=torch.long      ,   device=__lowerCamelCase      ,   )
                         lowerCamelCase__	:       Tuple					=						inputs_dict["labels"]
                         lowerCamelCase__	:       Any					=						inputs_dict["labels"]
                         lowerCamelCase__	:       Any					=						torch.zeros(
                             (self.model_tester.batch_size, self.model_tester.num_choices)      ,   dtype=torch.long      ,   device=__lowerCamelCase      ,   )
                         lowerCamelCase__	:       Union[str, Any]					=						torch.zeros(
                             self.model_tester.batch_size      ,   dtype=torch.long      ,   device=__lowerCamelCase )
             return inputs_dict





       def 		lowerCAmelCase  (     self							:    List[Any] ):


             '''simple docstring'''



             lowerCamelCase__	:       Tuple					=						OpenAIGPTModelTester(self )
             lowerCamelCase__	:       Union[str, Any]					=						ConfigTester(self      ,   config_class=__lowerCamelCase      ,   n_embd=37 )





       def 		lowerCAmelCase  (     self							:    int ):


             '''simple docstring'''



             self.config_tester.run_common_tests()





       def 		lowerCAmelCase  (     self							:    Dict ):


             '''simple docstring'''



             lowerCamelCase__	:       List[Any]					=						self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_openai_gpt_model(*__lowerCamelCase )





       def 		lowerCAmelCase  (     self							:    str ):


             '''simple docstring'''



             lowerCamelCase__	:       Union[str, Any]					=						self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_lm_head_model(*__lowerCamelCase )





       def 		lowerCAmelCase  (     self							:    Dict ):


             '''simple docstring'''



             lowerCamelCase__	:       Optional[int]					=						self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_double_lm_head_model(*__lowerCamelCase )





       def 		lowerCAmelCase  (     self							:    Optional[int] ):


             '''simple docstring'''



             lowerCamelCase__	:       Optional[int]					=						self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_openai_gpt_for_sequence_classification(*__lowerCamelCase )





       @slow
       def 		lowerCAmelCase  (     self							:    List[str] ):


             '''simple docstring'''



             for model_name in OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                   lowerCamelCase__	:       Any					=						OpenAIGPTModel.from_pretrained(__lowerCamelCase )
                   self.assertIsNotNone(__lowerCamelCase )







@require_torch
class     _lowercase				(							unittest.TestCase):


       """simple docstring"""




       @slow
       def 		lowerCAmelCase  (     self							:    Any ):


             '''simple docstring'''



             lowerCamelCase__	:       List[Any]					=						OpenAIGPTLMHeadModel.from_pretrained("openai-gpt" )
             model.to(__lowerCamelCase )
             lowerCamelCase__	:       int					=						torch.tensor([[481, 4735, 544]]      ,   dtype=torch.long      ,   device=__lowerCamelCase )  # the president is
             lowerCamelCase__	:       Union[str, Any]					=						[
                 481,
                 4735,
                 544,
                 246,
                 963,
                 870,
                 762,
                 239,
                 244,
                 40477,
                 244,
                 249,
                 719,
                 881,
                 487,
                 544,
                 240,
                 244,
                 603,
                 481,
             ]  # the president is a very good man. " \n " i\'m sure he is, " said the

             lowerCamelCase__	:       int					=						model.generate(__lowerCamelCase      ,   do_sample=__lowerCamelCase )
             self.assertListEqual(output_ids[0].tolist()      ,   __lowerCamelCase )


