




from pickle import UnpicklingError

import jax
import jax.numpy as jnp
import numpy as np
from flax.serialization import from_bytes
from flax.traverse_util import flatten_dict

from ..utils import logging


__magic_name__								=logging.get_logger(__name__)







def  __UpperCamelCase    (						A ,							A							):


     try:
          with open(A ,							'''rb'''							) as flax_state_f:
               UpperCamelCase__										=      from_bytes(A ,							flax_state_f.read()							)
     except UnpicklingError as e:
          try:
               with open(A							) as f:
                    if f.read().startswith('''version'''							):
                         raise OSError(
                             '''You seem to have cloned a repository without having git-lfs installed. Please'''
                             ''' install git-lfs and run `git lfs install` followed by `git lfs pull` in the'''
                             ''' folder you cloned.'''							)
                    else:
                         raise ValueError from e
          except (UnicodeDecodeError, ValueError):
               raise EnvironmentError(f"Unable to convert {model_file} to Flax deserializable object. "							)

     return load_flax_weights_in_pytorch_model(A ,							A							)







def  __UpperCamelCase    (						A ,							A							):


     try:
          import torch  # noqa: F401
     except ImportError:
          logger.error(
              '''Loading Flax weights in PyTorch requires both PyTorch and Flax to be installed. Please see'''
              ''' https://pytorch.org/ and https://flax.readthedocs.io/en/latest/installation.html for installation'''
              ''' instructions.'''							)
          raise

     # check if we have bf16 weights
     UpperCamelCase__										=      flatten_dict(jax.tree_util.tree_map(lambda A							: x.dtype == jnp.bfloataa ,							A							)							).values()
     if any(A							):
          # convert all weights to fp32 if they are bf16 since torch.from_numpy can-not handle bf16

          # and bf16 is not fully supported in PT yet.
          logger.warning(
              '''Found ``bfloat16`` weights in Flax model. Casting all ``bfloat16`` weights to ``float32`` '''
              '''before loading those in PyTorch model.'''							)
          UpperCamelCase__										=      jax.tree_util.tree_map(
              lambda A							: params.astype(np.floataa							) if params.dtype == jnp.bfloataa else params ,							A							)

     UpperCamelCase__										=      ''''''

     UpperCamelCase__										=      flatten_dict(A ,							sep='''.'''							)
     UpperCamelCase__										=      pt_model.state_dict()

     # keep track of unexpected & missing keys
     UpperCamelCase__										=      []
     UpperCamelCase__										=      set(pt_model_dict.keys()							)

     for flax_key_tuple, flax_tensor in flax_state_dict.items():
          UpperCamelCase__										=      flax_key_tuple.split('''.'''							)

          if flax_key_tuple_array[-1] == "kernel" and flax_tensor.ndim == 4:
               UpperCamelCase__										=      flax_key_tuple_array[:-1] + ['''weight''']
               UpperCamelCase__										=      jnp.transpose(A ,							(3, 2, 0, 1)							)
          elif flax_key_tuple_array[-1] == "kernel":
               UpperCamelCase__										=      flax_key_tuple_array[:-1] + ['''weight''']
               UpperCamelCase__										=      flax_tensor.T
          elif flax_key_tuple_array[-1] == "scale":
               UpperCamelCase__										=      flax_key_tuple_array[:-1] + ['''weight''']

          if "time_embedding" not in flax_key_tuple_array:
               for i, flax_key_tuple_string in enumerate(A							):
                    UpperCamelCase__										=      (
                        flax_key_tuple_string.replace('''_0''' ,							'''.0'''							)
                        .replace('''_1''' ,							'''.1'''							)
                        .replace('''_2''' ,							'''.2'''							)
                        .replace('''_3''' ,							'''.3'''							)
                        .replace('''_4''' ,							'''.4'''							)
                        .replace('''_5''' ,							'''.5'''							)
                        .replace('''_6''' ,							'''.6'''							)
                        .replace('''_7''' ,							'''.7'''							)
                        .replace('''_8''' ,							'''.8'''							)
                        .replace('''_9''' ,							'''.9'''							)
                    )

          UpperCamelCase__										=      '''.'''.join(A							)

          if flax_key in pt_model_dict:
               if flax_tensor.shape != pt_model_dict[flax_key].shape:
                    raise ValueError(
                        f"Flax checkpoint seems to be incorrect. Weight {flax_key_tuple} was expected "
                        f"to be of shape {pt_model_dict[flax_key].shape}, but is {flax_tensor.shape}."							)
               else:
                    # add weight to pytorch dict
                    UpperCamelCase__										=      np.asarray(A							) if not isinstance(A ,							np.ndarray							) else flax_tensor
                    UpperCamelCase__										=      torch.from_numpy(A							)
                    # remove from missing keys
                    missing_keys.remove(A							)
          else:
               # weight is not expected by PyTorch model
               unexpected_keys.append(A							)

     pt_model.load_state_dict(A							)

     # re-transform missing_keys to list
     UpperCamelCase__										=      list(A							)

     if len(A							) > 0:
          logger.warning(
              '''Some weights of the Flax model were not used when initializing the PyTorch model'''
              f" {pt_model.__class__.__name__}: {unexpected_keys}\n- This IS expected if you are initializing"
              f" {pt_model.__class__.__name__} from a Flax model trained on another task or with another architecture"
              ''' (e.g. initializing a BertForSequenceClassification model from a FlaxBertForPreTraining model).\n- This'''
              f" IS NOT expected if you are initializing {pt_model.__class__.__name__} from a Flax model that you expect"
              ''' to be exactly identical (e.g. initializing a BertForSequenceClassification model from a'''
              ''' FlaxBertForSequenceClassification model).'''							)
     if len(A							) > 0:
          logger.warning(
              f"Some weights of {pt_model.__class__.__name__} were not initialized from the Flax model and are newly"
              f" initialized: {missing_keys}\nYou should probably TRAIN this model on a down-stream task to be able to"
              ''' use it for predictions and inference.'''							)

     return pt_model




import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch


if is_torch_available():
     import torch

     from transformers.activations import gelu_new, gelu_python, get_activation



@require_torch
class     _A   (    unittest.TestCase							):






  def 		_a					(self	)		->      str:





       '''simple docstring'''





       UpperCamelCase__										=      torch.tensor([-100, -1, -0.1, 0, 0.1, 1.0, 100]	)
       UpperCamelCase__										=      get_activation('''gelu'''	)
       self.assertTrue(torch.allclose(gelu_python(SCREAMING_SNAKE_CASE_	)   ,				torch_builtin(SCREAMING_SNAKE_CASE_	)	)	)
       self.assertFalse(torch.allclose(gelu_python(SCREAMING_SNAKE_CASE_	)   ,				gelu_new(SCREAMING_SNAKE_CASE_	)	)	)






  def 		_a					(self	)		->      str:





       '''simple docstring'''





       UpperCamelCase__										=      torch.tensor([-100, -1, -0.1, 0, 0.1, 1.0, 100]	)
       UpperCamelCase__										=      get_activation('''gelu'''	)
       UpperCamelCase__										=      get_activation('''gelu_10'''	)

       UpperCamelCase__										=      torch_builtin(SCREAMING_SNAKE_CASE_	)
       UpperCamelCase__										=      geluaa(SCREAMING_SNAKE_CASE_	)

       UpperCamelCase__										=      torch.where(y_gelu_aa < 10.0   ,				1   ,				0	)

       self.assertTrue(torch.max(SCREAMING_SNAKE_CASE_	).item() == 10.0	)
       self.assertTrue(torch.allclose(y_gelu * clipped_mask   ,				y_gelu_aa * clipped_mask	)	)






  def 		_a					(self	)		->      Optional[Any]:





       '''simple docstring'''





       get_activation('''gelu'''	)
       get_activation('''gelu_10'''	)
       get_activation('''gelu_fast'''	)
       get_activation('''gelu_new'''	)
       get_activation('''gelu_python'''	)
       get_activation('''gelu_pytorch_tanh'''	)
       get_activation('''linear'''	)
       get_activation('''mish'''	)
       get_activation('''quick_gelu'''	)
       get_activation('''relu'''	)
       get_activation('''sigmoid'''	)
       get_activation('''silu'''	)
       get_activation('''swish'''	)
       get_activation('''tanh'''	)
       with self.assertRaises(SCREAMING_SNAKE_CASE_	):
            get_activation('''bogus'''	)
       with self.assertRaises(SCREAMING_SNAKE_CASE_	):
            get_activation(SCREAMING_SNAKE_CASE_	)






  def 		_a					(self	)		->      Optional[Any]:





       '''simple docstring'''





       UpperCamelCase__										=      get_activation('''gelu'''	)
       UpperCamelCase__										=      1
       UpperCamelCase__										=      get_activation('''gelu'''	)
       self.assertEqual(acta.a   ,				1	)
       with self.assertRaises(SCREAMING_SNAKE_CASE_	):
            UpperCamelCase__										=      acta.a



