# Copyright 2022 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os
import subprocess

from packaging.version import Version, parse

from accelerate.commands.config.config_args import default_config_file, load_config_from_file


__magic_name__								='''Run commands across TPU VMs for initial setup before running `accelerate launch`.'''







def  __UpperCamelCase    (						A=None							):
     if subparsers is not None:
          UpperCamelCase__										=      subparsers.add_parser('''tpu-config''' ,							description=_description							)
     else:
          UpperCamelCase__										=      argparse.ArgumentParser('''Accelerate tpu-config command''' ,							description=_description							)
     # Core arguments
     UpperCamelCase__										=      parser.add_argument_group(
         '''Config Arguments''' ,							'''Arguments that can be configured through `accelerate config`.'''							)
     config_args.add_argument(
         '''--config_file''' ,							type=A ,							default=A ,							help='''Path to the config file to use for accelerate.''' ,							)
     config_args.add_argument(
         '''--tpu_name''' ,							default=A ,							help='''The name of the TPU to use. If not specified, will use the TPU specified in the config file.''' ,							)
     config_args.add_argument(
         '''--tpu_zone''' ,							default=A ,							help='''The zone of the TPU to use. If not specified, will use the zone specified in the config file.''' ,							)
     UpperCamelCase__										=      parser.add_argument_group('''TPU Arguments''' ,							'''Arguments for options ran inside the TPU.'''							)
     pod_args.add_argument(
         '''--use_alpha''' ,							action='''store_true''' ,							help='''Whether to use `gcloud alpha` when running the TPU training script instead of `gcloud`.''' ,							)
     pod_args.add_argument(
         '''--command_file''' ,							default=A ,							help='''The path to the file containing the commands to run on the pod on startup.''' ,							)
     pod_args.add_argument(
         '''--command''' ,							action='''append''' ,							nargs='''+''' ,							help='''A command to run on the pod. Can be passed multiple times.''' ,							)
     pod_args.add_argument(
         '''--install_accelerate''' ,							action='''store_true''' ,							help='''Whether to install accelerate on the pod. Defaults to False.''' ,							)
     pod_args.add_argument(
         '''--accelerate_version''' ,							default='''latest''' ,							help='''The version of accelerate to install on the pod. If not specified, will use the latest pypi version. Specify \'dev\' to install from GitHub.''' ,							)
     pod_args.add_argument(
         '''--debug''' ,							action='''store_true''' ,							help='''If set, will print the command that would be run instead of running it.'''							)

     if subparsers is not None:
          parser.set_defaults(func=A							)
     return parser







def  __UpperCamelCase    (						A							):
     UpperCamelCase__										=      None

     # Get the default from the config file if it exists.
     if args.config_file is not None or os.path.isfile(A							):
          UpperCamelCase__										=      load_config_from_file(args.config_file							)
          if not args.command_file and defaults.command_file is not None and not args.command:
               UpperCamelCase__										=      defaults.command_file
          if not args.command and defaults.commands is not None:
               UpperCamelCase__										=      defaults.commands
          if not args.tpu_name:
               UpperCamelCase__										=      defaults.tpu_name
          if not args.tpu_zone:
               UpperCamelCase__										=      defaults.tpu_zone
     if args.accelerate_version == "dev":
          UpperCamelCase__										=      '''git+https://github.com/huggingface/accelerate.git'''
     elif args.accelerate_version == "latest":
          UpperCamelCase__										=      '''accelerate -U'''
     elif isinstance(parse(args.accelerate_version							) ,							A							):
          UpperCamelCase__										=      f"accelerate=={args.accelerate_version}"

     if not args.command_file and not args.command:
          raise ValueError('''You must specify either a command file or a command to run on the pod.'''							)

     if args.command_file:
          with open(args.command_file ,							'''r'''							) as f:
               UpperCamelCase__										=      [f.read().splitlines()]

    # To turn list of lists into list of strings
     if isinstance(args.command[0] ,							A							):
          UpperCamelCase__										=      [line for cmd in args.command for line in cmd]
     # Default to the shared folder and install accelerate
     UpperCamelCase__										=      ['''cd /usr/share''']
     if args.install_accelerate:
          new_cmd += [f"pip install {args.accelerate_version}"]
     new_cmd += args.command
     UpperCamelCase__										=      '''; '''.join(A							)

     # Then send it to gcloud
     # Eventually try to use google-api-core to do this instead of subprocess
     UpperCamelCase__										=      ['''gcloud''']
     if args.use_alpha:
          cmd += ["alpha"]
     cmd += [
         "compute",
         "tpus",
         "tpu-vm",
         "ssh",
         args.tpu_name,
         "--zone",
         args.tpu_zone,
         "--command",
         args.command,
         "--worker",
         "all",
     ]
     if args.debug:
          print(f"Running {' '.join(A							)}"							)
          return
     subprocess.run(A							)
     print('''Successfully setup pod.'''							)







def  __UpperCamelCase    (						):
     UpperCamelCase__										=      tpu_command_parser()
     UpperCamelCase__										=      parser.parse_args()

     tpu_command_launcher(A							)




import math

import tensorflow as tf
from packaging import version







def  __UpperCamelCase    (						A							):
     UpperCamelCase__										=      tf.convert_to_tensor(A							)
     UpperCamelCase__										=      0.5 * (1.0 + tf.math.erf(x / tf.cast(tf.sqrt(2.0							) ,							x.dtype							)							))

     return x * cdf







def  __UpperCamelCase    (						A							):
     UpperCamelCase__										=      tf.convert_to_tensor(A							)
     UpperCamelCase__										=      tf.cast(math.pi ,							x.dtype							)
     UpperCamelCase__										=      tf.cast(0.04_47_15 ,							x.dtype							)
     UpperCamelCase__										=      0.5 * (1.0 + tf.tanh(tf.sqrt(2.0 / pi							) * (x + coeff * tf.pow(A ,							3							))							))

     return x * cdf







def  __UpperCamelCase    (						A							):
     UpperCamelCase__										=      tf.convert_to_tensor(A							)

     return x * tf.tanh(tf.math.softplus(A							)							)







def  __UpperCamelCase    (						A							):
     UpperCamelCase__										=      tf.convert_to_tensor(A							)
     UpperCamelCase__										=      tf.cast(0.04_47_15 ,							x.dtype							)
     UpperCamelCase__										=      tf.cast(0.79_78_84_56_08 ,							x.dtype							)

     return 0.5 * x * (1.0 + tf.tanh(x * coeffa * (1.0 + coeffa * x * x)							))







def  __UpperCamelCase    (						A							):
     UpperCamelCase__										=      tf.convert_to_tensor(A							)
     UpperCamelCase__										=      tf.cast(1.7_02 ,							x.dtype							)
     return x * tf.math.sigmoid(coeff * x							)







def  __UpperCamelCase    (						A							):
     return tf.clip_by_value(_gelu(A							) ,							-10 ,							10							)







def  __UpperCamelCase    (						A ,							A=-1							):
     UpperCamelCase__						,       UpperCamelCase__										=      tf.split(A ,							2 ,							axis=A							)
     return a * tf.math.sigmoid(A							)


if version.parse(tf.version.VERSION) >= version.parse('''2.4'''):







     def  __UpperCamelCase    (						A							):
          return tf.keras.activations.gelu(A ,							approximate=A							)

     __magic_name__								=tf.keras.activations.gelu



     __magic_name__								=approximate_gelu_wrap
else:
     __magic_name__								=_gelu
     __magic_name__								=_gelu_new


__magic_name__								={
    '''gelu''': gelu,
    '''gelu_10''': gelu_aa,
    '''gelu_fast''': gelu_fast,
    '''gelu_new''': gelu_new,
    '''glu''': glu,
    '''mish''': mish,
    '''quick_gelu''': quick_gelu,
    '''relu''': tf.keras.activations.relu,
    '''sigmoid''': tf.keras.activations.sigmoid,
    '''silu''': tf.keras.activations.swish,
    '''swish''': tf.keras.activations.swish,
    '''tanh''': tf.keras.activations.tanh,
}







def  __UpperCamelCase    (						A							):
     if activation_string in ACTaFN:
          return ACTaFN[activation_string]
     else:
          raise KeyError(f"function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()							)}"							)



