



"""simple docstring"""





import logging
import re

import pytorch_quantization
import pytorch_quantization.nn as quant_nn
import torch
from pytorch_quantization import calib
from pytorch_quantization.tensor_quant import QuantDescriptor


__lowerCAmelCase       :					Dict      					=					logging.getLogger(__name__)

__lowerCAmelCase       :					Tuple      					=					50  # max width of layer names
__lowerCAmelCase       :					List[str]      					=					70  # max width of quantizer names
def 	__lowerCAmelCase						(		__UpperCamelCase      :			int    ):


						'''simple docstring'''






						snake_case_   : Union[str, Any]						=			parser.add_argument_group("""quant_trainer arguments"""    )
						group.add_argument("""--wprec"""					, type=__UpperCamelCase					, default=8					, help="""weight precision"""    )
						group.add_argument("""--aprec"""					, type=__UpperCamelCase					, default=8					, help="""activation precision"""    )
						group.add_argument("""--quant-per-tensor"""					, action="""store_true"""					, help="""per tensor weight scaling"""    )
						group.add_argument("""--quant-disable"""					, action="""store_true"""					, help="""disable all quantizers"""    )
						group.add_argument("""--quant-disable-embeddings"""					, action="""store_true"""					, help="""disable all embeddings quantizers"""    )
						group.add_argument("""--quant-disable-keyword"""					, type=__UpperCamelCase					, nargs="""+"""					, help="""disable quantizers by keyword"""    )
						group.add_argument("""--quant-disable-layer-module"""					, type=__UpperCamelCase					, help="""disable quantizers by keyword under layer."""    )
						group.add_argument("""--quant-enable-layer-module"""					, type=__UpperCamelCase					, help="""enable quantizers by keyword under layer"""    )
						group.add_argument("""--calibrator"""					, default="""max"""					, help="""which quantization range calibrator to use"""    )
						group.add_argument("""--percentile"""					, default=__UpperCamelCase					, type=__UpperCamelCase					, help="""percentile for PercentileCalibrator"""    )
						group.add_argument("""--fuse-qkv"""					, action="""store_true"""					, help="""use the same scale factor for qkv"""    )
						group.add_argument("""--clip-gelu"""					, metavar="""N"""					, type=__UpperCamelCase					, help="""clip gelu output maximum value to N"""    )
						group.add_argument(
						    """--recalibrate-weights"""					, action="""store_true"""					, help=(
						        """recalibrate weight amaxes by taking the max of the weights."""
						        """ amaxes will be computed with the current quantization granularity (axis)."""
						    )					, )
def 	__lowerCAmelCase						(		__UpperCamelCase      :			int    ):


						'''simple docstring'''






						if args.calibrator == "max":
												snake_case_   : Dict						=			"""max"""
						elif args.calibrator == "percentile":
												if args.percentile is None:
																		raise ValueError("""Specify --percentile when using percentile calibrator"""    )
												snake_case_   : List[Any]						=			"""histogram"""
						elif args.calibrator == "mse":
												snake_case_   : List[str]						=			"""histogram"""
						else:
												raise ValueError(F'Invalid calibrator {args.calibrator}'    )

						snake_case_   : Tuple						=			QuantDescriptor(num_bits=args.aprec					, calib_method=__UpperCamelCase    )
						snake_case_   : Union[str, Any]						=			QuantDescriptor(num_bits=args.wprec					, axis=(None if args.quant_per_tensor else (0,))    )
						quant_nn.QuantLinear.set_default_quant_desc_input(__UpperCamelCase    )
						quant_nn.QuantLinear.set_default_quant_desc_weight(__UpperCamelCase    )
def 	__lowerCAmelCase						(		__UpperCamelCase      :			int					, __UpperCamelCase      :			Dict					, __UpperCamelCase      :			Union[str, Any]=False					, __UpperCamelCase      :			List[Any]=False    ):


						'''simple docstring'''






						logger.info("""Configuring Model for Quantization"""    )
						logger.info(F'using quantization package {pytorch_quantization.__file__}'    )

						if not calib:
												if args.quant_disable_embeddings:
																		set_quantizer_by_name(__UpperCamelCase					, ["""embeddings"""]					, which="""weight"""					, _disabled=__UpperCamelCase    )

												if args.quant_disable:
																		set_quantizer_by_name(__UpperCamelCase					, [""""""]					, _disabled=__UpperCamelCase    )

												if args.quant_disable_keyword:
																		set_quantizer_by_name(__UpperCamelCase					, args.quant_disable_keyword					, _disabled=__UpperCamelCase    )

												if args.quant_disable_layer_module:
																		set_quantizer_by_name(__UpperCamelCase					, [r"""layer.\d+.""" + args.quant_disable_layer_module]					, _disabled=__UpperCamelCase    )

												if args.quant_enable_layer_module:
																		set_quantizer_by_name(__UpperCamelCase					, [r"""layer.\d+.""" + args.quant_enable_layer_module]					, _disabled=__UpperCamelCase    )

												if args.recalibrate_weights:
																		recalibrate_weights(__UpperCamelCase    )

												if args.fuse_qkv:
																		fuse_qkv(__UpperCamelCase					, __UpperCamelCase    )

						if args.clip_gelu:
												clip_gelu(__UpperCamelCase					, args.clip_gelu    )

						# if args.local_rank in [-1, 0] and not calib:
						print_quant_summary(__UpperCamelCase    )
def 	__lowerCAmelCase						(		__UpperCamelCase      :			Any    ):


						'''simple docstring'''






						logger.info("""Enabling Calibration"""    )
						for name, module in model.named_modules():
												if name.endswith("""_quantizer"""    ):
																		if module._calibrator is not None:
																								module.disable_quant()
																								module.enable_calib()
																		else:
																								module.disable()
																		logger.info(F'{name:80}: {module}'    )
def 	__lowerCAmelCase						(		__UpperCamelCase      :			List[str]					, __UpperCamelCase      :			Union[str, Any]    ):


						'''simple docstring'''






						logger.info("""Loading calibrated amax"""    )
						for name, module in model.named_modules():
												if name.endswith("""_quantizer"""    ):
																		if module._calibrator is not None:
																								if isinstance(module._calibrator					, calib.MaxCalibrator    ):
																														module.load_calib_amax()
																								else:
																														module.load_calib_amax("""percentile"""					, percentile=args.percentile    )
																								module.enable_quant()
																								module.disable_calib()
																		else:
																								module.enable()
						model.cuda()
						print_quant_summary(__UpperCamelCase    )
def 	__lowerCAmelCase						(		__UpperCamelCase      :			str					, __UpperCamelCase      :			Any    ):


						'''simple docstring'''






						def fusea(__UpperCamelCase      :			List[Any]					, __UpperCamelCase      :			int					, __UpperCamelCase      :			Tuple    ):
												for mod in [qq, qk, qv]:
																		if not hasattr(__UpperCamelCase					, """_amax"""    ):
																								print("""          WARNING: NO AMAX BUFFER"""    )
																								return
												snake_case_   : Optional[Any]						=			qq._amax.detach().item()
												snake_case_   : Dict						=			qk._amax.detach().item()
												snake_case_   : str						=			qv._amax.detach().item()

												snake_case_   : List[str]						=			max(__UpperCamelCase					, __UpperCamelCase					, __UpperCamelCase    )
												qq._amax.fill_(__UpperCamelCase    )
												qk._amax.fill_(__UpperCamelCase    )
												qv._amax.fill_(__UpperCamelCase    )
												logger.info(F'          q={q:5.2f} k={k:5.2f} v={v:5.2f} -> {amax:5.2f}'    )

						for name, mod in model.named_modules():
												if name.endswith(""".attention.self"""    ):
																		logger.info(F'FUSE_QKV: {name:{name_width}}'    )
																		fusea(mod.matmul_q_input_quantizer					, mod.matmul_k_input_quantizer					, mod.matmul_v_input_quantizer    )
																		if args.quant_per_tensor:
																								fusea(mod.query._weight_quantizer					, mod.key._weight_quantizer					, mod.value._weight_quantizer    )
def 	__lowerCAmelCase						(		__UpperCamelCase      :			Optional[Any]					, __UpperCamelCase      :			str    ):


						'''simple docstring'''






						for name, mod in model.named_modules():
												if name.endswith(""".output.dense"""    ) and not name.endswith("""attention.output.dense"""    ):
																		snake_case_   : Tuple						=			mod._input_quantizer._amax.data.detach().item()
																		mod._input_quantizer._amax.data.detach().clamp_(max=__UpperCamelCase    )
																		snake_case_   : Dict						=			mod._input_quantizer._amax.data.detach().item()
																		logger.info(F'CLIP_GELU: {name:{name_width}} amax: {amax_init:5.2f} -> {amax:5.2f}'    )
def 	__lowerCAmelCase						(		__UpperCamelCase      :			Dict    ):


						'''simple docstring'''






						for name, mod in model.named_modules():
												if hasattr(__UpperCamelCase					, """_weight_quantizer"""    ) and mod._weight_quantizer.axis is not None:
																		snake_case_   : Union[str, Any]						=			mod.weight.shape[0]
																		snake_case_   : List[str]						=			mod._weight_quantizer._amax.detach()
																		snake_case_   : int						=			torch.ones(__UpperCamelCase					, dtype=amax.dtype					, device=amax.device    ) * amax
																		print(F'expanding {name} {amax} -> {mod._weight_quantizer._amax}'    )
def 	__lowerCAmelCase						(		__UpperCamelCase      :			Union[str, Any]    ):


						'''simple docstring'''






						for name, mod in model.named_modules():
												if hasattr(__UpperCamelCase					, """_weight_quantizer"""    ):
																		if not hasattr(mod.weight_quantizer					, """_amax"""    ):
																								print("""RECALIB: {name:{name_width}} WARNING: NO AMAX BUFFER"""    )
																								continue

																		# determine which axes to reduce across
																		# e.g. a 4D tensor quantized per axis 0 should reduce over (1,2,3)
																		snake_case_   : Optional[int]						=			set() if mod._weight_quantizer.axis is None else set(mod._weight_quantizer.axis    )
																		snake_case_   : Any						=			set(range(len(mod.weight.size()    )    )    ) - axis_set
																		snake_case_   : Optional[Any]						=			pytorch_quantization.utils.reduce_amax(mod.weight					, axis=__UpperCamelCase					, keepdims=__UpperCamelCase    ).detach()
																		logger.info(F'RECALIB: {name:{name_width}} {mod._weight_quantizer._amax.flatten()} -> {amax.flatten()}'    )
																		snake_case_   : List[Any]						=			amax
def 	__lowerCAmelCase						(		__UpperCamelCase      :			Tuple					, __UpperCamelCase      :			List[str]=2_5					, __UpperCamelCase      :			Dict=1_8_0					, __UpperCamelCase      :			Optional[Any]=None    ):


						'''simple docstring'''






						if ignore is None:
												snake_case_   : Any						=			[]
						elif not isinstance(__UpperCamelCase					, __UpperCamelCase    ):
												snake_case_   : Union[str, Any]						=			[ignore]

						snake_case_   : int						=			0
						for name, mod in model.named_modules():
												if not hasattr(__UpperCamelCase					, """weight"""    ):
																		continue
												snake_case_   : Dict						=			max(__UpperCamelCase					, len(__UpperCamelCase    )    )

						for name, mod in model.named_modules():
												snake_case_   : Tuple						=			getattr(__UpperCamelCase					, """_input_quantizer"""					, __UpperCamelCase    )
												snake_case_   : Dict						=			getattr(__UpperCamelCase					, """_weight_quantizer"""					, __UpperCamelCase    )
												if not hasattr(__UpperCamelCase					, """weight"""    ):
																		continue
												if type(__UpperCamelCase    ) in ignore:
																		continue
												if [True for s in ignore if type(__UpperCamelCase    ) is str and s in name]:
																		continue
												snake_case_   : Tuple						=			F'Act:{input_q.extra_repr()}'
												snake_case_   : Tuple						=			F'Wgt:{weight_q.extra_repr()}'
												snake_case_   : Union[str, Any]						=			F'{name:{name_width}} {act_str} {wgt_str}'
												if len(__UpperCamelCase    ) <= line_width:
																		logger.info(__UpperCamelCase    )
												else:
																		logger.info(F'{name:{name_width}} {act_str}'    )
																		logger.info(F'{"  ":{name_width}} {wgt_str}'    )
def 	__lowerCAmelCase						(		__UpperCamelCase      :			Union[str, Any]    ):


						'''simple docstring'''






						snake_case_   : List[Any]						=			0
						for name, mod in model.named_modules():
												if isinstance(__UpperCamelCase					, pytorch_quantization.nn.TensorQuantizer    ):
																		print(F'{name:80} {mod}'    )
																		count += 1
						print(F'{count} TensorQuantizers found in model'    )
def 	__lowerCAmelCase						(		__UpperCamelCase      :			Dict					, __UpperCamelCase      :			str					, __UpperCamelCase      :			List[Any]					, __UpperCamelCase      :			List[Any]					, __UpperCamelCase      :			int    ):


						'''simple docstring'''






						snake_case_   : Dict						=			getattr(__UpperCamelCase					, __UpperCamelCase					, __UpperCamelCase    )
						if quantizer_mod is not None:
												assert hasattr(__UpperCamelCase					, __UpperCamelCase    )
												setattr(__UpperCamelCase					, __UpperCamelCase					, __UpperCamelCase    )
						else:
												logger.warning(F'{name} has no {quantizer}'    )
def 	__lowerCAmelCase						(		__UpperCamelCase      :			Optional[Any]					, __UpperCamelCase      :			Optional[int]					, __UpperCamelCase      :			List[Any]="both"					, **__UpperCamelCase      :			Any    ):


						'''simple docstring'''






						snake_case_   : Any						=			F'Warning: changing {which} quantizers of {name:{qname_width}}'
						for k, v in kwargs.items():
												s += F' {k}={v}'
												if which in ["input", "both"]:
																		set_quantizer(__UpperCamelCase					, __UpperCamelCase					, """_input_quantizer"""					, __UpperCamelCase					, __UpperCamelCase    )
												if which in ["weight", "both"]:
																		set_quantizer(__UpperCamelCase					, __UpperCamelCase					, """_weight_quantizer"""					, __UpperCamelCase					, __UpperCamelCase    )
						logger.info(__UpperCamelCase    )






def 	__lowerCAmelCase						(		__UpperCamelCase      :			Dict					, __UpperCamelCase      :			int					, **__UpperCamelCase      :			int    ):


						'''simple docstring'''






						for name, mod in model.named_modules():
												if hasattr(__UpperCamelCase					, """_input_quantizer"""    ) or hasattr(__UpperCamelCase					, """_weight_quantizer"""    ):
																		for n in names:
																								if re.search(__UpperCamelCase					, __UpperCamelCase    ):
																														set_quantizers(__UpperCamelCase					, __UpperCamelCase					, **__UpperCamelCase    )
												elif name.endswith("""_quantizer"""    ):
																		for n in names:
																								if re.search(__UpperCamelCase					, __UpperCamelCase    ):
																														snake_case_   : Optional[int]						=			F'Warning: changing {name:{name_width}}'
																														for k, v in kwargs.items():
																																				s += F' {k}={v}'
																																				setattr(__UpperCamelCase					, __UpperCamelCase					, __UpperCamelCase    )
																														logger.info(__UpperCamelCase    )






"""simple docstring"""





import math

import tensorflow as tf
from packaging import version
def 	__lowerCAmelCase						(		__UpperCamelCase      :			List[Any]    ):


						'''simple docstring'''






						snake_case_   : int						=			tf.convert_to_tensor(__UpperCamelCase    )
						snake_case_   : int						=			0.5 * (1.0 + tf.math.erf(x / tf.cast(tf.sqrt(2.0    )					, x.dtype    )    ))

						return x * cdf
def 	__lowerCAmelCase						(		__UpperCamelCase      :			List[str]    ):


						'''simple docstring'''






						snake_case_   : int						=			tf.convert_to_tensor(__UpperCamelCase    )
						snake_case_   : List[Any]						=			tf.cast(math.pi					, x.dtype    )
						snake_case_   : int						=			tf.cast(0.044_715					, x.dtype    )
						snake_case_   : Optional[int]						=			0.5 * (1.0 + tf.tanh(tf.sqrt(2.0 / pi    ) * (x + coeff * tf.pow(__UpperCamelCase					, 3    ))    ))

						return x * cdf
def 	__lowerCAmelCase						(		__UpperCamelCase      :			str    ):


						'''simple docstring'''






						snake_case_   : Optional[Any]						=			tf.convert_to_tensor(__UpperCamelCase    )

						return x * tf.tanh(tf.math.softplus(__UpperCamelCase    )    )
def 	__lowerCAmelCase						(		__UpperCamelCase      :			Any    ):


						'''simple docstring'''






						snake_case_   : str						=			tf.convert_to_tensor(__UpperCamelCase    )
						snake_case_   : int						=			tf.cast(0.044_715					, x.dtype    )
						snake_case_   : Optional[int]						=			tf.cast(0.7_978_845_608					, x.dtype    )

						return 0.5 * x * (1.0 + tf.tanh(x * coeffa * (1.0 + coeffa * x * x)    ))
def 	__lowerCAmelCase						(		__UpperCamelCase      :			List[str]    ):


						'''simple docstring'''






						snake_case_   : Tuple						=			tf.convert_to_tensor(__UpperCamelCase    )
						snake_case_   : str						=			tf.cast(1.702					, x.dtype    )
						return x * tf.math.sigmoid(coeff * x    )
def 	__lowerCAmelCase						(		__UpperCamelCase      :			int    ):


						'''simple docstring'''






						return tf.clip_by_value(_gelu(__UpperCamelCase    )					, -1_0					, 1_0    )
def 	__lowerCAmelCase						(		__UpperCamelCase      :			Optional[int]					, __UpperCamelCase      :			List[str]=-1    ):


						'''simple docstring'''






						snake_case_			,		snake_case_   : List[Any]						=			tf.split(__UpperCamelCase					, 2					, axis=__UpperCamelCase    )
						return a * tf.math.sigmoid(__UpperCamelCase    )


if version.parse(tf.version.VERSION) >= version.parse('''2.4'''):
					def 	__lowerCAmelCase						(		__UpperCamelCase      :			List[Any]    ):


											'''simple docstring'''






											return tf.keras.activations.gelu(__UpperCamelCase					, approximate=__UpperCamelCase    )

					__lowerCAmelCase       :					int      					=					tf.keras.activations.gelu






					__lowerCAmelCase       :					Optional[Any]      					=					approximate_gelu_wrap
else:
					__lowerCAmelCase       :					List[Any]      					=					_gelu
					__lowerCAmelCase       :					Any      					=					_gelu_new


__lowerCAmelCase       :					Dict      					=					{
    '''gelu''': gelu,
    '''gelu_10''': gelu_aa,
    '''gelu_fast''': gelu_fast,
    '''gelu_new''': gelu_new,
    '''glu''': glu,
    '''mish''': mish,
    '''quick_gelu''': quick_gelu,
    '''relu''': tf.keras.activations.relu,
    '''sigmoid''': tf.keras.activations.sigmoid,
    '''silu''': tf.keras.activations.swish,
    '''swish''': tf.keras.activations.swish,
    '''tanh''': tf.keras.activations.tanh,
}






def 	__lowerCAmelCase						(		__UpperCamelCase      :			Any    ):


						'''simple docstring'''






						if activation_string in ACTaFN:
												return ACTaFN[activation_string]
						else:
												raise KeyError(F'function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()    )}'    )

