
"""simple docstring"""


import logging
import re

import pytorch_quantization
import pytorch_quantization.nn as quant_nn
import torch
from pytorch_quantization import calib
from pytorch_quantization.tensor_quant import QuantDescriptor


_lowercase							   =				logging.getLogger(__name__)

_lowercase							   =				50  # max width of layer names
_lowercase							   =				70  # max width of quantizer names






def       _snake_case				(		snake_case__   :     Optional[int]     ):
	A						   =   parser.add_argument_group('quant_trainer arguments'     )
	group.add_argument('--wprec'	,      type=_lowerCamelCase	,      default=8	,      help='weight precision'     )
	group.add_argument('--aprec'	,      type=_lowerCamelCase	,      default=8	,      help='activation precision'     )
	group.add_argument('--quant-per-tensor'	,      action='store_true'	,      help='per tensor weight scaling'     )
	group.add_argument('--quant-disable'	,      action='store_true'	,      help='disable all quantizers'     )
	group.add_argument('--quant-disable-embeddings'	,      action='store_true'	,      help='disable all embeddings quantizers'     )
	group.add_argument('--quant-disable-keyword'	,      type=_lowerCamelCase	,      nargs='+'	,      help='disable quantizers by keyword'     )
	group.add_argument('--quant-disable-layer-module'	,      type=_lowerCamelCase	,      help='disable quantizers by keyword under layer.'     )
	group.add_argument('--quant-enable-layer-module'	,      type=_lowerCamelCase	,      help='enable quantizers by keyword under layer'     )
	group.add_argument('--calibrator'	,      default='max'	,      help='which quantization range calibrator to use'     )
	group.add_argument('--percentile'	,      default=_lowerCamelCase	,      type=_lowerCamelCase	,      help='percentile for PercentileCalibrator'     )
	group.add_argument('--fuse-qkv'	,      action='store_true'	,      help='use the same scale factor for qkv'     )
	group.add_argument('--clip-gelu'	,      metavar='N'	,      type=_lowerCamelCase	,      help='clip gelu output maximum value to N'     )
	group.add_argument(
	    '--recalibrate-weights'	,      action='store_true'	,      help=(
	        'recalibrate weight amaxes by taking the max of the weights.'
	        ' amaxes will be computed with the current quantization granularity (axis).'
	    )	,      )






def       _snake_case				(		snake_case__   :     str     ):

	if args.calibrator == "max":
		A						   =   'max'
	elif args.calibrator == "percentile":
		if args.percentile is None:
			raise ValueError('Specify --percentile when using percentile calibrator'     )
		A						   =   'histogram'
	elif args.calibrator == "mse":
		A						   =   'histogram'
	else:
		raise ValueError(F'Invalid calibrator {args.calibrator}'     )

	A						   =   QuantDescriptor(num_bits=args.aprec	,      calib_method=_lowerCamelCase     )
	A						   =   QuantDescriptor(num_bits=args.wprec	,      axis=(None if args.quant_per_tensor else (0,))     )
	quant_nn.QuantLinear.set_default_quant_desc_input(_lowerCamelCase     )
	quant_nn.QuantLinear.set_default_quant_desc_weight(_lowerCamelCase     )






def       _snake_case				(		snake_case__   :     List[str]	,      snake_case__   :     Union[str, Any]	,      snake_case__   :     Dict=False	,      snake_case__   :     Union[str, Any]=False     ):
	logger.info('Configuring Model for Quantization'     )
	logger.info(F'using quantization package {pytorch_quantization.__file__}'     )

	if not calib:
		if args.quant_disable_embeddings:
			set_quantizer_by_name(_lowerCamelCase	,      ['embeddings']	,      which='weight'	,      _disabled=_lowerCamelCase     )

		if args.quant_disable:
			set_quantizer_by_name(_lowerCamelCase	,      ['']	,      _disabled=_lowerCamelCase     )

		if args.quant_disable_keyword:
			set_quantizer_by_name(_lowerCamelCase	,      args.quant_disable_keyword	,      _disabled=_lowerCamelCase     )

		if args.quant_disable_layer_module:
			set_quantizer_by_name(_lowerCamelCase	,      [r'layer.\d+.' + args.quant_disable_layer_module]	,      _disabled=_lowerCamelCase     )

		if args.quant_enable_layer_module:
			set_quantizer_by_name(_lowerCamelCase	,      [r'layer.\d+.' + args.quant_enable_layer_module]	,      _disabled=_lowerCamelCase     )

		if args.recalibrate_weights:
			recalibrate_weights(_lowerCamelCase     )

		if args.fuse_qkv:
			fuse_qkv(_lowerCamelCase	,      _lowerCamelCase     )

	if args.clip_gelu:
		clip_gelu(_lowerCamelCase	,      args.clip_gelu     )

	# if args.local_rank in [-1, 0] and not calib:
	print_quant_summary(_lowerCamelCase     )






def       _snake_case				(		snake_case__   :     str     ):
	logger.info('Enabling Calibration'     )
	for name, module in model.named_modules():
		if name.endswith('_quantizer'     ):
			if module._calibrator is not None:
				module.disable_quant()
				module.enable_calib()
			else:
				module.disable()
			logger.info(F'{name:80}: {module}'     )






def       _snake_case				(		snake_case__   :     Tuple	,      snake_case__   :     List[Any]     ):
	logger.info('Loading calibrated amax'     )
	for name, module in model.named_modules():
		if name.endswith('_quantizer'     ):
			if module._calibrator is not None:
				if isinstance(module._calibrator	,      calib.MaxCalibrator     ):
					module.load_calib_amax()
				else:
					module.load_calib_amax('percentile'	,      percentile=args.percentile     )
				module.enable_quant()
				module.disable_calib()
			else:
				module.enable()
	model.cuda()
	print_quant_summary(_lowerCamelCase     )






def       _snake_case				(		snake_case__   :     Optional[int]	,      snake_case__   :     Dict     ):

	def fusea(snake_case__   :     int	,      snake_case__   :     str	,      snake_case__   :     List[str]     ):
		for mod in [qq, qk, qv]:
			if not hasattr(_lowerCamelCase	,      '_amax'     ):
				print('          WARNING: NO AMAX BUFFER'     )
				return
		A						   =   qq._amax.detach().item()
		A						   =   qk._amax.detach().item()
		A						   =   qv._amax.detach().item()

		A						   =   max(_lowerCamelCase	,      _lowerCamelCase	,      _lowerCamelCase     )
		qq._amax.fill_(_lowerCamelCase     )
		qk._amax.fill_(_lowerCamelCase     )
		qv._amax.fill_(_lowerCamelCase     )
		logger.info(F'          q={q:5.2f} k={k:5.2f} v={v:5.2f} -> {amax:5.2f}'     )

	for name, mod in model.named_modules():
		if name.endswith('.attention.self'     ):
			logger.info(F'FUSE_QKV: {name:{name_width}}'     )
			fusea(mod.matmul_q_input_quantizer	,      mod.matmul_k_input_quantizer	,      mod.matmul_v_input_quantizer     )
			if args.quant_per_tensor:
				fusea(mod.query._weight_quantizer	,      mod.key._weight_quantizer	,      mod.value._weight_quantizer     )






def       _snake_case				(		snake_case__   :     Any	,      snake_case__   :     Dict     ):

	for name, mod in model.named_modules():
		if name.endswith('.output.dense'     ) and not name.endswith('attention.output.dense'     ):
			A						   =   mod._input_quantizer._amax.data.detach().item()
			mod._input_quantizer._amax.data.detach().clamp_(max=_lowerCamelCase     )
			A						   =   mod._input_quantizer._amax.data.detach().item()
			logger.info(F'CLIP_GELU: {name:{name_width}} amax: {amax_init:5.2f} -> {amax:5.2f}'     )






def       _snake_case				(		snake_case__   :     int     ):

	for name, mod in model.named_modules():
		if hasattr(_lowerCamelCase	,      '_weight_quantizer'     ) and mod._weight_quantizer.axis is not None:
			A						   =   mod.weight.shape[0]
			A						   =   mod._weight_quantizer._amax.detach()
			A						   =   torch.ones(_lowerCamelCase	,      dtype=amax.dtype	,      device=amax.device     ) * amax
			print(F'expanding {name} {amax} -> {mod._weight_quantizer._amax}'     )






def       _snake_case				(		snake_case__   :     Dict     ):

	for name, mod in model.named_modules():
		if hasattr(_lowerCamelCase	,      '_weight_quantizer'     ):
			if not hasattr(mod.weight_quantizer	,      '_amax'     ):
				print('RECALIB: {name:{name_width}} WARNING: NO AMAX BUFFER'     )
				continue

			# determine which axes to reduce across
			# e.g. a 4D tensor quantized per axis 0 should reduce over (1,2,3)
			A						   =   set() if mod._weight_quantizer.axis is None else set(mod._weight_quantizer.axis     )
			A						   =   set(range(len(mod.weight.size()     )     )     ) - axis_set
			A						   =   pytorch_quantization.utils.reduce_amax(mod.weight	,      axis=_lowerCamelCase	,      keepdims=_lowerCamelCase     ).detach()
			logger.info(F'RECALIB: {name:{name_width}} {mod._weight_quantizer._amax.flatten()} -> {amax.flatten()}'     )
			A						   =   amax






def       _snake_case				(		snake_case__   :     Optional[int]	,      snake_case__   :     Any=25	,      snake_case__   :     Optional[int]=180	,      snake_case__   :     Any=None     ):

	if ignore is None:
		A						   =   []
	elif not isinstance(_lowerCamelCase	,      _lowerCamelCase     ):
		A						   =   [ignore]

	A						   =   0
	for name, mod in model.named_modules():
		if not hasattr(_lowerCamelCase	,      'weight'     ):
			continue
		A						   =   max(_lowerCamelCase	,      len(_lowerCamelCase     )     )

	for name, mod in model.named_modules():
		A						   =   getattr(_lowerCamelCase	,      '_input_quantizer'	,      _lowerCamelCase     )
		A						   =   getattr(_lowerCamelCase	,      '_weight_quantizer'	,      _lowerCamelCase     )
		if not hasattr(_lowerCamelCase	,      'weight'     ):
			continue
		if type(_lowerCamelCase     ) in ignore:
			continue
		if [True for s in ignore if type(_lowerCamelCase     ) is str and s in name]:
			continue
		A						   =   F'Act:{input_q.extra_repr()}'
		A						   =   F'Wgt:{weight_q.extra_repr()}'
		A						   =   F'{name:{name_width}} {act_str} {wgt_str}'
		if len(_lowerCamelCase     ) <= line_width:
			logger.info(_lowerCamelCase     )
		else:
			logger.info(F'{name:{name_width}} {act_str}'     )
			logger.info(F'{"  ":{name_width}} {wgt_str}'     )






def       _snake_case				(		snake_case__   :     Optional[Any]     ):
	A						   =   0
	for name, mod in model.named_modules():
		if isinstance(_lowerCamelCase	,      pytorch_quantization.nn.TensorQuantizer     ):
			print(F'{name:80} {mod}'     )
			count += 1
	print(F'{count} TensorQuantizers found in model'     )






def       _snake_case				(		snake_case__   :     Tuple	,      snake_case__   :     List[str]	,      snake_case__   :     str	,      snake_case__   :     str	,      snake_case__   :     str     ):
	A						   =   getattr(_lowerCamelCase	,      _lowerCamelCase	,      _lowerCamelCase     )
	if quantizer_mod is not None:
		assert hasattr(_lowerCamelCase	,      _lowerCamelCase     )
		setattr(_lowerCamelCase	,      _lowerCamelCase	,      _lowerCamelCase     )
	else:
		logger.warning(F'{name} has no {quantizer}'     )






def       _snake_case				(		snake_case__   :     List[str]	,      snake_case__   :     Any	,      snake_case__   :     Union[str, Any]="both"	,      **snake_case__   :     Any     ):
	A						   =   F'Warning: changing {which} quantizers of {name:{qname_width}}'
	for k, v in kwargs.items():
		s += F' {k}={v}'
		if which in ["input", "both"]:
			set_quantizer(_lowerCamelCase	,      _lowerCamelCase	,      '_input_quantizer'	,      _lowerCamelCase	,      _lowerCamelCase     )
		if which in ["weight", "both"]:
			set_quantizer(_lowerCamelCase	,      _lowerCamelCase	,      '_weight_quantizer'	,      _lowerCamelCase	,      _lowerCamelCase     )
	logger.info(_lowerCamelCase     )






def       _snake_case				(		snake_case__   :     Tuple	,      snake_case__   :     Union[str, Any]	,      **snake_case__   :     List[Any]     ):

	for name, mod in model.named_modules():
		if hasattr(_lowerCamelCase	,      '_input_quantizer'     ) or hasattr(_lowerCamelCase	,      '_weight_quantizer'     ):
			for n in names:
				if re.search(_lowerCamelCase	,      _lowerCamelCase     ):
					set_quantizers(_lowerCamelCase	,      _lowerCamelCase	,      **_lowerCamelCase     )
		elif name.endswith('_quantizer'     ):
			for n in names:
				if re.search(_lowerCamelCase	,      _lowerCamelCase     ):
					A						   =   F'Warning: changing {name:{name_width}}'
					for k, v in kwargs.items():
						s += F' {k}={v}'
						setattr(_lowerCamelCase	,      _lowerCamelCase	,      _lowerCamelCase     )
					logger.info(_lowerCamelCase     )

"""simple docstring"""


from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
    is_vision_available,
)


_lowercase							   =				{
    '''configuration_clip''': [
        '''CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''CLIPConfig''',
        '''CLIPOnnxConfig''',
        '''CLIPTextConfig''',
        '''CLIPVisionConfig''',
    ],
    '''processing_clip''': ['''CLIPProcessor'''],
    '''tokenization_clip''': ['''CLIPTokenizer'''],
}

try:
		if not is_tokenizers_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		_lowercase							   =				['''CLIPTokenizerFast''']

try:
		if not is_vision_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		_lowercase							   =				['''CLIPFeatureExtractor''']
		_lowercase							   =				['''CLIPImageProcessor''']

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		_lowercase							   =				[
		    '''CLIP_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''CLIPModel''',
		    '''CLIPPreTrainedModel''',
		    '''CLIPTextModel''',
		    '''CLIPTextModelWithProjection''',
		    '''CLIPVisionModel''',
		    '''CLIPVisionModelWithProjection''',
		]

try:
		if not is_tf_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		_lowercase							   =				[
		    '''TF_CLIP_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''TFCLIPModel''',
		    '''TFCLIPPreTrainedModel''',
		    '''TFCLIPTextModel''',
		    '''TFCLIPVisionModel''',
		]

try:
		if not is_flax_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		_lowercase							   =				[
		    '''FlaxCLIPModel''',
		    '''FlaxCLIPPreTrainedModel''',
		    '''FlaxCLIPTextModel''',
		    '''FlaxCLIPTextPreTrainedModel''',
		    '''FlaxCLIPVisionModel''',
		    '''FlaxCLIPVisionPreTrainedModel''',
		]


if TYPE_CHECKING:
		from .configuration_clip import (
		    CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
		    CLIPConfig,
		    CLIPOnnxConfig,
		    CLIPTextConfig,
		    CLIPVisionConfig,
		)
		from .processing_clip import CLIPProcessor
		from .tokenization_clip import CLIPTokenizer

		try:
				if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .tokenization_clip_fast import CLIPTokenizerFast

		try:
				if not is_vision_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .feature_extraction_clip import CLIPFeatureExtractor
				from .image_processing_clip import CLIPImageProcessor

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_clip import (
				    CLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
				    CLIPModel,
				    CLIPPreTrainedModel,
				    CLIPTextModel,
				    CLIPTextModelWithProjection,
				    CLIPVisionModel,
				    CLIPVisionModelWithProjection,
				)

		try:
				if not is_tf_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_tf_clip import (
				    TF_CLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TFCLIPModel,
				    TFCLIPPreTrainedModel,
				    TFCLIPTextModel,
				    TFCLIPVisionModel,
				)

		try:
				if not is_flax_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_flax_clip import (
				    FlaxCLIPModel,
				    FlaxCLIPPreTrainedModel,
				    FlaxCLIPTextModel,
				    FlaxCLIPTextPreTrainedModel,
				    FlaxCLIPVisionModel,
				    FlaxCLIPVisionPreTrainedModel,
				)


else:
		import sys

		_lowercase							   =				_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)