



'''simple docstring'''



import argparse
import logging
import os
from datetime import datetime

import numpy as np
import torch
from torch import nn
from torch.utils.data import DataLoader, RandomSampler, TensorDataset
from tqdm import tqdm

from transformers import GPTaLMHeadModel


lowerCAmelCase_     :       Optional[Any]								=						logging.getLogger(__name__)
def 					UpperCAmelCase   (						A  :				Any					,		A  :				Optional[Any]				):
							# save results
							if os.path.exists(A				):
														if os.path.exists(os.path.join(A					,		'''config.json'''				)				) and os.path.isfile(
														    os.path.join(A					,		'''config.json'''				)				):
																					os.remove(os.path.join(A					,		'''config.json'''				)				)
														if os.path.exists(os.path.join(A					,		'''pytorch_model.bin'''				)				) and os.path.isfile(
														    os.path.join(A					,		'''pytorch_model.bin'''				)				):
																					os.remove(os.path.join(A					,		'''pytorch_model.bin'''				)				)
							else:
														os.makedirs(A				)
							model.save_pretrained(A				)
def 					UpperCAmelCase   (						A  :				Union[str, Any]					,		A  :				List[Any]=False				):
							SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			2
							if unlogit:
														SCREAMING_SNAKE_CASE					:      Any     					=			torch.pow(A					,		A				)
							SCREAMING_SNAKE_CASE					:      Any     					=			p * torch.log(A				)
							SCREAMING_SNAKE_CASE					:      Dict     					=			0
							return -plogp.sum(dim=-1				)
def 					UpperCAmelCase   (						A  :				Optional[int]				):
							logger.info('''lv, h >\t''' + '''\t'''.join(F"""{x + 1}""" for x in range(len(A				)				)				)				)
							for row in range(len(A				)				):
														if tensor.dtype != torch.long:
																					logger.info(F"""layer {row + 1}:\t""" + '''\t'''.join(F"""{x:.5f}""" for x in tensor[row].cpu().data				)				)
														else:
																					logger.info(F"""layer {row + 1}:\t""" + '''\t'''.join(F"""{x:d}""" for x in tensor[row].cpu().data				)				)
def 					UpperCAmelCase   (						A  :				Dict					,		A  :				List[Any]					,		A  :				Any					,		A  :				Optional[int]=True					,		A  :				Optional[Any]=True					,		A  :				str=None					,		A  :				List[str]=False				):
							SCREAMING_SNAKE_CASE					:      List[str]     					=			model.config.num_hidden_layers, model.config.num_attention_heads
							SCREAMING_SNAKE_CASE					:      int     					=			torch.zeros(A					,		A				).to(args.device				)
							SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			torch.zeros(A					,		A				).to(args.device				)

							if head_mask is None:
														SCREAMING_SNAKE_CASE					:      List[str]     					=			torch.ones(A					,		A				).to(args.device				)

							head_mask.requires_grad_(requires_grad=A				)
							# If actually pruned attention multi-head, set head mask to None to avoid shape mismatch
							if actually_pruned:
														SCREAMING_SNAKE_CASE					:      Optional[int]     					=			None

							SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			0.0
							SCREAMING_SNAKE_CASE					:      Any     					=			0.0
							for step, inputs in enumerate(tqdm(A					,		desc='''Iteration'''					,		disable=args.local_rank not in [-1, 0]				)				):
														SCREAMING_SNAKE_CASE					:      int     					=			tuple(t.to(args.device				) for t in inputs				)
														(SCREAMING_SNAKE_CASE)					:      str     					=			inputs

														# Do a forward pass (not with torch.no_grad() since we need gradients for importance score - see below)
														SCREAMING_SNAKE_CASE					:      Dict     					=			model(A					,		labels=A					,		head_mask=A				)
														#  (loss), lm_logits, presents, (all hidden_states), (attentions)
														SCREAMING_SNAKE_CASE					:      Dict     					=			(
														    outputs[0],
														    outputs[1],
														    outputs[-1],
														)  # Loss and logits are the first, attention the last
														loss.backward()  # Backpropagate to populate the gradients in the head mask
														total_loss += loss.detach().cpu().numpy()
														if compute_entropy:
																					for layer, attn in enumerate(A				):
																												SCREAMING_SNAKE_CASE					:      List[str]     					=			entropy(attn.detach()					,		A				)
																												attn_entropy[layer] += masked_entropy.sum(-1				).sum(0				).sum(0				).detach()

														if compute_importance:
																					head_importance += head_mask.grad.abs().detach()
														tot_tokens += torch.ones_like(A				).float().detach().sum().data

							# Normalize
							attn_entropy /= tot_tokens
							head_importance /= tot_tokens
							# Layerwise importance normalization
							if not args.dont_normalize_importance_by_layer:
														SCREAMING_SNAKE_CASE					:      Any     					=			2
														SCREAMING_SNAKE_CASE					:      List[str]     					=			torch.pow(torch.pow(A					,		A				).sum(-1				)					,		1 / exponent				)
														head_importance /= norm_by_layer.unsqueeze(-1				) + 1e-2_0

							if not args.dont_normalize_global_importance:
														SCREAMING_SNAKE_CASE					:      List[str]     					=			(head_importance - head_importance.min()) / (head_importance.max() - head_importance.min())

							# Print matrices
							if compute_entropy:
														logger.info('''Attention entropies'''				)
														print_ad_tensor(A				)
							if compute_importance:
														logger.info('''Head importance scores'''				)
														print_ad_tensor(A				)
							logger.info('''Head ranked by importance scores'''				)
							SCREAMING_SNAKE_CASE					:      Dict     					=			torch.zeros(head_importance.numel()					,		dtype=torch.long					,		device=args.device				)
							SCREAMING_SNAKE_CASE					:      List[Any]     					=			torch.arange(
							    head_importance.numel()					,		device=args.device				)
							SCREAMING_SNAKE_CASE					:      Any     					=			head_ranks.view_as(A				)
							print_ad_tensor(A				)
							return attn_entropy, head_importance, total_loss
def 					UpperCAmelCase   (						A  :				str					,		A  :				List[str]					,		A  :				Dict				):
							SCREAMING_SNAKE_CASE					:      List[str]     					=			compute_heads_importance(A					,		A					,		A					,		compute_entropy=A				)
							SCREAMING_SNAKE_CASE					:      Dict     					=			1 / loss  # instead of downsteam score use the LM loss
							logger.info('''Pruning: original score: %f, threshold: %f'''					,		A					,		original_score * args.masking_threshold				)

							SCREAMING_SNAKE_CASE					:      List[str]     					=			torch.ones_like(A				)
							SCREAMING_SNAKE_CASE					:      Any     					=			max(1					,		int(new_head_mask.numel() * args.masking_amount				)				)

							SCREAMING_SNAKE_CASE					:      List[str]     					=			original_score
							while current_score >= original_score * args.masking_threshold:
														SCREAMING_SNAKE_CASE					:      Dict     					=			new_head_mask.clone().detach()  # save current head mask
														# heads from least important to most - keep only not-masked heads
														SCREAMING_SNAKE_CASE					:      Tuple     					=			float('''Inf'''				)
														SCREAMING_SNAKE_CASE					:      Optional[int]     					=			head_importance.view(-1				).sort()[1]

														if len(A				) <= num_to_mask:
																					print('''BREAK BY num_to_mask'''				)
																					break

														# mask heads
														SCREAMING_SNAKE_CASE					:      Any     					=			current_heads_to_mask[:num_to_mask]
														logger.info('''Heads to mask: %s'''					,		str(current_heads_to_mask.tolist()				)				)
														SCREAMING_SNAKE_CASE					:      Tuple     					=			new_head_mask.view(-1				)
														SCREAMING_SNAKE_CASE					:      Any     					=			0.0
														SCREAMING_SNAKE_CASE					:      Any     					=			new_head_mask.view_as(A				)
														SCREAMING_SNAKE_CASE					:      Dict     					=			new_head_mask.clone().detach()
														print_ad_tensor(A				)

														# Compute metric and head importance again
														SCREAMING_SNAKE_CASE					:      List[str]     					=			compute_heads_importance(
														    A					,		A					,		A					,		compute_entropy=A					,		head_mask=A				)
														SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			1 / loss
														logger.info(
														    '''Masking: current score: %f, remaining heads %d (%.1f percents)'''					,		A					,		new_head_mask.sum()					,		new_head_mask.sum() / new_head_mask.numel() * 100					,		)

							logger.info('''Final head mask'''				)
							print_ad_tensor(A				)
							np.save(os.path.join(args.output_dir					,		'''head_mask.npy'''				)					,		head_mask.detach().cpu().numpy()				)

							return head_mask
def 					UpperCAmelCase   (						A  :				int					,		A  :				Optional[int]					,		A  :				Any					,		A  :				Dict				):
							SCREAMING_SNAKE_CASE					:      str     					=			datetime.now()
							SCREAMING_SNAKE_CASE					:      Any     					=			compute_heads_importance(
							    A					,		A					,		A					,		compute_entropy=A					,		compute_importance=A					,		head_mask=A				)
							SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			1 / loss
							SCREAMING_SNAKE_CASE					:      Dict     					=			datetime.now() - before_time

							SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			sum(p.numel() for p in model.parameters()				)
							SCREAMING_SNAKE_CASE					:      Any     					=			{
							    layer: (1 - head_mask[layer].long()).nonzero().squeeze().tolist() for layer in range(len(A				)				)
							}

							for k, v in heads_to_prune.items():
														if isinstance(A					,		A				):
																					SCREAMING_SNAKE_CASE					:      int     					=			[
																					    v,
																					]

							assert sum(len(A				) for h in heads_to_prune.values()				) == (1 - head_mask.long()).sum().item()
							model.prune_heads(A				)
							SCREAMING_SNAKE_CASE					:      Dict     					=			sum(p.numel() for p in model.parameters()				)

							SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			datetime.now()
							SCREAMING_SNAKE_CASE					:      int     					=			compute_heads_importance(
							    A					,		A					,		A					,		compute_entropy=A					,		compute_importance=A					,		head_mask=A					,		actually_pruned=A					,		)

							SCREAMING_SNAKE_CASE					:      Tuple     					=			1 / loss
							SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			datetime.now() - before_time

							logger.info(
							    '''Pruning: original num of params: %.2e, after pruning %.2e (%.1f percents)'''					,		A					,		A					,		pruned_num_params / original_num_params * 100					,		)
							logger.info('''Pruning: score with masking: %f score with pruning: %f'''					,		A					,		A				)
							logger.info('''Pruning: speed ratio (original timing / new timing): %f percents'''					,		original_time / new_time * 100				)
							save_model(A					,		args.output_dir				)
def 					UpperCAmelCase   (						):
							SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			argparse.ArgumentParser()
							# Required parameters
							parser.add_argument(
							    '''--data_dir'''					,		default=A					,		type=A					,		required=A					,		help='''The input data dir. Should contain the .tsv files (or other data files) for the task.'''					,		)
							parser.add_argument(
							    '''--model_name_or_path'''					,		default=A					,		type=A					,		required=A					,		help='''Path to pretrained model or model identifier from huggingface.co/models'''					,		)
							parser.add_argument(
							    '''--output_dir'''					,		default=A					,		type=A					,		required=A					,		help='''The output directory where the model predictions and checkpoints will be written.'''					,		)

							# Other parameters
							parser.add_argument(
							    '''--config_name'''					,		default=''''''					,		type=A					,		help='''Pretrained config name or path if not the same as model_name_or_path'''					,		)
							parser.add_argument(
							    '''--tokenizer_name'''					,		default=''''''					,		type=A					,		help='''Pretrained tokenizer name or path if not the same as model_name_or_path'''					,		)
							parser.add_argument(
							    '''--cache_dir'''					,		default=A					,		type=A					,		help='''Where do you want to store the pre-trained models downloaded from s3'''					,		)
							parser.add_argument(
							    '''--data_subset'''					,		type=A					,		default=-1					,		help='''If > 0: limit the data to a subset of data_subset instances.'''				)
							parser.add_argument(
							    '''--overwrite_output_dir'''					,		action='''store_true'''					,		help='''Whether to overwrite data in output directory'''				)
							parser.add_argument(
							    '''--overwrite_cache'''					,		action='''store_true'''					,		help='''Overwrite the cached training and evaluation sets'''				)

							parser.add_argument(
							    '''--dont_normalize_importance_by_layer'''					,		action='''store_true'''					,		help='''Don\'t normalize importance score by layers'''				)
							parser.add_argument(
							    '''--dont_normalize_global_importance'''					,		action='''store_true'''					,		help='''Don\'t normalize all importance scores between 0 and 1'''					,		)

							parser.add_argument(
							    '''--try_masking'''					,		action='''store_true'''					,		help='''Whether to try to mask head until a threshold of accuracy.'''				)
							parser.add_argument(
							    '''--masking_threshold'''					,		default=0.9					,		type=A					,		help='''masking threshold in term of metrics (stop masking when metric < threshold * original metric value).'''					,		)
							parser.add_argument(
							    '''--masking_amount'''					,		default=0.1					,		type=A					,		help='''Amount to heads to masking at each masking step.'''				)
							parser.add_argument('''--metric_name'''					,		default='''acc'''					,		type=A					,		help='''Metric to use for head masking.'''				)

							parser.add_argument(
							    '''--max_seq_length'''					,		default=128					,		type=A					,		help=(
							        '''The maximum total input sequence length after WordPiece tokenization. \n'''
							        '''Sequences longer than this will be truncated, sequences shorter padded.'''
							    )					,		)
							parser.add_argument('''--batch_size'''					,		default=1					,		type=A					,		help='''Batch size.'''				)

							parser.add_argument('''--seed'''					,		type=A					,		default=42				)
							parser.add_argument('''--local_rank'''					,		type=A					,		default=-1					,		help='''local_rank for distributed training on gpus'''				)
							parser.add_argument('''--no_cuda'''					,		action='''store_true'''					,		help='''Whether not to use CUDA when available'''				)
							parser.add_argument('''--server_ip'''					,		type=A					,		default=''''''					,		help='''Can be used for distant debugging.'''				)
							parser.add_argument('''--server_port'''					,		type=A					,		default=''''''					,		help='''Can be used for distant debugging.'''				)
							SCREAMING_SNAKE_CASE					:      Tuple     					=			parser.parse_args()

							if args.server_ip and args.server_port:
														# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
														import ptvsd

														print('''Waiting for debugger attach'''				)
														ptvsd.enable_attach(address=(args.server_ip, args.server_port)					,		redirect_output=A				)
														ptvsd.wait_for_attach()

							# Setup devices and distributed training
							if args.local_rank == -1 or args.no_cuda:
														SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			torch.device('''cuda''' if torch.cuda.is_available() and not args.no_cuda else '''cpu'''				)
														SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			0 if args.no_cuda else torch.cuda.device_count()
							else:
														torch.cuda.set_device(args.local_rank				)
														SCREAMING_SNAKE_CASE					:      str     					=			torch.device('''cuda'''					,		args.local_rank				)
														SCREAMING_SNAKE_CASE					:      Tuple     					=			1
														torch.distributed.init_process_group(backend='''nccl'''				)  # Initializes the distributed backend

							# Setup logging
							logging.basicConfig(level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN				)
							logger.info('''device: {} n_gpu: {}, distributed: {}'''.format(args.device					,		args.n_gpu					,		bool(args.local_rank != -1				)				)				)

							SCREAMING_SNAKE_CASE					:      int     					=			GPTaLMHeadModel.from_pretrained(args.model_name_or_path				)

							# Distributed and parallel training
							model.to(args.device				)
							if args.local_rank != -1:
														SCREAMING_SNAKE_CASE					:      int     					=			nn.parallel.DistributedDataParallel(
														    A					,		device_ids=[args.local_rank]					,		output_device=args.local_rank					,		find_unused_parameters=A				)
							elif args.n_gpu > 1:
														SCREAMING_SNAKE_CASE					:      int     					=			nn.DataParallel(A				)

							# Print/save training arguments
							os.makedirs(args.output_dir					,		exist_ok=A				)
							torch.save(A					,		os.path.join(args.output_dir					,		'''run_args.bin'''				)				)
							logger.info('''Training/evaluation parameters %s'''					,		A				)

							# Prepare dataset
							SCREAMING_SNAKE_CASE					:      Dict     					=			np.concatenate(
							    [
							        np.loadtxt(args.data_dir					,		dtype=np.intaa				),
							    ]				)
							SCREAMING_SNAKE_CASE					:      List[Any]     					=			(torch.from_numpy(A				),)
							SCREAMING_SNAKE_CASE					:      Any     					=			TensorDataset(*A				)
							SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			RandomSampler(A				)
							SCREAMING_SNAKE_CASE					:      Tuple     					=			DataLoader(A					,		sampler=A					,		batch_size=args.batch_size				)

							# Compute head entropy and importance score
							compute_heads_importance(A					,		A					,		A				)

							# Try head masking (set heads to zero until the score goes under a threshole)
							# and head pruning (remove masked heads and see the effect on the network)
							if args.try_masking and args.masking_threshold > 0.0 and args.masking_threshold < 1.0:
														SCREAMING_SNAKE_CASE					:      Optional[int]     					=			mask_heads(A					,		A					,		A				)
														prune_heads(A					,		A					,		A					,		A				)


if __name__ == "__main__":
							main()









'''simple docstring'''



from __future__ import annotations

import inspect
import unittest
from typing import List, Tuple

from transformers import RegNetConfig
from transformers.testing_utils import require_tf, require_vision, slow
from transformers.utils import cached_property, is_tf_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
							import tensorflow as tf

							from transformers import TF_REGNET_PRETRAINED_MODEL_ARCHIVE_LIST, TFRegNetForImageClassification, TFRegNetModel


if is_vision_available():
							from PIL import Image

							from transformers import AutoImageProcessor
class 	lowerCamelCase_							:







						def __init__(  self   :      Optional[Any]				,    lowerCAmelCase__   :      Tuple				,    lowerCAmelCase__   :      int=3				,    lowerCAmelCase__   :      Tuple=32				,    lowerCAmelCase__   :      Union[str, Any]=3				,    lowerCAmelCase__   :      Dict=10				,    lowerCAmelCase__   :      List[Any]=[10, 20, 30, 40]				,    lowerCAmelCase__   :      List[Any]=[1, 1, 2, 1]				,    lowerCAmelCase__   :      Optional[int]=True				,    lowerCAmelCase__   :      Any=True				,    lowerCAmelCase__   :      List[str]="relu"				,    lowerCAmelCase__   :      int=3				,    lowerCAmelCase__   :      Tuple=None				,    ):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Tuple     					=			parent
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			batch_size
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			image_size
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			num_channels
													SCREAMING_SNAKE_CASE					:      str     					=			embeddings_size
													SCREAMING_SNAKE_CASE					:      str     					=			hidden_sizes
													SCREAMING_SNAKE_CASE					:      Tuple     					=			depths
													SCREAMING_SNAKE_CASE					:      Dict     					=			is_training
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			use_labels
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			hidden_act
													SCREAMING_SNAKE_CASE					:      str     					=			num_labels
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			scope
													SCREAMING_SNAKE_CASE					:      List[Any]     					=			len(lowerCAmelCase__				)







						def 					__lowercase    (  self   :      str				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]				)

													SCREAMING_SNAKE_CASE					:      List[Any]     					=			None
													if self.use_labels:
																				SCREAMING_SNAKE_CASE					:      List[Any]     					=			ids_tensor([self.batch_size]				,    self.num_labels				)

													SCREAMING_SNAKE_CASE					:      Optional[int]     					=			self.get_config()
													return config, pixel_values, labels







						def 					__lowercase    (  self   :      Any				):

													"""simple docstring"""
													return RegNetConfig(
													    num_channels=self.num_channels				,    embeddings_size=self.embeddings_size				,    hidden_sizes=self.hidden_sizes				,    depths=self.depths				,    hidden_act=self.hidden_act				,    num_labels=self.num_labels				,    )







						def 					__lowercase    (  self   :      Dict				,    lowerCAmelCase__   :      Tuple				,    lowerCAmelCase__   :      Union[str, Any]				,    lowerCAmelCase__   :      Optional[int]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      int     					=			TFRegNetModel(config=lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      int     					=			model(lowerCAmelCase__				,    training=lowerCAmelCase__				)
													# expected last hidden states: B, C, H // 32, W // 32
													self.parent.assertEqual(
													    result.last_hidden_state.shape				,    (self.batch_size, self.hidden_sizes[-1], self.image_size // 32, self.image_size // 32)				,    )







						def 					__lowercase    (  self   :      int				,    lowerCAmelCase__   :      str				,    lowerCAmelCase__   :      int				,    lowerCAmelCase__   :      List[str]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			self.num_labels
													SCREAMING_SNAKE_CASE					:      Any     					=			TFRegNetForImageClassification(lowerCAmelCase__				)
													SCREAMING_SNAKE_CASE					:      Dict     					=			model(lowerCAmelCase__				,    labels=lowerCAmelCase__				,    training=lowerCAmelCase__				)
													self.parent.assertEqual(result.logits.shape				,    (self.batch_size, self.num_labels)				)







						def 					__lowercase    (  self   :      List[Any]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      int     					=			self.prepare_config_and_inputs()
													SCREAMING_SNAKE_CASE					,SCREAMING_SNAKE_CASE					,SCREAMING_SNAKE_CASE					:      List[str]     					=			config_and_inputs
													SCREAMING_SNAKE_CASE					:      Tuple     					=			{'''pixel_values''': pixel_values}
													return config, inputs_dict


@require_tf
class 	lowerCamelCase_							(					snake_case_    ,				snake_case_    ,				unittest.TestCase			):
						_lowerCAmelCase     :   str	     =				(TFRegNetModel, TFRegNetForImageClassification) if is_tf_available() else ()
						_lowerCAmelCase     :   str	     =				(
						    {'feature-extraction': TFRegNetModel, 'image-classification': TFRegNetForImageClassification}
						    if is_tf_available()
						    else {}
						)

						_lowerCAmelCase     :   Tuple	     =				False
						_lowerCAmelCase     :   List[Any]	     =				False
						_lowerCAmelCase     :   List[str]	     =				False
						_lowerCAmelCase     :   str	     =				False
						_lowerCAmelCase     :   str	     =				False







						def 					__lowercase    (  self   :      Union[str, Any]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Tuple     					=			TFRegNetModelTester(self				)
													SCREAMING_SNAKE_CASE					:      Dict     					=			ConfigTester(self				,    config_class=lowerCAmelCase__				,    has_text_modality=lowerCAmelCase__				)







						def 					__lowercase    (  self   :      List[str]				):

													"""simple docstring"""
													return







						@unittest.skip(reason='''RegNet does not use inputs_embeds'''				)
						def 					__lowercase    (  self   :      Optional[int]				):

													"""simple docstring"""
													pass







						@unittest.skipIf(
						    not is_tf_available() or len(tf.config.list_physical_devices('''GPU'''				)				) == 0				,    reason='''TF does not support backprop for grouped convolutions on CPU.'''				,    )
						@slow
						def 					__lowercase    (  self   :      int				):

													"""simple docstring"""
													super().test_keras_fit()







						@unittest.skip(reason='''RegNet does not support input and output embeddings'''				)
						def 					__lowercase    (  self   :      Dict				):

													"""simple docstring"""
													pass







						def 					__lowercase    (  self   :      int				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					,SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			self.model_tester.prepare_config_and_inputs_for_common()

													for model_class in self.all_model_classes:
																				SCREAMING_SNAKE_CASE					:      int     					=			model_class(lowerCAmelCase__				)
																				SCREAMING_SNAKE_CASE					:      Any     					=			inspect.signature(model.call				)
																				# signature.parameters is an OrderedDict => so arg_names order is deterministic
																				SCREAMING_SNAKE_CASE					:      Tuple     					=			[*signature.parameters.keys()]

																				SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			['''pixel_values''']
																				self.assertListEqual(arg_names[:1]				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      Any				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_model(*lowerCAmelCase__				)







						def 					__lowercase    (  self   :      Dict				):

													"""simple docstring"""
													def check_hidden_states_output(lowerCAmelCase__   :      Any				,    lowerCAmelCase__   :      int				,    lowerCAmelCase__   :      Tuple				):
																				SCREAMING_SNAKE_CASE					:      List[Any]     					=			model_class(lowerCAmelCase__				)
																				SCREAMING_SNAKE_CASE					:      Dict     					=			model(**self._prepare_for_class(lowerCAmelCase__				,    lowerCAmelCase__				)				,    training=lowerCAmelCase__				)

																				SCREAMING_SNAKE_CASE					:      int     					=			outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

																				SCREAMING_SNAKE_CASE					:      int     					=			self.model_tester.num_stages
																				self.assertEqual(len(lowerCAmelCase__				)				,    expected_num_stages + 1				)

																				# RegNet's feature maps are of shape (batch_size, num_channels, height, width)
																				self.assertListEqual(
																				    list(hidden_states[0].shape[-2:]				)				,    [self.model_tester.image_size // 2, self.model_tester.image_size // 2]				,    )

													SCREAMING_SNAKE_CASE					,SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			self.model_tester.prepare_config_and_inputs_for_common()
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			['''basic''', '''bottleneck''']
													for model_class in self.all_model_classes:
																				for layer_type in layers_type:
																											SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			layer_type
																											SCREAMING_SNAKE_CASE					:      Tuple     					=			True
																											check_hidden_states_output(lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				)

																											# check that output_hidden_states also work using config
																											del inputs_dict["output_hidden_states"]
																											SCREAMING_SNAKE_CASE					:      Optional[int]     					=			True

																											check_hidden_states_output(lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				)







						def 					__lowercase    (  self   :      Union[str, Any]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					,SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			self.model_tester.prepare_config_and_inputs_for_common()

													def check_equivalence(lowerCAmelCase__   :      Tuple				,    lowerCAmelCase__   :      int				,    lowerCAmelCase__   :      Dict				,    lowerCAmelCase__   :      str={}				):
																				SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			model(lowerCAmelCase__				,    return_dict=lowerCAmelCase__				,    **lowerCAmelCase__				)
																				SCREAMING_SNAKE_CASE					:      Any     					=			model(lowerCAmelCase__				,    return_dict=lowerCAmelCase__				,    **lowerCAmelCase__				).to_tuple()

																				def recursive_check(lowerCAmelCase__   :      Dict				,    lowerCAmelCase__   :      str				):
																											if isinstance(lowerCAmelCase__				,    (List, Tuple)				):
																																		for tuple_iterable_value, dict_iterable_value in zip(lowerCAmelCase__				,    lowerCAmelCase__				):
																																									recursive_check(lowerCAmelCase__				,    lowerCAmelCase__				)
																											elif tuple_object is None:
																																		return
																											else:
																																		self.assertTrue(
																																		    all(tf.equal(lowerCAmelCase__				,    lowerCAmelCase__				)				)				,    msg=(
																																		        '''Tuple and dict output are not equal. Difference:'''
																																		        F""" {tf.math.reduce_max(tf.abs(tuple_object - dict_object				)				)}"""
																																		    )				,    )

																											recursive_check(lowerCAmelCase__				,    lowerCAmelCase__				)

													for model_class in self.all_model_classes:
																				SCREAMING_SNAKE_CASE					:      List[Any]     					=			model_class(lowerCAmelCase__				)

																				SCREAMING_SNAKE_CASE					:      int     					=			self._prepare_for_class(lowerCAmelCase__				,    lowerCAmelCase__				)
																				SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			self._prepare_for_class(lowerCAmelCase__				,    lowerCAmelCase__				)
																				check_equivalence(lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				)

																				SCREAMING_SNAKE_CASE					:      Optional[int]     					=			self._prepare_for_class(lowerCAmelCase__				,    lowerCAmelCase__				,    return_labels=lowerCAmelCase__				)
																				SCREAMING_SNAKE_CASE					:      str     					=			self._prepare_for_class(lowerCAmelCase__				,    lowerCAmelCase__				,    return_labels=lowerCAmelCase__				)
																				check_equivalence(lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				)

																				SCREAMING_SNAKE_CASE					:      Tuple     					=			self._prepare_for_class(lowerCAmelCase__				,    lowerCAmelCase__				)
																				SCREAMING_SNAKE_CASE					:      Dict     					=			self._prepare_for_class(lowerCAmelCase__				,    lowerCAmelCase__				)
																				check_equivalence(lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    {'''output_hidden_states''': True}				)

																				SCREAMING_SNAKE_CASE					:      Any     					=			self._prepare_for_class(lowerCAmelCase__				,    lowerCAmelCase__				,    return_labels=lowerCAmelCase__				)
																				SCREAMING_SNAKE_CASE					:      List[Any]     					=			self._prepare_for_class(lowerCAmelCase__				,    lowerCAmelCase__				,    return_labels=lowerCAmelCase__				)
																				check_equivalence(lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    {'''output_hidden_states''': True}				)







						def 					__lowercase    (  self   :      List[Any]				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      Dict     					=			self.model_tester.prepare_config_and_inputs()
													self.model_tester.create_and_check_for_image_classification(*lowerCAmelCase__				)







						@slow
						def 					__lowercase    (  self   :      Any				):

													"""simple docstring"""
													for model_name in TF_REGNET_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																				SCREAMING_SNAKE_CASE					:      Optional[Any]     					=			TFRegNetModel.from_pretrained(lowerCAmelCase__				)
																				self.assertIsNotNone(lowerCAmelCase__				)


def 					UpperCAmelCase   (						):
							SCREAMING_SNAKE_CASE					:      List[Any]     					=			Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''				)
							return image






@require_tf
@require_vision
class 	lowerCamelCase_							(					unittest.TestCase			):







						@cached_property
						def 					__lowercase    (  self   :      List[str]				):

													"""simple docstring"""
													return (
													    AutoImageProcessor.from_pretrained(TF_REGNET_PRETRAINED_MODEL_ARCHIVE_LIST[0]				)
													    if is_vision_available()
													    else None
													)







						@slow
						def 					__lowercase    (  self   :      str				):

													"""simple docstring"""
													SCREAMING_SNAKE_CASE					:      str     					=			TFRegNetForImageClassification.from_pretrained(TF_REGNET_PRETRAINED_MODEL_ARCHIVE_LIST[0]				)

													SCREAMING_SNAKE_CASE					:      Dict     					=			self.default_image_processor
													SCREAMING_SNAKE_CASE					:      int     					=			prepare_img()
													SCREAMING_SNAKE_CASE					:      List[str]     					=			image_processor(images=lowerCAmelCase__				,    return_tensors='''tf'''				)

													# forward pass
													SCREAMING_SNAKE_CASE					:      Union[str, Any]     					=			model(**lowerCAmelCase__				,    training=lowerCAmelCase__				)

													# verify the logits
													SCREAMING_SNAKE_CASE					:      List[str]     					=			tf.TensorShape((1, 10_00)				)
													self.assertEqual(outputs.logits.shape				,    lowerCAmelCase__				)

													SCREAMING_SNAKE_CASE					:      List[str]     					=			tf.constant([-0.4180, -1.5051, -3.4836]				)

													tf.debugging.assert_near(outputs.logits[0, :3]				,    lowerCAmelCase__				,    atol=1e-4				)




