






'''simple docstring'''




import collections.abc
from typing import Optional, Tuple, Union

import torch
import torch.utils.checkpoint
from torch import nn
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss

from ...activations import ACTaFN
from ...modeling_outputs import BaseModelOutputWithNoAttention, ImageClassifierOutputWithNoAttention
from ...modeling_utils import PreTrainedModel
from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging
from .configuration_poolformer import PoolFormerConfig


lowerCAmelCase__          =						logging.get_logger(__name__)

# General docstring
lowerCAmelCase__          =						'PoolFormerConfig'

# Base docstring
lowerCAmelCase__          =						'sail/poolformer_s12'
lowerCAmelCase__          =						[1, 512, 7, 7]

# Image classification docstring
lowerCAmelCase__          =						'sail/poolformer_s12'
lowerCAmelCase__          =						'tabby, tabby cat'

lowerCAmelCase__          =						[
    'sail/poolformer_s12',
    # See all PoolFormer models at https://huggingface.co/models?filter=poolformer
]







def       __UpperCAmelCase						(  lowerCamelCase_     ,      lowerCamelCase_ = 0.0     ,      lowerCamelCase_ = False)							->     Union[str, Any]:
		if drop_prob == 0.0 or not training:
				return input
		UpperCamelCase__				:					Optional[Any]					=					1 - drop_prob
		UpperCamelCase__				:					Dict					=					(input.shape[0],) + (1,) * (input.ndim - 1)  # work with diff dim tensors, not just 2D ConvNets
		UpperCamelCase__				:					Dict					=					keep_prob + torch.rand(lowerCamelCase_     ,      dtype=input.dtype     ,      device=input.device)
		random_tensor.floor_()  # binarize
		UpperCamelCase__				:					int					=					input.div(lowerCamelCase_) * random_tensor
		return output







class       __lowercase					(nn.Module    ):

	def __init__(      self   :						Optional[Any]  ,					UpperCAmelCase_   :						Optional[float] = None):
			super().__init__()
			UpperCamelCase__				:					Any					=					drop_prob

	def      __UpperCamelCase							(      self   :						List[str]  ,					UpperCAmelCase_   :						torch.Tensor):
			return drop_path(UpperCAmelCase_  ,					self.drop_prob  ,					self.training)





	def      __UpperCamelCase							(      self   :						Any):
			return "p={}".format(self.drop_prob)







class       __lowercase					(nn.Module    ):

	def __init__(      self   :						Union[str, Any]  ,					UpperCAmelCase_   :						Union[str, Any]  ,					UpperCAmelCase_   :						str  ,					UpperCAmelCase_   :						Tuple  ,					UpperCAmelCase_   :						Tuple  ,					UpperCAmelCase_   :						Union[str, Any]  ,					UpperCAmelCase_   :						Optional[Any]=None):
			super().__init__()
			UpperCamelCase__				:					Dict					=					patch_size if isinstance(UpperCAmelCase_  ,					collections.abc.Iterable) else (patch_size, patch_size)
			UpperCamelCase__				:					Dict					=					stride if isinstance(UpperCAmelCase_  ,					collections.abc.Iterable) else (stride, stride)
			UpperCamelCase__				:					Optional[Any]					=					padding if isinstance(UpperCAmelCase_  ,					collections.abc.Iterable) else (padding, padding)

			UpperCamelCase__				:					Dict					=					nn.Convad(UpperCAmelCase_  ,					UpperCAmelCase_  ,					kernel_size=UpperCAmelCase_  ,					stride=UpperCAmelCase_  ,					padding=UpperCAmelCase_)
			UpperCamelCase__				:					int					=					norm_layer(UpperCAmelCase_) if norm_layer else nn.Identity()





	def      __UpperCamelCase							(      self   :						str  ,					UpperCAmelCase_   :						List[str]):
			UpperCamelCase__				:					Optional[int]					=					self.projection(UpperCAmelCase_)
			UpperCamelCase__				:					Dict					=					self.norm(UpperCAmelCase_)
			return embeddings







class       __lowercase					(nn.GroupNorm    ):

	def __init__(      self   :						Optional[int]  ,					UpperCAmelCase_   :						Optional[Any]  ,					**UpperCAmelCase_   :						Tuple):
			super().__init__(1  ,					UpperCAmelCase_  ,					**UpperCAmelCase_)







class       __lowercase					(nn.Module    ):

	def __init__(      self   :						Union[str, Any]  ,					UpperCAmelCase_   :						Any):
			super().__init__()
			UpperCamelCase__				:					Optional[int]					=					nn.AvgPoolad(UpperCAmelCase_  ,					stride=1  ,					padding=pool_size // 2  ,					count_include_pad=UpperCAmelCase_)





	def      __UpperCamelCase							(      self   :						Any  ,					UpperCAmelCase_   :						Optional[int]):
			return self.pool(UpperCAmelCase_) - hidden_states







class       __lowercase					(nn.Module    ):

	def __init__(      self   :						int  ,					UpperCAmelCase_   :						int  ,					UpperCAmelCase_   :						Any  ,					UpperCAmelCase_   :						List[Any]  ,					UpperCAmelCase_   :						List[Any]):
			super().__init__()
			UpperCamelCase__				:					Union[str, Any]					=					nn.Convad(UpperCAmelCase_  ,					UpperCAmelCase_  ,					1)
			UpperCamelCase__				:					str					=					nn.Convad(UpperCAmelCase_  ,					UpperCAmelCase_  ,					1)
			UpperCamelCase__				:					Dict					=					PoolFormerDropPath(UpperCAmelCase_)
			if isinstance(config.hidden_act  ,					UpperCAmelCase_):
					UpperCamelCase__				:					Tuple					=					ACTaFN[config.hidden_act]
			else:
					UpperCamelCase__				:					int					=					config.hidden_act





	def      __UpperCamelCase							(      self   :						Optional[int]  ,					UpperCAmelCase_   :						Tuple):
			UpperCamelCase__				:					Dict					=					self.conva(UpperCAmelCase_)
			UpperCamelCase__				:					List[str]					=					self.act_fn(UpperCAmelCase_)
			UpperCamelCase__				:					List[Any]					=					self.drop(UpperCAmelCase_)
			UpperCamelCase__				:					Dict					=					self.conva(UpperCAmelCase_)
			UpperCamelCase__				:					List[Any]					=					self.drop(UpperCAmelCase_)

			return hidden_states







class       __lowercase					(nn.Module    ):

	def __init__(      self   :						List[Any]  ,					UpperCAmelCase_   :						List[str]  ,					UpperCAmelCase_   :						Union[str, Any]  ,					UpperCAmelCase_   :						int  ,					UpperCAmelCase_   :						List[Any]  ,					UpperCAmelCase_   :						Dict  ,					UpperCAmelCase_   :						Any):
			super().__init__()
			UpperCamelCase__				:					Any					=					PoolFormerPooling(UpperCAmelCase_)
			UpperCamelCase__				:					Dict					=					PoolFormerOutput(UpperCAmelCase_  ,					UpperCAmelCase_  ,					UpperCAmelCase_  ,					UpperCAmelCase_)
			UpperCamelCase__				:					List[str]					=					PoolFormerGroupNorm(UpperCAmelCase_)
			UpperCamelCase__				:					str					=					PoolFormerGroupNorm(UpperCAmelCase_)

			# Useful for training neural nets
			UpperCamelCase__				:					str					=					PoolFormerDropPath(UpperCAmelCase_) if drop_path > 0.0 else nn.Identity()
			UpperCamelCase__				:					Optional[Any]					=					config.use_layer_scale
			if config.use_layer_scale:
					UpperCamelCase__				:					Union[str, Any]					=					nn.Parameter(
					    config.layer_scale_init_value * torch.ones((UpperCAmelCase_))  ,					requires_grad=UpperCAmelCase_)
					UpperCamelCase__				:					Union[str, Any]					=					nn.Parameter(
					    config.layer_scale_init_value * torch.ones((UpperCAmelCase_))  ,					requires_grad=UpperCAmelCase_)





	def      __UpperCamelCase							(      self   :						Tuple  ,					UpperCAmelCase_   :						str):
			if self.use_layer_scale:
					UpperCamelCase__				:					Dict					=					self.pooling(self.before_norm(UpperCAmelCase_))
					UpperCamelCase__				:					int					=					self.layer_scale_a.unsqueeze(-1).unsqueeze(-1) * pooling_output
					# First residual connection
					UpperCamelCase__				:					Tuple					=					hidden_states + self.drop_path(UpperCAmelCase_)
					UpperCamelCase__				:					int					=					()

					UpperCamelCase__				:					List[str]					=					self.output(self.after_norm(UpperCAmelCase_))
					UpperCamelCase__				:					Optional[Any]					=					self.layer_scale_a.unsqueeze(-1).unsqueeze(-1) * layer_output
					# Second residual connection
					UpperCamelCase__				:					Tuple					=					hidden_states + self.drop_path(UpperCAmelCase_)

					UpperCamelCase__				:					Dict					=					(output,) + outputs
					return outputs

			else:
					UpperCamelCase__				:					Union[str, Any]					=					self.drop_path(self.pooling(self.before_norm(UpperCAmelCase_)))
					# First residual connection
					UpperCamelCase__				:					Optional[Any]					=					pooling_output + hidden_states
					UpperCamelCase__				:					Dict					=					()

					# Second residual connection inside the PoolFormerOutput block
					UpperCamelCase__				:					int					=					self.drop_path(self.output(self.after_norm(UpperCAmelCase_)))
					UpperCamelCase__				:					int					=					hidden_states + layer_output

					UpperCamelCase__				:					Optional[int]					=					(output,) + outputs
					return outputs







class       __lowercase					(nn.Module    ):

	def __init__(      self   :						Dict  ,					UpperCAmelCase_   :						Optional[Any]):
			super().__init__()
			UpperCamelCase__				:					Optional[int]					=					config
			# stochastic depth decay rule
			UpperCamelCase__				:					Optional[Any]					=					[x.item() for x in torch.linspace(0  ,					config.drop_path_rate  ,					sum(config.depths))]

			# patch embeddings
			UpperCamelCase__				:					List[str]					=					[]
			for i in range(config.num_encoder_blocks):
					embeddings.append(
					    PoolFormerEmbeddings(
					        patch_size=config.patch_sizes[i]  ,					stride=config.strides[i]  ,					padding=config.padding[i]  ,					num_channels=config.num_channels if i == 0 else config.hidden_sizes[i - 1]  ,					hidden_size=config.hidden_sizes[i]  ,					))
			UpperCamelCase__				:					Union[str, Any]					=					nn.ModuleList(UpperCAmelCase_)

			# Transformer blocks
			UpperCamelCase__				:					Optional[int]					=					[]
			UpperCamelCase__				:					Union[str, Any]					=					0
			for i in range(config.num_encoder_blocks):
					# each block consists of layers
					UpperCamelCase__				:					Dict					=					[]
					if i != 0:
							cur += config.depths[i - 1]
					for j in range(config.depths[i]):
							layers.append(
							    PoolFormerLayer(
							        UpperCAmelCase_  ,					num_channels=config.hidden_sizes[i]  ,					pool_size=config.pool_size  ,					hidden_size=config.hidden_sizes[i]  ,					intermediate_size=int(config.hidden_sizes[i] * config.mlp_ratio)  ,					drop_path=dpr[cur + j]  ,					))
					blocks.append(nn.ModuleList(UpperCAmelCase_))

			UpperCamelCase__				:					Union[str, Any]					=					nn.ModuleList(UpperCAmelCase_)





	def      __UpperCamelCase							(      self   :						Dict  ,					UpperCAmelCase_   :						Any  ,					UpperCAmelCase_   :						Optional[Any]=False  ,					UpperCAmelCase_   :						int=True):
			UpperCamelCase__				:					Optional[int]					=					() if output_hidden_states else None

			UpperCamelCase__				:					List[Any]					=					pixel_values
			for idx, layers in enumerate(zip(self.patch_embeddings  ,					self.block)):
					UpperCamelCase__,	UpperCamelCase__				:					Optional[int]					=					layers
					# Get patch embeddings from hidden_states
					UpperCamelCase__				:					str					=					embedding_layer(UpperCAmelCase_)
					# Send the embeddings through the blocks
					for _, blk in enumerate(UpperCAmelCase_):
							UpperCamelCase__				:					str					=					blk(UpperCAmelCase_)
							UpperCamelCase__				:					Any					=					layer_outputs[0]

					if output_hidden_states:
							UpperCamelCase__				:					Optional[int]					=					all_hidden_states + (hidden_states,)

			if not return_dict:
					return tuple(v for v in [hidden_states, all_hidden_states] if v is not None)

			return BaseModelOutputWithNoAttention(last_hidden_state=UpperCAmelCase_  ,					hidden_states=UpperCAmelCase_)







class       __lowercase					(__lowerCamelCase    ):
	_lowerCamelCase              =  PoolFormerConfig
	_lowerCamelCase              =  '''poolformer'''
	_lowerCamelCase              =  '''pixel_values'''
	_lowerCamelCase              =  True

	def      __UpperCamelCase							(      self   :						int  ,					UpperCAmelCase_   :						Dict):
			if isinstance(UpperCAmelCase_  ,					(nn.Linear, nn.Convad)):
					module.weight.data.normal_(mean=0.0  ,					std=self.config.initializer_range)
					if module.bias is not None:
							module.bias.data.zero_()
			elif isinstance(UpperCAmelCase_  ,					nn.LayerNorm):
					module.bias.data.zero_()
					module.weight.data.fill_(1.0)





	def      __UpperCamelCase							(      self   :						int  ,					UpperCAmelCase_   :						Optional[Any]  ,					UpperCAmelCase_   :						Optional[Any]=False):
			if isinstance(UpperCAmelCase_  ,					UpperCAmelCase_):
					UpperCamelCase__				:					Optional[int]					=					value


lowerCAmelCase__          =						R'\n    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use\n    it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and\n    behavior.\n\n    Parameters:\n        config ([`PoolFormerConfig`]): Model configuration class with all the parameters of the model.\n            Initializing with a config file does not load the weights associated with the model, only the\n            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.\n'

lowerCAmelCase__          =						R'\n    Args:\n        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):\n            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See\n            [`PoolFormerImageProcessor.__call__`] for details.\n'







@add_start_docstrings(
    '''The bare PoolFormer Model transformer outputting raw hidden-states without any specific head on top.'''   ,				__lowerCamelCase   ,				)
class       __lowercase					(__lowerCamelCase    ):

	def __init__(      self   :						Tuple  ,					UpperCAmelCase_   :						int):
			super().__init__(UpperCAmelCase_)
			UpperCamelCase__				:					Optional[Any]					=					config

			UpperCamelCase__				:					Optional[Any]					=					PoolFormerEncoder(UpperCAmelCase_)

			# Initialize weights and apply final processing
			self.post_init()

	def      __UpperCamelCase							(      self   :						int):
			return self.embeddings.patch_embeddings





	@add_start_docstrings_to_model_forward(UpperCAmelCase_)
	@add_code_sample_docstrings(
	    checkpoint=_CHECKPOINT_FOR_DOC  ,					output_type=UpperCAmelCase_  ,					config_class=_CONFIG_FOR_DOC  ,					modality='vision'  ,					expected_output=_EXPECTED_OUTPUT_SHAPE  ,					)
	def      __UpperCamelCase							(      self   :						Union[str, Any]  ,					UpperCAmelCase_   :						Optional[torch.FloatTensor] = None  ,					UpperCAmelCase_   :						Optional[bool] = None  ,					UpperCAmelCase_   :						Optional[bool] = None  ,					):
			UpperCamelCase__				:					str					=					(
			    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
			)
			UpperCamelCase__				:					Optional[Any]					=					return_dict if return_dict is not None else self.config.use_return_dict

			if pixel_values is None:
					raise ValueError('You have to specify pixel_values')

			UpperCamelCase__				:					Tuple					=					self.encoder(
			    UpperCAmelCase_  ,					output_hidden_states=UpperCAmelCase_  ,					return_dict=UpperCAmelCase_  ,					)
			UpperCamelCase__				:					Dict					=					encoder_outputs[0]

			if not return_dict:
					return (sequence_output, None) + encoder_outputs[1:]

			return BaseModelOutputWithNoAttention(
			    last_hidden_state=UpperCAmelCase_  ,					hidden_states=encoder_outputs.hidden_states  ,					)







class       __lowercase					(nn.Module    ):

	def __init__(      self   :						Union[str, Any]  ,					UpperCAmelCase_   :						Tuple):
			super().__init__()
			UpperCamelCase__				:					List[str]					=					nn.Linear(config.hidden_size  ,					config.hidden_size)





	def      __UpperCamelCase							(      self   :						List[Any]  ,					UpperCAmelCase_   :						List[Any]):
			UpperCamelCase__				:					Optional[Any]					=					self.dense(UpperCAmelCase_)
			return output







@add_start_docstrings(
    '''
    PoolFormer Model transformer with an image classification head on top
    '''   ,				__lowerCamelCase   ,				)
class       __lowercase					(__lowerCamelCase    ):

	def __init__(      self   :						Optional[int]  ,					UpperCAmelCase_   :						Dict):
			super().__init__(UpperCAmelCase_)
			UpperCamelCase__				:					Dict					=					config.num_labels
			UpperCamelCase__				:					int					=					PoolFormerModel(UpperCAmelCase_)

			# Final norm
			UpperCamelCase__				:					int					=					PoolFormerGroupNorm(config.hidden_sizes[-1])
			# Classifier head
			UpperCamelCase__				:					str					=					(
			    nn.Linear(config.hidden_sizes[-1]  ,					config.num_labels) if config.num_labels > 0 else nn.Identity()
			)

			# Initialize weights and apply final processing
			self.post_init()





	@add_start_docstrings_to_model_forward(UpperCAmelCase_)
	@add_code_sample_docstrings(
	    checkpoint=_IMAGE_CLASS_CHECKPOINT  ,					output_type=UpperCAmelCase_  ,					config_class=_CONFIG_FOR_DOC  ,					expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT  ,					)
	def      __UpperCamelCase							(      self   :						List[str]  ,					UpperCAmelCase_   :						Optional[torch.FloatTensor] = None  ,					UpperCAmelCase_   :						Optional[torch.LongTensor] = None  ,					UpperCAmelCase_   :						Optional[bool] = None  ,					UpperCAmelCase_   :						Optional[bool] = None  ,					):
			UpperCamelCase__				:					int					=					return_dict if return_dict is not None else self.config.use_return_dict

			UpperCamelCase__				:					int					=					self.poolformer(
			    UpperCAmelCase_  ,					output_hidden_states=UpperCAmelCase_  ,					return_dict=UpperCAmelCase_  ,					)

			UpperCamelCase__				:					List[Any]					=					outputs[0]

			UpperCamelCase__				:					Optional[Any]					=					self.classifier(self.norm(UpperCAmelCase_).mean([-2, -1]))

			UpperCamelCase__				:					Any					=					None
			if labels is not None:
					if self.config.problem_type is None:
							if self.num_labels == 1:
									UpperCamelCase__				:					List[Any]					=					'regression'
							elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
									UpperCamelCase__				:					List[Any]					=					'single_label_classification'
							else:
									UpperCamelCase__				:					List[Any]					=					'multi_label_classification'

					if self.config.problem_type == "regression":
							UpperCamelCase__				:					Any					=					MSELoss()
							if self.num_labels == 1:
									UpperCamelCase__				:					Tuple					=					loss_fct(logits.squeeze()  ,					labels.squeeze())
							else:
									UpperCamelCase__				:					str					=					loss_fct(UpperCAmelCase_  ,					UpperCAmelCase_)
					elif self.config.problem_type == "single_label_classification":
							UpperCamelCase__				:					List[Any]					=					CrossEntropyLoss()
							UpperCamelCase__				:					List[Any]					=					loss_fct(logits.view(-1  ,					self.num_labels)  ,					labels.view(-1))
					elif self.config.problem_type == "multi_label_classification":
							UpperCamelCase__				:					Optional[int]					=					BCEWithLogitsLoss()
							UpperCamelCase__				:					Union[str, Any]					=					loss_fct(UpperCAmelCase_  ,					UpperCAmelCase_)

			if not return_dict:
					UpperCamelCase__				:					str					=					(logits,) + outputs[2:]
					return ((loss,) + output) if loss is not None else output

			return ImageClassifierOutputWithNoAttention(loss=UpperCAmelCase_  ,					logits=UpperCAmelCase_  ,					hidden_states=outputs.hidden_states)












'''simple docstring'''




import logging
import math
from functools import partial
from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union

import torch

from .tensor_utils import tensor_tree_map, tree_map







def       __UpperCAmelCase						(  lowerCamelCase_)							->     List[Tuple[int, ...]]:
		UpperCamelCase__				:					int					=					[]
		if isinstance(lowerCamelCase_     ,      lowerCamelCase_):
				for v in tree.values():
						shapes.extend(_fetch_dims(lowerCamelCase_))
		elif isinstance(lowerCamelCase_     ,      (list, tuple)):
				for t in tree:
						shapes.extend(_fetch_dims(lowerCamelCase_))
		elif isinstance(lowerCamelCase_     ,      torch.Tensor):
				shapes.append(tree.shape)
		else:
				raise ValueError('Not supported')

		return shapes







@torch.jit.ignore
def       __UpperCAmelCase						(  lowerCamelCase_     ,      lowerCamelCase_)							->     Tuple[int, ...]:
		UpperCamelCase__				:					int					=					[]
		for d in reversed(lowerCamelCase_):
				idx.append(flat_idx % d)
				UpperCamelCase__				:					Any					=					flat_idx // d

		return tuple(reversed(lowerCamelCase_))







@torch.jit.ignore
def       __UpperCAmelCase						(  lowerCamelCase_     ,      lowerCamelCase_     ,      lowerCamelCase_     ,      lowerCamelCase_ = None     ,      lowerCamelCase_ = None     ,      )							->     List[Tuple[slice, ...]]:

		# start_edges and end_edges both indicate whether, starting from any given
		# dimension, the start/end index is at the top/bottom edge of the
		# corresponding tensor, modeled as a tree
		def reduce_edge_list(lowerCamelCase_) -> None:
				UpperCamelCase__				:					Tuple					=					True
				for i in range(len(lowerCamelCase_)):
						UpperCamelCase__				:					List[Any]					=					-1 * (i + 1)
						l[reversed_idx] &= tally
						UpperCamelCase__				:					Optional[Any]					=					l[reversed_idx]

		if start_edges is None:
				UpperCamelCase__				:					int					=					[s == 0 for s in start]
				reduce_edge_list(lowerCamelCase_)
		if end_edges is None:
				UpperCamelCase__				:					List[str]					=					[e == (d - 1) for e, d in zip(lowerCamelCase_     ,      lowerCamelCase_)]
				reduce_edge_list(lowerCamelCase_)

		# Base cases. Either start/end are empty and we're done, or the final,
		# one-dimensional tensor can be simply sliced
		if len(lowerCamelCase_) == 0:
				return [()]
		elif len(lowerCamelCase_) == 1:
				return [(slice(start[0]     ,      end[0] + 1),)]

		UpperCamelCase__				:					List[Tuple[slice, ...]]					=					[]
		UpperCamelCase__				:					List[slice]					=					[]

		# Dimensions common to start and end can be selected directly
		for s, e in zip(lowerCamelCase_     ,      lowerCamelCase_):
				if s == e:
						path_list.append(slice(lowerCamelCase_     ,      s + 1))
				else:
						break

		UpperCamelCase__				:					Tuple[slice, ...]					=					tuple(lowerCamelCase_)
		UpperCamelCase__				:					Dict					=					len(lowerCamelCase_)

		# start == end, and we're done
		if divergence_idx == len(lowerCamelCase_):
				return [path]

		def upper() -> Tuple[Tuple[slice, ...], ...]:
				assert start_edges is not None
				assert end_edges is not None

				UpperCamelCase__				:					str					=					start[divergence_idx]
				return tuple(
				    path + (slice(lowerCamelCase_     ,      sdi + 1),) + s
				    for s in _get_minimal_slice_set(
				        start[divergence_idx + 1 :]     ,      [d - 1 for d in dims[divergence_idx + 1 :]]     ,      dims[divergence_idx + 1 :]     ,      start_edges=start_edges[divergence_idx + 1 :]     ,      end_edges=[True for _ in end_edges[divergence_idx + 1 :]]     ,      ))

		def lower() -> Tuple[Tuple[slice, ...], ...]:
				assert start_edges is not None
				assert end_edges is not None

				UpperCamelCase__				:					Optional[int]					=					end[divergence_idx]
				return tuple(
				    path + (slice(lowerCamelCase_     ,      edi + 1),) + s
				    for s in _get_minimal_slice_set(
				        [0 for _ in start[divergence_idx + 1 :]]     ,      end[divergence_idx + 1 :]     ,      dims[divergence_idx + 1 :]     ,      start_edges=[True for _ in start_edges[divergence_idx + 1 :]]     ,      end_edges=end_edges[divergence_idx + 1 :]     ,      ))

		# If both start and end are at the edges of the subtree rooted at
		# divergence_idx, we can just select the whole subtree at once
		if start_edges[divergence_idx] and end_edges[divergence_idx]:
				slices.append(path + (slice(start[divergence_idx]     ,      end[divergence_idx] + 1),))
		# If just start is at the edge, we can grab almost all of the subtree,
		# treating only the ragged bottom edge as an edge case
		elif start_edges[divergence_idx]:
				slices.append(path + (slice(start[divergence_idx]     ,      end[divergence_idx]),))
				slices.extend(lower())
		# Analogous to the previous case, but the top is ragged this time
		elif end_edges[divergence_idx]:
				slices.extend(upper())
				slices.append(path + (slice(start[divergence_idx] + 1     ,      end[divergence_idx] + 1),))
		# If both sides of the range are ragged, we need to handle both sides
		# separately. If there's contiguous meat in between them, we can index it
		# in one big chunk
		else:
				slices.extend(upper())
				UpperCamelCase__				:					Dict					=					end[divergence_idx] - start[divergence_idx]
				if middle_ground > 1:
						slices.append(path + (slice(start[divergence_idx] + 1     ,      end[divergence_idx]),))
				slices.extend(lower())

		return slices







@torch.jit.ignore
def       __UpperCAmelCase						(  lowerCamelCase_     ,      lowerCamelCase_     ,      lowerCamelCase_     ,      lowerCamelCase_)							->     torch.Tensor:
		UpperCamelCase__				:					List[Any]					=					t.shape[:no_batch_dims]
		UpperCamelCase__				:					Optional[int]					=					list(_flat_idx_to_idx(lowerCamelCase_     ,      lowerCamelCase_))
		# _get_minimal_slice_set is inclusive
		UpperCamelCase__				:					Dict					=					list(_flat_idx_to_idx(flat_end - 1     ,      lowerCamelCase_))

		# Get an ordered list of slices to perform
		UpperCamelCase__				:					int					=					_get_minimal_slice_set(
		    lowerCamelCase_     ,      lowerCamelCase_     ,      lowerCamelCase_     ,      )

		UpperCamelCase__				:					List[Any]					=					[t[s] for s in slices]

		return torch.cat([s.view((-1,) + t.shape[no_batch_dims:]) for s in sliced_tensors])







def       __UpperCAmelCase						(  lowerCamelCase_     ,      lowerCamelCase_     ,      lowerCamelCase_     ,      lowerCamelCase_     ,      lowerCamelCase_ = False     ,      lowerCamelCase_ = None     ,      lowerCamelCase_ = False     ,      )							->     Any:
		if not (len(lowerCamelCase_) > 0):
				raise ValueError('Must provide at least one input')

		UpperCamelCase__				:					int					=					[shape[:no_batch_dims] for shape in _fetch_dims(lowerCamelCase_)]
		UpperCamelCase__				:					int					=					tuple([max(lowerCamelCase_) for s in zip(*lowerCamelCase_)])

		def _prep_inputs(lowerCamelCase_) -> torch.Tensor:
				if not low_mem:
						if not sum(t.shape[:no_batch_dims]) == no_batch_dims:
								UpperCamelCase__				:					List[Any]					=					t.expand(orig_batch_dims + t.shape[no_batch_dims:])
						UpperCamelCase__				:					Optional[int]					=					t.reshape(-1     ,      *t.shape[no_batch_dims:])
				else:
						UpperCamelCase__				:					Optional[int]					=					t.expand(orig_batch_dims + t.shape[no_batch_dims:])
				return t

		UpperCamelCase__				:					Dict[str, Any]					=					tensor_tree_map(_prep_inputs     ,      lowerCamelCase_)
		UpperCamelCase__				:					int					=					None
		if _out is not None:
				UpperCamelCase__				:					Optional[int]					=					tensor_tree_map(lambda lowerCamelCase_: t.view([-1] + list(t.shape[no_batch_dims:]))     ,      _out)

		UpperCamelCase__				:					Dict					=					1
		for d in orig_batch_dims:
				flat_batch_dim *= d

		UpperCamelCase__				:					int					=					flat_batch_dim // chunk_size + (flat_batch_dim % chunk_size != 0)

		def _select_chunk(lowerCamelCase_) -> torch.Tensor:
				return t[i : i + chunk_size] if t.shape[0] != 1 else t

		UpperCamelCase__				:					List[Any]					=					0
		UpperCamelCase__				:					Optional[Any]					=					prepped_outputs
		for _ in range(lowerCamelCase_):
				# Chunk the input
				if not low_mem:
						UpperCamelCase__				:					str					=					_select_chunk
				else:
						UpperCamelCase__				:					List[Any]					=					partial(
						    _chunk_slice     ,      flat_start=lowerCamelCase_     ,      flat_end=min(lowerCamelCase_     ,      i + chunk_size)     ,      no_batch_dims=len(lowerCamelCase_)     ,      )

				UpperCamelCase__				:					Dict[str, Any]					=					tensor_tree_map(lowerCamelCase_     ,      lowerCamelCase_)

				# Run the layer on the chunk
				UpperCamelCase__				:					List[Any]					=					layer(**lowerCamelCase_)

				# Allocate space for the output
				if out is None:
						UpperCamelCase__				:					Optional[int]					=					tensor_tree_map(lambda lowerCamelCase_: t.new_zeros((flat_batch_dim,) + t.shape[1:])     ,      lowerCamelCase_)

				# Put the chunk in its pre-allocated space
				if isinstance(lowerCamelCase_     ,      lowerCamelCase_):

						def assign(lowerCamelCase_     ,      lowerCamelCase_) -> None:
								for k, v in da.items():
										if isinstance(lowerCamelCase_     ,      lowerCamelCase_):
												assign(lowerCamelCase_     ,      da[k])
										else:
												if _add_into_out:
														v[i : i + chunk_size] += da[k]
												else:
														UpperCamelCase__				:					List[str]					=					da[k]

						assign(lowerCamelCase_     ,      lowerCamelCase_)
				elif isinstance(lowerCamelCase_     ,      lowerCamelCase_):
						for xa, xa in zip(lowerCamelCase_     ,      lowerCamelCase_):
								if _add_into_out:
										xa[i : i + chunk_size] += xa
								else:
										UpperCamelCase__				:					int					=					xa
				elif isinstance(lowerCamelCase_     ,      torch.Tensor):
						if _add_into_out:
								out[i : i + chunk_size] += output_chunk
						else:
								UpperCamelCase__				:					Dict					=					output_chunk
				else:
						raise ValueError('Not supported')

				i += chunk_size

		UpperCamelCase__				:					int					=					tensor_tree_map(lambda lowerCamelCase_: t.view(orig_batch_dims + t.shape[1:])     ,      lowerCamelCase_)

		return out







class       __lowercase					:

	def __init__(      self   :						List[str]  ,					UpperCAmelCase_   :						int = 512  ,					):
			UpperCamelCase__				:					str					=					max_chunk_size
			UpperCamelCase__				:					Optional[int]					=					None
			UpperCamelCase__				:					Optional[tuple]					=					None

	def      __UpperCamelCase							(      self   :						str  ,					UpperCAmelCase_   :						Callable  ,					UpperCAmelCase_   :						tuple  ,					UpperCAmelCase_   :						int):
			logging.info('Tuning chunk size...')

			if min_chunk_size >= self.max_chunk_size:
					return min_chunk_size

			UpperCamelCase__				:					List[int]					=					[2**l for l in range(int(math.log(self.max_chunk_size  ,					2)) + 1)]
			UpperCamelCase__				:					List[Any]					=					[c for c in candidates if c > min_chunk_size]
			UpperCamelCase__				:					List[Any]					=					[min_chunk_size] + candidates
			candidates[-1] += 4

			def test_chunk_size(UpperCAmelCase_   :						int) -> bool:
					try:
							with torch.no_grad():
									fn(*UpperCAmelCase_  ,					chunk_size=UpperCAmelCase_)
							return True
					except RuntimeError:
							return False

			UpperCamelCase__				:					Tuple					=					0
			UpperCamelCase__				:					Dict					=					len(UpperCAmelCase_) - 1
			while i > min_viable_chunk_size_index:
					UpperCamelCase__				:					Optional[int]					=					test_chunk_size(candidates[i])
					if not viable:
							UpperCamelCase__				:					Tuple					=					(min_viable_chunk_size_index + i) // 2
					else:
							UpperCamelCase__				:					Optional[int]					=					i
							UpperCamelCase__				:					Dict					=					(i + len(UpperCAmelCase_) - 1) // 2

			return candidates[min_viable_chunk_size_index]

	def      __UpperCamelCase							(      self   :						Any  ,					UpperCAmelCase_   :						Iterable  ,					UpperCAmelCase_   :						Iterable):
			UpperCamelCase__				:					List[str]					=					True
			for aa, aa in zip(UpperCAmelCase_  ,					UpperCAmelCase_):
					assert type(UpperCAmelCase_) == type(UpperCAmelCase_)
					if isinstance(UpperCAmelCase_  ,					(list, tuple)):
							consistent &= self._compare_arg_caches(UpperCAmelCase_  ,					UpperCAmelCase_)
					elif isinstance(UpperCAmelCase_  ,					UpperCAmelCase_):
							UpperCamelCase__				:					Union[str, Any]					=					[v for _, v in sorted(aa.items()  ,					key=lambda UpperCAmelCase_: x[0])]
							UpperCamelCase__				:					str					=					[v for _, v in sorted(aa.items()  ,					key=lambda UpperCAmelCase_: x[0])]
							consistent &= self._compare_arg_caches(UpperCAmelCase_  ,					UpperCAmelCase_)
					else:
							consistent &= aa == aa

			return consistent





	def      __UpperCamelCase							(      self   :						List[Any]  ,					UpperCAmelCase_   :						Callable  ,					UpperCAmelCase_   :						tuple  ,					UpperCAmelCase_   :						int  ,					):
			UpperCamelCase__				:					List[Any]					=					True
			UpperCamelCase__				:					tuple					=					tree_map(lambda UpperCAmelCase_: a.shape if isinstance(UpperCAmelCase_  ,					torch.Tensor) else a  ,					UpperCAmelCase_  ,					UpperCAmelCase_)
			if self.cached_arg_data is not None:
					# If args have changed shape/value, we need to re-tune
					assert len(self.cached_arg_data) == len(UpperCAmelCase_)
					UpperCamelCase__				:					Union[str, Any]					=					self._compare_arg_caches(self.cached_arg_data  ,					UpperCAmelCase_)
			else:
					# Otherwise, we can reuse the precomputed value
					UpperCamelCase__				:					Optional[int]					=					False

			if not consistent:
					UpperCamelCase__				:					Tuple					=					self._determine_favorable_chunk_size(
					    UpperCAmelCase_  ,					UpperCAmelCase_  ,					UpperCAmelCase_  ,					)
					UpperCamelCase__				:					Optional[Any]					=					arg_data

			assert self.cached_chunk_size is not None

			return self.cached_chunk_size




