
import logging
import math
from functools import partial
from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union

import torch

from .tensor_utils import tensor_tree_map, tree_map




def 	A_ (							snake_case							:   int )      ->    List[Tuple[int, ...]]:



						'''simple docstring'''

						__UpperCamelCase       =			[]
						if isinstance(lowerCamelCase_ ,      lowerCamelCase_ ):
												for v in tree.values():
																		shapes.extend(_fetch_dims(lowerCamelCase_ ) )
						elif isinstance(lowerCamelCase_ ,      (list, tuple) ):
												for t in tree:
																		shapes.extend(_fetch_dims(lowerCamelCase_ ) )
						elif isinstance(lowerCamelCase_ ,      torch.Tensor ):
												shapes.append(tree.shape )
						else:
												raise ValueError('''Not supported''' )

						return shapes




@torch.jit.ignore
def 	A_ (							snake_case							:   Dict ,      snake_case							:   Union[str, Any] )      ->    Tuple[int, ...]:



						'''simple docstring'''

						__UpperCamelCase       =			[]
						for d in reversed(lowerCamelCase_ ):
												idx.append(flat_idx % d )
												__UpperCamelCase       =			flat_idx // d

						return tuple(reversed(lowerCamelCase_ ) )




@torch.jit.ignore
def 	A_ (							snake_case							:   Optional[Any] ,      snake_case							:   List[str] ,      snake_case							:   List[Any] ,      snake_case							:   List[Any] = None ,      snake_case							:   Union[str, Any] = None ,      )      ->    List[Tuple[slice, ...]]:



						'''simple docstring'''

						def reduce_edge_list(snake_case							:   Dict ) -> None:
												__UpperCamelCase       =			True
												for i in range(len(lowerCamelCase_ ) ):
																		__UpperCamelCase       =			-1 * (i + 1)
																		l[reversed_idx] &= tally
																		__UpperCamelCase       =			l[reversed_idx]

						if start_edges is None:
												__UpperCamelCase       =			[s == 0 for s in start]
												reduce_edge_list(lowerCamelCase_ )
						if end_edges is None:
												__UpperCamelCase       =			[e == (d - 1) for e, d in zip(lowerCamelCase_ ,      lowerCamelCase_ )]
												reduce_edge_list(lowerCamelCase_ )

						# Base cases. Either start/end are empty and we're done, or the final,
						# one-dimensional tensor can be simply sliced
						if len(lowerCamelCase_ ) == 0:
												return [()]
						elif len(lowerCamelCase_ ) == 1:
												return [(slice(start[0] ,      end[0] + 1 ),)]

						__UpperCamelCase       =			[]
						__UpperCamelCase       =			[]

						# Dimensions common to start and end can be selected directly
						for s, e in zip(lowerCamelCase_ ,      lowerCamelCase_ ):
												if s == e:
																		path_list.append(slice(lowerCamelCase_ ,      s + 1 ) )
												else:
																		break

						__UpperCamelCase       =			tuple(lowerCamelCase_ )
						__UpperCamelCase       =			len(lowerCamelCase_ )

						# start == end, and we're done
						if divergence_idx == len(lowerCamelCase_ ):
												return [path]

						def upper() -> Tuple[Tuple[slice, ...], ...]:
												assert start_edges is not None
												assert end_edges is not None

												__UpperCamelCase       =			start[divergence_idx]
												return tuple(
												    path + (slice(lowerCamelCase_ ,      sdi + 1 ),) + s
												    for s in _get_minimal_slice_set(
												        start[divergence_idx + 1 :] ,      [d - 1 for d in dims[divergence_idx + 1 :]] ,      dims[divergence_idx + 1 :] ,      start_edges=start_edges[divergence_idx + 1 :] ,      end_edges=[True for _ in end_edges[divergence_idx + 1 :]] ,      ) )

						def lower() -> Tuple[Tuple[slice, ...], ...]:
												assert start_edges is not None
												assert end_edges is not None

												__UpperCamelCase       =			end[divergence_idx]
												return tuple(
												    path + (slice(lowerCamelCase_ ,      edi + 1 ),) + s
												    for s in _get_minimal_slice_set(
												        [0 for _ in start[divergence_idx + 1 :]] ,      end[divergence_idx + 1 :] ,      dims[divergence_idx + 1 :] ,      start_edges=[True for _ in start_edges[divergence_idx + 1 :]] ,      end_edges=end_edges[divergence_idx + 1 :] ,      ) )

						# If both start and end are at the edges of the subtree rooted at
						# divergence_idx, we can just select the whole subtree at once
						if start_edges[divergence_idx] and end_edges[divergence_idx]:
												slices.append(path + (slice(start[divergence_idx] ,      end[divergence_idx] + 1 ),) )
						# If just start is at the edge, we can grab almost all of the subtree,
						# treating only the ragged bottom edge as an edge case
						elif start_edges[divergence_idx]:
												slices.append(path + (slice(start[divergence_idx] ,      end[divergence_idx] ),) )
												slices.extend(lower() )
						# Analogous to the previous case, but the top is ragged this time
						elif end_edges[divergence_idx]:
												slices.extend(upper() )
												slices.append(path + (slice(start[divergence_idx] + 1 ,      end[divergence_idx] + 1 ),) )
						# If both sides of the range are ragged, we need to handle both sides
						# separately. If there's contiguous meat in between them, we can index it
						# in one big chunk
						else:
												slices.extend(upper() )
												__UpperCamelCase       =			end[divergence_idx] - start[divergence_idx]
												if middle_ground > 1:
																		slices.append(path + (slice(start[divergence_idx] + 1 ,      end[divergence_idx] ),) )
												slices.extend(lower() )

						return slices




@torch.jit.ignore
def 	A_ (							snake_case							:   Any ,      snake_case							:   Optional[Any] ,      snake_case							:   int ,      snake_case							:   Optional[Any] )      ->    torch.Tensor:



						'''simple docstring'''

						__UpperCamelCase       =			t.shape[:no_batch_dims]
						__UpperCamelCase       =			list(_flat_idx_to_idx(lowerCamelCase_ ,      lowerCamelCase_ ) )
						# _get_minimal_slice_set is inclusive
						__UpperCamelCase       =			list(_flat_idx_to_idx(flat_end - 1 ,      lowerCamelCase_ ) )

						# Get an ordered list of slices to perform
						__UpperCamelCase       =			_get_minimal_slice_set(
						    lowerCamelCase_ ,      lowerCamelCase_ ,      lowerCamelCase_ ,      )

						__UpperCamelCase       =			[t[s] for s in slices]

						return torch.cat([s.view((-1,) + t.shape[no_batch_dims:] ) for s in sliced_tensors] )




def 	A_ (							snake_case							:   List[str] ,      snake_case							:   Optional[int] ,      snake_case							:   Dict ,      snake_case							:   List[str] ,      snake_case							:   Dict = False ,      snake_case							:   List[str] = None ,      snake_case							:   Tuple = False ,      )      ->    Any:



						'''simple docstring'''

						if not (len(lowerCamelCase_ ) > 0):
												raise ValueError('''Must provide at least one input''' )

						__UpperCamelCase       =			[shape[:no_batch_dims] for shape in _fetch_dims(lowerCamelCase_ )]
						__UpperCamelCase       =			tuple([max(lowerCamelCase_ ) for s in zip(*lowerCamelCase_ )] )

						def _prep_inputs(snake_case							:   Tuple ) -> torch.Tensor:
												if not low_mem:
																		if not sum(t.shape[:no_batch_dims] ) == no_batch_dims:
																								__UpperCamelCase       =			t.expand(orig_batch_dims + t.shape[no_batch_dims:] )
																		__UpperCamelCase       =			t.reshape(-1 ,      *t.shape[no_batch_dims:] )
												else:
																		__UpperCamelCase       =			t.expand(orig_batch_dims + t.shape[no_batch_dims:] )
												return t

						__UpperCamelCase       =			tensor_tree_map(_prep_inputs ,      lowerCamelCase_ )
						__UpperCamelCase       =			None
						if _out is not None:
												__UpperCamelCase       =			tensor_tree_map(lambda snake_case : t.view([-1] + list(t.shape[no_batch_dims:] ) ) ,      _out )

						__UpperCamelCase       =			1
						for d in orig_batch_dims:
												flat_batch_dim *= d

						__UpperCamelCase       =			flat_batch_dim // chunk_size + (flat_batch_dim % chunk_size != 0)

						def _select_chunk(snake_case							:   int ) -> torch.Tensor:
												return t[i : i + chunk_size] if t.shape[0] != 1 else t

						__UpperCamelCase       =			0
						__UpperCamelCase       =			prepped_outputs
						for _ in range(lowerCamelCase_ ):
												# Chunk the input
												if not low_mem:
																		__UpperCamelCase       =			_select_chunk
												else:
																		__UpperCamelCase       =			partial(
																		    _chunk_slice ,      flat_start=lowerCamelCase_ ,      flat_end=min(lowerCamelCase_ ,      i + chunk_size ) ,      no_batch_dims=len(lowerCamelCase_ ) ,      )

												__UpperCamelCase       =			tensor_tree_map(lowerCamelCase_ ,      lowerCamelCase_ )

												# Run the layer on the chunk
												__UpperCamelCase       =			layer(**lowerCamelCase_ )

												# Allocate space for the output
												if out is None:
																		__UpperCamelCase       =			tensor_tree_map(lambda snake_case : t.new_zeros((flat_batch_dim,) + t.shape[1:] ) ,      lowerCamelCase_ )

												# Put the chunk in its pre-allocated space
												if isinstance(lowerCamelCase_ ,      lowerCamelCase_ ):

																		def assign(snake_case							:   Tuple ,      snake_case							:   Any ) -> None:
																								for k, v in da.items():
																														if isinstance(lowerCamelCase_ ,      lowerCamelCase_ ):
																																				assign(lowerCamelCase_ ,      da[k] )
																														else:
																																				if _add_into_out:
																																										v[i : i + chunk_size] += da[k]
																																				else:
																																										__UpperCamelCase       =			da[k]

																		assign(lowerCamelCase_ ,      lowerCamelCase_ )
												elif isinstance(lowerCamelCase_ ,      lowerCamelCase_ ):
																		for xa, xa in zip(lowerCamelCase_ ,      lowerCamelCase_ ):
																								if _add_into_out:
																														xa[i : i + chunk_size] += xa
																								else:
																														__UpperCamelCase       =			xa
												elif isinstance(lowerCamelCase_ ,      torch.Tensor ):
																		if _add_into_out:
																								out[i : i + chunk_size] += output_chunk
																		else:
																								__UpperCamelCase       =			output_chunk
												else:
																		raise ValueError('''Not supported''' )

												i += chunk_size

						__UpperCamelCase       =			tensor_tree_map(lambda snake_case : t.view(orig_batch_dims + t.shape[1:] ) ,      lowerCamelCase_ )

						return out






class        SCREAMING_SNAKE_CASE__    :

							"""simple docstring"""






							def __init__(  self     ,	SCREAMING_SNAKE_CASE_ = 512     ,	)->   Any:





													'''simple docstring'''






													__UpperCamelCase       =			max_chunk_size
													__UpperCamelCase       =			None
													__UpperCamelCase       =			None



							def     A__		(  self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_			)->   Dict:





													'''simple docstring'''






													logging.info('''Tuning chunk size...'''			)

													if min_chunk_size >= self.max_chunk_size:
																			return min_chunk_size

													__UpperCamelCase       =			[2**l for l in range(int(math.log(self.max_chunk_size     ,	2			)			) + 1			)]
													__UpperCamelCase       =			[c for c in candidates if c > min_chunk_size]
													__UpperCamelCase       =			[min_chunk_size] + candidates
													candidates[-1] += 4

													def test_chunk_size(SCREAMING_SNAKE_CASE_			) -> bool:
																			try:
																									with torch.no_grad():
																															fn(*UpperCAmelCase_     ,	chunk_size=UpperCAmelCase_			)
																									return True
																			except RuntimeError:
																									return False

													__UpperCamelCase       =			0
													__UpperCamelCase       =			len(UpperCAmelCase_			) - 1
													while i > min_viable_chunk_size_index:
																			__UpperCamelCase       =			test_chunk_size(candidates[i]			)
																			if not viable:
																									__UpperCamelCase       =			(min_viable_chunk_size_index + i) // 2
																			else:
																									__UpperCamelCase       =			i
																									__UpperCamelCase       =			(i + len(UpperCAmelCase_			) - 1) // 2

													return candidates[min_viable_chunk_size_index]



							def     A__		(  self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_			)->   List[Any]:





													'''simple docstring'''






													__UpperCamelCase       =			True
													for aa, aa in zip(UpperCAmelCase_     ,	UpperCAmelCase_			):
																			assert type(UpperCAmelCase_			) == type(UpperCAmelCase_			)
																			if isinstance(UpperCAmelCase_     ,	(list, tuple)			):
																									consistent &= self._compare_arg_caches(UpperCAmelCase_     ,	UpperCAmelCase_			)
																			elif isinstance(UpperCAmelCase_     ,	UpperCAmelCase_			):
																									__UpperCamelCase       =			[v for _, v in sorted(aa.items()     ,	key=lambda SCREAMING_SNAKE_CASE_			: x[0]			)]
																									__UpperCamelCase       =			[v for _, v in sorted(aa.items()     ,	key=lambda SCREAMING_SNAKE_CASE_			: x[0]			)]
																									consistent &= self._compare_arg_caches(UpperCAmelCase_     ,	UpperCAmelCase_			)
																			else:
																									consistent &= aa == aa

													return consistent



							def     A__		(  self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	)->   List[str]:





													'''simple docstring'''






													__UpperCamelCase       =			True
													__UpperCamelCase       =			tree_map(lambda SCREAMING_SNAKE_CASE_			: a.shape if isinstance(UpperCAmelCase_     ,	torch.Tensor			) else a     ,	UpperCAmelCase_     ,	UpperCAmelCase_			)
													if self.cached_arg_data is not None:
																			# If args have changed shape/value, we need to re-tune
																			assert len(self.cached_arg_data			) == len(UpperCAmelCase_			)
																			__UpperCamelCase       =			self._compare_arg_caches(self.cached_arg_data     ,	UpperCAmelCase_			)
													else:
																			# Otherwise, we can reuse the precomputed value
																			__UpperCamelCase       =			False

													if not consistent:
																			__UpperCamelCase       =			self._determine_favorable_chunk_size(
																			    UpperCAmelCase_     ,	UpperCAmelCase_     ,	UpperCAmelCase_     ,	)
																			__UpperCamelCase       =			arg_data

													assert self.cached_chunk_size is not None

													return self.cached_chunk_size






from .configuration_bert_masked import MaskedBertConfig
from .modeling_bert_masked import (
    MaskedBertForMultipleChoice,
    MaskedBertForQuestionAnswering,
    MaskedBertForSequenceClassification,
    MaskedBertForTokenClassification,
    MaskedBertModel,
)
from .modules import *




