'''simple docstring'''





import logging
import math
from functools import partial
from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union

import torch

from .tensor_utils import tensor_tree_map, tree_map







def 						A__  (					__A	):







    '''simple docstring'''
    _lowerCamelCase     :		Optional[Any] 						=						[]
    if isinstance(__A   , __A	):
        for v in tree.values():
            shapes.extend(_fetch_dims(__A	)	)
    elif isinstance(__A   , (list, tuple)	):
        for t in tree:
            shapes.extend(_fetch_dims(__A	)	)
    elif isinstance(__A   , torch.Tensor	):
        shapes.append(tree.shape	)
    else:
        raise ValueError("""Not supported"""	)

    return shapes







@torch.jit.ignore
def 						A__  (					__A   , __A	):







    '''simple docstring'''
    _lowerCamelCase     :		str 						=						[]
    for d in reversed(__A	):
        idx.append(flat_idx % d	)
        _lowerCamelCase     :		List[str] 						=						flat_idx // d

    return tuple(reversed(__A	)	)







@torch.jit.ignore
def 						A__  (					__A   , __A   , __A   , __A = None   , __A = None   , ):







    '''simple docstring'''

    # start_edges and end_edges both indicate whether, starting from any given
    # dimension, the start/end index is at the top/bottom edge of the
    # corresponding tensor, modeled as a tree
    def reduce_edge_list(__A	) -> None:
        _lowerCamelCase     :		Optional[int] 						=						True
        for i in range(len(__A	)	):
            _lowerCamelCase     :		int 						=						-1 * (i + 1)
            l[reversed_idx] &= tally
            _lowerCamelCase     :		str 						=						l[reversed_idx]

    if start_edges is None:
        _lowerCamelCase     :		Optional[int] 						=						[s == 0 for s in start]
        reduce_edge_list(__A	)
    if end_edges is None:
        _lowerCamelCase     :		List[str] 						=						[e == (d - 1) for e, d in zip(__A   , __A	)]
        reduce_edge_list(__A	)

    # Base cases. Either start/end are empty and we're done, or the final,
    # one-dimensional tensor can be simply sliced
    if len(__A	) == 0:
        return [()]
    elif len(__A	) == 1:
        return [(slice(start[0]   , end[0] + 1	),)]

    _lowerCamelCase     :		List[Tuple[slice, ...]] 						=						[]
    _lowerCamelCase     :		List[slice] 						=						[]

    # Dimensions common to start and end can be selected directly
    for s, e in zip(__A   , __A	):
        if s == e:
            path_list.append(slice(__A   , s + 1	)	)
        else:
            break

    _lowerCamelCase     :		Tuple[slice, ...] 						=						tuple(__A	)
    _lowerCamelCase     :		Tuple 						=						len(__A	)

    # start == end, and we're done
    if divergence_idx == len(__A	):
        return [path]

    def upper() -> Tuple[Tuple[slice, ...], ...]:
        assert start_edges is not None
        assert end_edges is not None

        _lowerCamelCase     :		List[Any] 						=						start[divergence_idx]
        return tuple(
            path + (slice(__A   , sdi + 1	),) + s
            for s in _get_minimal_slice_set(
                start[divergence_idx + 1 :]   , [d - 1 for d in dims[divergence_idx + 1 :]]   , dims[divergence_idx + 1 :]   , start_edges=start_edges[divergence_idx + 1 :]   , end_edges=[True for _ in end_edges[divergence_idx + 1 :]]   , )	)

    def lower() -> Tuple[Tuple[slice, ...], ...]:
        assert start_edges is not None
        assert end_edges is not None

        _lowerCamelCase     :		List[str] 						=						end[divergence_idx]
        return tuple(
            path + (slice(__A   , edi + 1	),) + s
            for s in _get_minimal_slice_set(
                [0 for _ in start[divergence_idx + 1 :]]   , end[divergence_idx + 1 :]   , dims[divergence_idx + 1 :]   , start_edges=[True for _ in start_edges[divergence_idx + 1 :]]   , end_edges=end_edges[divergence_idx + 1 :]   , )	)

    # If both start and end are at the edges of the subtree rooted at
    # divergence_idx, we can just select the whole subtree at once
    if start_edges[divergence_idx] and end_edges[divergence_idx]:
        slices.append(path + (slice(start[divergence_idx]   , end[divergence_idx] + 1	),)	)
    # If just start is at the edge, we can grab almost all of the subtree,
    # treating only the ragged bottom edge as an edge case
    elif start_edges[divergence_idx]:
        slices.append(path + (slice(start[divergence_idx]   , end[divergence_idx]	),)	)
        slices.extend(lower()	)
    # Analogous to the previous case, but the top is ragged this time
    elif end_edges[divergence_idx]:
        slices.extend(upper()	)
        slices.append(path + (slice(start[divergence_idx] + 1   , end[divergence_idx] + 1	),)	)
    # If both sides of the range are ragged, we need to handle both sides
    # separately. If there's contiguous meat in between them, we can index it
    # in one big chunk
    else:
        slices.extend(upper()	)
        _lowerCamelCase     :		Dict 						=						end[divergence_idx] - start[divergence_idx]
        if middle_ground > 1:
            slices.append(path + (slice(start[divergence_idx] + 1   , end[divergence_idx]	),)	)
        slices.extend(lower()	)

    return slices







@torch.jit.ignore
def 						A__  (					__A   , __A   , __A   , __A	):







    '''simple docstring'''
    _lowerCamelCase     :		List[str] 						=						t.shape[:no_batch_dims]
    _lowerCamelCase     :		str 						=						list(_flat_idx_to_idx(__A   , __A	)	)
    # _get_minimal_slice_set is inclusive
    _lowerCamelCase     :		List[str] 						=						list(_flat_idx_to_idx(flat_end - 1   , __A	)	)

    # Get an ordered list of slices to perform
    _lowerCamelCase     :		int 						=						_get_minimal_slice_set(
        __A   , __A   , __A   , )

    _lowerCamelCase     :		List[Any] 						=						[t[s] for s in slices]

    return torch.cat([s.view((-1,) + t.shape[no_batch_dims:]	) for s in sliced_tensors]	)







def 						A__  (					__A   , __A   , __A   , __A   , __A = False   , __A = None   , __A = False   , ):







    '''simple docstring'''
    if not (len(__A	) > 0):
        raise ValueError("""Must provide at least one input"""	)

    _lowerCamelCase     :		Dict 						=						[shape[:no_batch_dims] for shape in _fetch_dims(__A	)]
    _lowerCamelCase     :		Union[str, Any] 						=						tuple([max(__A	) for s in zip(*__A	)]	)

    def _prep_inputs(__A	) -> torch.Tensor:
        if not low_mem:
            if not sum(t.shape[:no_batch_dims]	) == no_batch_dims:
                _lowerCamelCase     :		Tuple 						=						t.expand(orig_batch_dims + t.shape[no_batch_dims:]	)
            _lowerCamelCase     :		int 						=						t.reshape(-1   , *t.shape[no_batch_dims:]	)
        else:
            _lowerCamelCase     :		Union[str, Any] 						=						t.expand(orig_batch_dims + t.shape[no_batch_dims:]	)
        return t

    _lowerCamelCase     :		Dict[str, Any] 						=						tensor_tree_map(_prep_inputs   , __A	)
    _lowerCamelCase     :		str 						=						None
    if _out is not None:
        _lowerCamelCase     :		int 						=						tensor_tree_map(lambda __A	: t.view([-1] + list(t.shape[no_batch_dims:]	)	)   , _out	)

    _lowerCamelCase     :		List[str] 						=						1
    for d in orig_batch_dims:
        flat_batch_dim *= d

    _lowerCamelCase     :		Union[str, Any] 						=						flat_batch_dim // chunk_size + (flat_batch_dim % chunk_size != 0)

    def _select_chunk(__A	) -> torch.Tensor:
        return t[i : i + chunk_size] if t.shape[0] != 1 else t

    _lowerCamelCase     :		Any 						=						0
    _lowerCamelCase     :		List[Any] 						=						prepped_outputs
    for _ in range(__A	):
        # Chunk the input
        if not low_mem:
            _lowerCamelCase     :		Dict 						=						_select_chunk
        else:
            _lowerCamelCase     :		Optional[int] 						=						partial(
                _chunk_slice   , flat_start=__A   , flat_end=min(__A   , i + chunk_size	)   , no_batch_dims=len(__A	)   , )

        _lowerCamelCase     :		Dict[str, Any] 						=						tensor_tree_map(__A   , __A	)

        # Run the layer on the chunk
        _lowerCamelCase     :		Dict 						=						layer(**__A	)

        # Allocate space for the output
        if out is None:
            _lowerCamelCase     :		Dict 						=						tensor_tree_map(lambda __A	: t.new_zeros((flat_batch_dim,) + t.shape[1:]	)   , __A	)

        # Put the chunk in its pre-allocated space
        if isinstance(__A   , __A	):

            def assign(__A   , __A	) -> None:
                for k, v in da.items():
                    if isinstance(__A   , __A	):
                        assign(__A   , da[k]	)
                    else:
                        if _add_into_out:
                            v[i : i + chunk_size] += da[k]
                        else:
                            _lowerCamelCase     :		List[str] 						=						da[k]

            assign(__A   , __A	)
        elif isinstance(__A   , __A	):
            for xa, xa in zip(__A   , __A	):
                if _add_into_out:
                    xa[i : i + chunk_size] += xa
                else:
                    _lowerCamelCase     :		Optional[int] 						=						xa
        elif isinstance(__A   , torch.Tensor	):
            if _add_into_out:
                out[i : i + chunk_size] += output_chunk
            else:
                _lowerCamelCase     :		Tuple 						=						output_chunk
        else:
            raise ValueError("""Not supported"""	)

        i += chunk_size

    _lowerCamelCase     :		Optional[int] 						=						tensor_tree_map(lambda __A	: t.view(orig_batch_dims + t.shape[1:]	)   , __A	)

    return out




class 	__snake_case	:





      '''simple docstring'''






      def __init__(					self     :				List[Any]    ,  _UpperCamelCase     :				int = 512    ,  )				->int:


          """simple docstring"""


          _lowerCamelCase     :		List[str] 						=						max_chunk_size
          _lowerCamelCase     :		Optional[int] 						=						None
          _lowerCamelCase     :		Optional[tuple] 						=						None



      def 	_SCREAMING_SNAKE_CASE    (					self     :				Union[str, Any]    ,  _UpperCamelCase     :				Callable    ,  _UpperCamelCase     :				tuple    ,  _UpperCamelCase     :				int)				->int:


          """simple docstring"""


          logging.info("""Tuning chunk size...""")

          if min_chunk_size >= self.max_chunk_size:
              return min_chunk_size

          _lowerCamelCase     :		List[int] 						=						[2**l for l in range(int(math.log(self.max_chunk_size    ,  2)) + 1)]
          _lowerCamelCase     :		Dict 						=						[c for c in candidates if c > min_chunk_size]
          _lowerCamelCase     :		str 						=						[min_chunk_size] + candidates
          candidates[-1] += 4

          def test_chunk_size(_UpperCamelCase     :				int) -> bool:
              try:
                  with torch.no_grad():
                      fn(*_UpperCamelCase    ,  chunk_size=_UpperCamelCase)
                  return True
              except RuntimeError:
                  return False

          _lowerCamelCase     :		int 						=						0
          _lowerCamelCase     :		Union[str, Any] 						=						len(_UpperCamelCase) - 1
          while i > min_viable_chunk_size_index:
              _lowerCamelCase     :		Optional[int] 						=						test_chunk_size(candidates[i])
              if not viable:
                  _lowerCamelCase     :		str 						=						(min_viable_chunk_size_index + i) // 2
              else:
                  _lowerCamelCase     :		Optional[int] 						=						i
                  _lowerCamelCase     :		List[Any] 						=						(i + len(_UpperCamelCase) - 1) // 2

          return candidates[min_viable_chunk_size_index]



      def 	_SCREAMING_SNAKE_CASE    (					self     :				Optional[int]    ,  _UpperCamelCase     :				Iterable    ,  _UpperCamelCase     :				Iterable)				->bool:


          """simple docstring"""


          _lowerCamelCase     :		Dict 						=						True
          for aa, aa in zip(_UpperCamelCase    ,  _UpperCamelCase):
              assert type(_UpperCamelCase) == type(_UpperCamelCase)
              if isinstance(_UpperCamelCase    ,  (list, tuple)):
                  consistent &= self._compare_arg_caches(_UpperCamelCase    ,  _UpperCamelCase)
              elif isinstance(_UpperCamelCase    ,  _UpperCamelCase):
                  _lowerCamelCase     :		Dict 						=						[v for _, v in sorted(aa.items()    ,  key=lambda _UpperCamelCase: x[0])]
                  _lowerCamelCase     :		Any 						=						[v for _, v in sorted(aa.items()    ,  key=lambda _UpperCamelCase: x[0])]
                  consistent &= self._compare_arg_caches(_UpperCamelCase    ,  _UpperCamelCase)
              else:
                  consistent &= aa == aa

          return consistent




      def 	_SCREAMING_SNAKE_CASE    (					self     :				Dict    ,  _UpperCamelCase     :				Callable    ,  _UpperCamelCase     :				tuple    ,  _UpperCamelCase     :				int    ,  )				->int:


          """simple docstring"""


          _lowerCamelCase     :		List[str] 						=						True
          _lowerCamelCase     :		tuple 						=						tree_map(lambda _UpperCamelCase: a.shape if isinstance(_UpperCamelCase    ,  torch.Tensor) else a    ,  _UpperCamelCase    ,  _UpperCamelCase)
          if self.cached_arg_data is not None:
              # If args have changed shape/value, we need to re-tune
              assert len(self.cached_arg_data) == len(_UpperCamelCase)
              _lowerCamelCase     :		Optional[Any] 						=						self._compare_arg_caches(self.cached_arg_data    ,  _UpperCamelCase)
          else:
              # Otherwise, we can reuse the precomputed value
              _lowerCamelCase     :		List[str] 						=						False

          if not consistent:
              _lowerCamelCase     :		Optional[Any] 						=						self._determine_favorable_chunk_size(
                  _UpperCamelCase    ,  _UpperCamelCase    ,  _UpperCamelCase    ,  )
              _lowerCamelCase     :		str 						=						arg_data

          assert self.cached_chunk_size is not None

          return self.cached_chunk_size





import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow


if is_torch_available():
    import torch

    from transformers import XLMRobertaModel




@require_sentencepiece
@require_tokenizers
@require_torch
class 	__snake_case	( unittest.TestCase		):





      '''simple docstring'''






      @slow
      def 	_SCREAMING_SNAKE_CASE    (					self     :				Tuple)				->int:


          """simple docstring"""


          _lowerCamelCase     :		Tuple 						=						XLMRobertaModel.from_pretrained("""xlm-roberta-base""")
          _lowerCamelCase     :		Optional[int] 						=						torch.tensor([[0, 581, 1_0269, 83, 9_9942, 136, 6_0742, 23, 70, 8_0583, 1_8276, 2]])
          # The dog is cute and lives in the garden house

          _lowerCamelCase     :		Optional[Any] 						=						torch.Size((1, 12, 768))  # batch_size, sequence_length, embedding_vector_dim
          _lowerCamelCase     :		str 						=						torch.tensor(
              [[-0.0_1_0_1, 0.1_2_1_8, -0.0_8_0_3, 0.0_8_0_1, 0.1_3_2_7, 0.0_7_7_6, -0.1_2_1_5, 0.2_3_8_3, 0.3_3_3_8, 0.3_1_0_6, 0.0_3_0_0, 0.0_2_5_2]])
          #  xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')
          #  xlmr.eval()
          #  expected_output_values_last_dim = xlmr.extract_features(input_ids[0])[:, :, -1]
          with torch.no_grad():
              _lowerCamelCase     :		List[str] 						=						model(_UpperCamelCase)["""last_hidden_state"""].detach()
          self.assertEqual(output.shape    ,  _UpperCamelCase)
          # compare the actual values for a slice of last dim
          self.assertTrue(torch.allclose(output[:, :, -1]    ,  _UpperCamelCase    ,  atol=1E-3))




      @slow
      def 	_SCREAMING_SNAKE_CASE    (					self     :				List[Any])				->Optional[Any]:


          """simple docstring"""


          _lowerCamelCase     :		List[Any] 						=						XLMRobertaModel.from_pretrained("""xlm-roberta-large""")
          _lowerCamelCase     :		Optional[Any] 						=						torch.tensor([[0, 581, 1_0269, 83, 9_9942, 136, 6_0742, 23, 70, 8_0583, 1_8276, 2]])
          # The dog is cute and lives in the garden house

          _lowerCamelCase     :		str 						=						torch.Size((1, 12, 1024))  # batch_size, sequence_length, embedding_vector_dim
          _lowerCamelCase     :		Union[str, Any] 						=						torch.tensor(
              [[-0.0_6_9_9, -0.0_3_1_8, 0.0_7_0_5, -0.1_2_4_1, 0.0_9_9_9, -0.0_5_2_0, 0.1_0_0_4, -0.1_8_3_8, -0.4_7_0_4, 0.1_4_3_7, 0.0_8_2_1, 0.0_1_2_6]])
          #  xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.large')
          #  xlmr.eval()
          #  expected_output_values_last_dim = xlmr.extract_features(input_ids[0])[:, :, -1]
          with torch.no_grad():
              _lowerCamelCase     :		int 						=						model(_UpperCamelCase)["""last_hidden_state"""].detach()
          self.assertEqual(output.shape    ,  _UpperCamelCase)
          # compare the actual values for a slice of last dim
          self.assertTrue(torch.allclose(output[:, :, -1]    ,  _UpperCamelCase    ,  atol=1E-3))




