


"""simple docstring"""

import torch

from accelerate import PartialState
from accelerate.utils.operations import broadcast, gather, gather_object, pad_across_processes, reduce




def __lowercase    (							_a					):
    return (torch.arange(state.num_processes					) + 1.0 + (state.num_processes * state.process_index)).to(state.device					)




def __lowercase    (							_a					):
    snake_case_				:  str       =			create_tensor(_a					)
    snake_case_				:  Tuple       =			gather(_a					)
    assert gathered_tensor.tolist() == list(range(1	,    state.num_processes**2 + 1					)					)




def __lowercase    (							_a					):
    snake_case_				:  List[Any]       =			[state.process_index]
    snake_case_				:  List[Any]       =			gather_object(_a					)
    assert len(_a					) == state.num_processes, f"{gathered_obj}, {len(_a					)} != {state.num_processes}"
    assert gathered_obj == list(range(state.num_processes					)					), f"{gathered_obj} != {list(range(state.num_processes					)					)}"




def __lowercase    (							_a					):
    snake_case_				:  Optional[int]       =			create_tensor(_a					)
    snake_case_				:  Tuple       =			broadcast(_a					)
    assert broadcasted_tensor.shape == torch.Size([state.num_processes]					)
    assert broadcasted_tensor.tolist() == list(range(1	,    state.num_processes + 1					)					)




def __lowercase    (							_a					):
    # We need to pad the tensor with one more element if we are the main process
    # to ensure that we can pad
    if state.is_main_process:
        snake_case_				:  Optional[int]       =			torch.arange(state.num_processes + 1					).to(state.device					)
    else:
        snake_case_				:  Union[str, Any]       =			torch.arange(state.num_processes					).to(state.device					)
    snake_case_				:  Tuple       =			pad_across_processes(_a					)
    assert padded_tensor.shape == torch.Size([state.num_processes + 1]					)
    if not state.is_main_process:
        assert padded_tensor.tolist() == list(range(0	,    state.num_processes					)					) + [0]




def __lowercase    (							_a					):
    # For now runs on only two processes
    if state.num_processes != 2:
        return
    snake_case_				:  int       =			create_tensor(_a					)
    snake_case_				:  Dict       =			reduce(_a	,    '''sum'''					)
    snake_case_				:  int       =			torch.tensor([4.0, 6]					).to(state.device					)
    assert torch.allclose(_a	,    _a					), f"{reduced_tensor} != {truth_tensor}"




def __lowercase    (							_a					):
    # For now runs on only two processes
    if state.num_processes != 2:
        return
    snake_case_				:  Any       =			create_tensor(_a					)
    snake_case_				:  Union[str, Any]       =			reduce(_a	,    '''mean'''					)
    snake_case_				:  List[Any]       =			torch.tensor([2.0, 3]					).to(state.device					)
    assert torch.allclose(_a	,    _a					), f"{reduced_tensor} != {truth_tensor}"




def __lowercase    (							_a					):
    # For xla_spawn (TPUs)
    main()




def __lowercase    (							):
    snake_case_				:  List[str]       =			PartialState()
    state.print(f"State: {state}"					)
    state.print('''testing gather'''					)
    test_gather(_a					)
    state.print('''testing gather_object'''					)
    test_gather_object(_a					)
    state.print('''testing broadcast'''					)
    test_broadcast(_a					)
    state.print('''testing pad_across_processes'''					)
    test_pad_across_processes(_a					)
    state.print('''testing reduce_sum'''					)
    test_reduce_sum(_a					)
    state.print('''testing reduce_mean'''					)
    test_reduce_mean(_a					)


if __name__ == "__main__":
     main()










"""simple docstring"""

from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowercase__					:						List[Any]												=			{'''configuration_xglm''': ['''XGLM_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''XGLMConfig''']}

try:
     if not is_sentencepiece_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowercase__					:						List[Any]												=			['''XGLMTokenizer''']

try:
     if not is_tokenizers_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowercase__					:						List[str]												=			['''XGLMTokenizerFast''']

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowercase__					:						Tuple												=			[
         '''XGLM_PRETRAINED_MODEL_ARCHIVE_LIST''',
         '''XGLMForCausalLM''',
         '''XGLMModel''',
         '''XGLMPreTrainedModel''',
     ]


try:
     if not is_flax_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowercase__					:						int												=			[
         '''FlaxXGLMForCausalLM''',
         '''FlaxXGLMModel''',
         '''FlaxXGLMPreTrainedModel''',
     ]


try:
     if not is_tf_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowercase__					:						Tuple												=			[
         '''TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST''',
         '''TFXGLMForCausalLM''',
         '''TFXGLMModel''',
         '''TFXGLMPreTrainedModel''',
     ]


if TYPE_CHECKING:
     from .configuration_xglm import XGLM_PRETRAINED_CONFIG_ARCHIVE_MAP, XGLMConfig

     try:
          if not is_sentencepiece_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_xglm import XGLMTokenizer

     try:
          if not is_tokenizers_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_xglm_fast import XGLMTokenizerFast

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_xglm import XGLM_PRETRAINED_MODEL_ARCHIVE_LIST, XGLMForCausalLM, XGLMModel, XGLMPreTrainedModel

     try:
          if not is_flax_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_flax_xglm import FlaxXGLMForCausalLM, FlaxXGLMModel, FlaxXGLMPreTrainedModel

     try:
          if not is_tf_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_tf_xglm import (
              TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST,
              TFXGLMForCausalLM,
              TFXGLMModel,
              TFXGLMPreTrainedModel,
          )


else:
     import sys

     lowercase__					:						List[Any]												=			_LazyModule(__name__, globals()['''__file__'''], _import_structure)





