# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_gpt2/convert_megatron_gpt2_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig



def lowerCamelCase						(     SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE=0			):
    '''simple docstring'''







    if name is None:
        __UpperCamelCase  :Optional[Any]    						=	None
    else:
        __UpperCamelCase  :int    						=	'''.''' * max(0						,    spaces - 2			) + '''# {:''' + str(50 - spaces			) + '''s}'''
        __UpperCamelCase  :Optional[int]    						=	fmt.format(SCREAMING_SNAKE_CASE			)

    # Print and recurse (if needed).
    if isinstance(SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE			):
        if msg is not None:
            print(SCREAMING_SNAKE_CASE			)
        for k in val.keys():
            recursive_print(SCREAMING_SNAKE_CASE						,    val[k]						,    spaces + 2			)
    elif isinstance(SCREAMING_SNAKE_CASE						,    torch.Tensor			):
        print(SCREAMING_SNAKE_CASE						,    ''':'''						,    val.size()			)
    else:
        print(SCREAMING_SNAKE_CASE						,    ''':'''						,    SCREAMING_SNAKE_CASE			)



def lowerCamelCase						(     SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE			):
    '''simple docstring'''







    __UpperCamelCase  :List[str]    						=	param.size()
    if checkpoint_version == 1.0:
        # version 1.0 stores [num_heads * hidden_size * num_splits, :]
        __UpperCamelCase  :Tuple    						=	(num_heads, hidden_size, num_splits) + input_shape[1:]
        __UpperCamelCase  :int    						=	param.view(*SCREAMING_SNAKE_CASE			)
        __UpperCamelCase  :Tuple    						=	param.transpose(0						,    2			)
        __UpperCamelCase  :List[Any]    						=	param.transpose(1						,    2			).contiguous()
    elif checkpoint_version >= 2.0:
        # other versions store [num_heads * num_splits * hidden_size, :]
        __UpperCamelCase  :Optional[int]    						=	(num_heads, num_splits, hidden_size) + input_shape[1:]
        __UpperCamelCase  :List[Any]    						=	param.view(*SCREAMING_SNAKE_CASE			)
        __UpperCamelCase  :Union[str, Any]    						=	param.transpose(0						,    1			).contiguous()
    __UpperCamelCase  :int    						=	param.view(*SCREAMING_SNAKE_CASE			)
    return param



def lowerCamelCase						(     SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE			):
    '''simple docstring'''







    __UpperCamelCase  :Any    						=	{}

    # old versions did not store training args
    __UpperCamelCase  :List[str]    						=	input_state_dict.get('''args'''						,    SCREAMING_SNAKE_CASE			)
    if ds_args is not None:
        # do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
        # from pprint import pprint
        # pprint(vars(ds_args))

        __UpperCamelCase  :Union[str, Any]    						=	ds_args.padded_vocab_size
        __UpperCamelCase  :Union[str, Any]    						=	ds_args.max_position_embeddings
        __UpperCamelCase  :Union[str, Any]    						=	ds_args.hidden_size
        __UpperCamelCase  :Dict    						=	ds_args.num_layers
        __UpperCamelCase  :Optional[int]    						=	ds_args.num_attention_heads
        __UpperCamelCase  :List[str]    						=	ds_args.ffn_hidden_size
        # pprint(config)

    # The number of heads.
    __UpperCamelCase  :List[str]    						=	config.n_head
    # The hidden_size per head.
    __UpperCamelCase  :Union[str, Any]    						=	config.n_embd // config.n_head
    # Megatron-LM checkpoint version
    if "checkpoint_version" in input_state_dict.keys():
        __UpperCamelCase  :Optional[Any]    						=	input_state_dict['''checkpoint_version''']
    else:
        __UpperCamelCase  :Tuple    						=	0.0

    # The model.
    __UpperCamelCase  :Tuple    						=	input_state_dict['''model''']
    # The language model.
    __UpperCamelCase  :List[str]    						=	model['''language_model''']
    # The embeddings.
    __UpperCamelCase  :List[Any]    						=	lm['''embedding''']

    # The word embeddings.
    __UpperCamelCase  :Dict    						=	embeddings['''word_embeddings''']['''weight''']
    # Truncate the embedding table to vocab_size rows.
    __UpperCamelCase  :Optional[int]    						=	word_embeddings[: config.vocab_size, :]
    __UpperCamelCase  :Optional[Any]    						=	word_embeddings

    # The position embeddings.
    __UpperCamelCase  :Dict    						=	embeddings['''position_embeddings''']['''weight''']
    # Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
    __UpperCamelCase  :Optional[Any]    						=	pos_embeddings.size(0			)
    if n_positions != config.n_positions:
        raise ValueError(
            f"""pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don't match"""			)
    # Store the position embeddings.
    __UpperCamelCase  :Optional[Any]    						=	pos_embeddings

    # The transformer.
    __UpperCamelCase  :Dict    						=	lm['''transformer'''] if '''transformer''' in lm.keys() else lm['''encoder''']

    # The regex to extract layer names.
    __UpperCamelCase  :Union[str, Any]    						=	re.compile(R'''layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)'''			)

    # The simple map of names for "automated" rules.
    __UpperCamelCase  :List[str]    						=	{
        '''attention.dense''': '''.attn.c_proj.''',
        '''self_attention.dense''': '''.attn.c_proj.''',
        '''mlp.dense_h_to_4h''': '''.mlp.c_fc.''',
        '''mlp.dense_4h_to_h''': '''.mlp.c_proj.''',
    }

    # Extract the layers.
    for key, val in transformer.items():
        # Match the name.
        __UpperCamelCase  :str    						=	layer_re.match(SCREAMING_SNAKE_CASE			)

        # Stop if that's not a layer
        if m is None:
            break

        # The index of the layer.
        __UpperCamelCase  :Any    						=	int(m.group(1			)			)
        # The name of the operation.
        __UpperCamelCase  :Union[str, Any]    						=	m.group(2			)
        # Is it a weight or a bias?
        __UpperCamelCase  :str    						=	m.group(3			)

        # The name of the layer.
        __UpperCamelCase  :Optional[int]    						=	f"""transformer.h.{layer_idx}"""

        # For layernorm(s), simply store the layer norm.
        if op_name.endswith('''layernorm'''			):
            __UpperCamelCase  :List[str]    						=	'''ln_1''' if op_name.startswith('''input'''			) else '''ln_2'''
            __UpperCamelCase  :str    						=	val

        # Transpose the QKV matrix.
        elif (
            op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
        ) and weight_or_bias == "weight":
            # Insert a tensor of 1x1xDxD bias.
            __UpperCamelCase  :Tuple    						=	torch.tril(torch.ones((n_positions, n_positions)						,    dtype=torch.floataa			)			).view(
                1						,    1						,    SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE			)
            __UpperCamelCase  :Optional[int]    						=	causal_mask

            # Insert a "dummy" tensor for masked_bias.
            __UpperCamelCase  :Union[str, Any]    						=	torch.tensor(-1e4						,    dtype=torch.floataa			)
            __UpperCamelCase  :int    						=	masked_bias

            __UpperCamelCase  :Any    						=	fix_query_key_value_ordering(SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE						,    3						,    SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE			)
            # Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
            __UpperCamelCase  :Tuple    						=	out_val.transpose(0						,    1			).contiguous()
            # Store.
            __UpperCamelCase  :str    						=	out_val

        # Transpose the bias.
        elif (
            op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
        ) and weight_or_bias == "bias":
            __UpperCamelCase  :str    						=	fix_query_key_value_ordering(SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE						,    3						,    SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE			)
            # Store. No change of shape.
            __UpperCamelCase  :Union[str, Any]    						=	out_val

        # Transpose the weights.
        elif weight_or_bias == "weight":
            __UpperCamelCase  :List[Any]    						=	megatron_to_transformers[op_name]
            __UpperCamelCase  :Optional[int]    						=	val.transpose(0						,    1			)

        # Copy the bias.
        elif weight_or_bias == "bias":
            __UpperCamelCase  :Union[str, Any]    						=	megatron_to_transformers[op_name]
            __UpperCamelCase  :str    						=	val

    # DEBUG.
    assert config.n_layer == layer_idx + 1

    # The final layernorm.
    __UpperCamelCase  :List[Any]    						=	transformer['''final_layernorm.weight''']
    __UpperCamelCase  :str    						=	transformer['''final_layernorm.bias''']

    # For LM head, transformers' wants the matrix to weight embeddings.
    __UpperCamelCase  :Dict    						=	word_embeddings

    # It should be done!
    return output_state_dict



def lowerCamelCase						(     ):
    '''simple docstring'''







    __UpperCamelCase  :Any    						=	argparse.ArgumentParser()
    parser.add_argument('''--print-checkpoint-structure'''						,    action='''store_true'''			)
    parser.add_argument(
        '''path_to_checkpoint'''						,    type=SCREAMING_SNAKE_CASE						,    help='''Path to the checkpoint file (.zip archive or direct .pt file)'''						,    )
    parser.add_argument(
        '''--config_file'''						,    default=''''''						,    type=SCREAMING_SNAKE_CASE						,    help='''An optional config json file describing the pre-trained model.'''						,    )
    __UpperCamelCase  :Union[str, Any]    						=	parser.parse_args()

    # Extract the basename.
    __UpperCamelCase  :List[Any]    						=	os.path.dirname(args.path_to_checkpoint			)

    # Load the model.
    # the .zip is very optional, let's keep it for backward compatibility
    print(f"""Extracting PyTorch state dictionary from {args.path_to_checkpoint}"""			)
    if args.path_to_checkpoint.endswith('''.zip'''			):
        with zipfile.ZipFile(args.path_to_checkpoint						,    '''r'''			) as checkpoint:
            with checkpoint.open('''release/mp_rank_00/model_optim_rng.pt'''			) as pytorch_dict:
                __UpperCamelCase  :Union[str, Any]    						=	torch.load(SCREAMING_SNAKE_CASE						,    map_location='''cpu'''			)
    else:
        __UpperCamelCase  :Union[str, Any]    						=	torch.load(args.path_to_checkpoint						,    map_location='''cpu'''			)

    __UpperCamelCase  :Dict    						=	input_state_dict.get('''args'''						,    SCREAMING_SNAKE_CASE			)

    # Read the config, or default to the model released by NVIDIA.
    if args.config_file == "":
        if ds_args is not None:
            if ds_args.bias_gelu_fusion:
                __UpperCamelCase  :Tuple    						=	'''gelu_fast'''
            elif ds_args.openai_gelu:
                __UpperCamelCase  :List[str]    						=	'''gelu_new'''
            else:
                __UpperCamelCase  :List[Any]    						=	'''gelu'''
        else:
            # in the very early days this used to be "gelu_new"
            __UpperCamelCase  :str    						=	'''gelu_new'''

        # Spell out all parameters in case the defaults change.
        __UpperCamelCase  :Dict    						=	GPTaConfig(
            vocab_size=50_257						,    n_positions=1_024						,    n_embd=1_024						,    n_layer=24						,    n_head=16						,    n_inner=4_096						,    activation_function=SCREAMING_SNAKE_CASE						,    resid_pdrop=0.1						,    embd_pdrop=0.1						,    attn_pdrop=0.1						,    layer_norm_epsilon=1e-5						,    initializer_range=0.02						,    summary_type='''cls_index'''						,    summary_use_proj=SCREAMING_SNAKE_CASE						,    summary_activation=SCREAMING_SNAKE_CASE						,    summary_proj_to_labels=SCREAMING_SNAKE_CASE						,    summary_first_dropout=0.1						,    scale_attn_weights=SCREAMING_SNAKE_CASE						,    use_cache=SCREAMING_SNAKE_CASE						,    bos_token_id=50_256						,    eos_token_id=50_256						,    )
    else:
        __UpperCamelCase  :Optional[int]    						=	GPTaConfig.from_json_file(args.config_file			)

    __UpperCamelCase  :Union[str, Any]    						=	['''GPT2LMHeadModel''']

    # Convert.
    print('''Converting'''			)
    __UpperCamelCase  :Optional[int]    						=	convert_megatron_checkpoint(SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE			)

    # Print the structure of converted state dict.
    if args.print_checkpoint_structure:
        recursive_print(SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE			)

    # Add tokenizer class info to config
    # see https://github.com/huggingface/transformers/issues/13906)
    if ds_args is not None:
        __UpperCamelCase  :Any    						=	ds_args.tokenizer_type
        if tokenizer_type == "GPT2BPETokenizer":
            __UpperCamelCase  :Dict    						=	'''gpt2'''
        elif tokenizer_type == "PretrainedFromHF":
            __UpperCamelCase  :int    						=	ds_args.tokenizer_name_or_path
        else:
            raise ValueError(f"""Unrecognized tokenizer_type {tokenizer_type}"""			)
    else:
        __UpperCamelCase  :Dict    						=	'''gpt2'''

    __UpperCamelCase  :Optional[int]    						=	AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE			)
    __UpperCamelCase  :str    						=	type(SCREAMING_SNAKE_CASE			).__name__
    __UpperCamelCase  :Union[str, Any]    						=	tokenizer_class

    # Store the config to file.
    print('''Saving config'''			)
    config.save_pretrained(SCREAMING_SNAKE_CASE			)

    # Save tokenizer based on args
    print(f"""Adding {tokenizer_class} tokenizer files"""			)
    tokenizer.save_pretrained(SCREAMING_SNAKE_CASE			)

    # Store the state_dict to file.
    __UpperCamelCase  :Any    						=	os.path.join(SCREAMING_SNAKE_CASE						,    '''pytorch_model.bin'''			)
    print(f"""Saving checkpoint to \"{output_checkpoint_file}\""""			)
    torch.save(SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE			)


####################################################################################################

if __name__ == "__main__":
    main()

####################################################################################################






import time
from dataclasses import dataclass
from multiprocessing import Pool
from unittest import TestCase
from unittest.mock import patch

import multiprocess
import numpy as np
import pytest

from datasets.utils.py_utils import (
    NestedDataStructure,
    asdict,
    iflatmap_unordered,
    map_nested,
    temp_seed,
    temporary_assignment,
    zip_dict,
)

from .utils import require_tf, require_torch



def lowerCamelCase						(     SCREAMING_SNAKE_CASE			):  # picklable for multiprocessing
    '''simple docstring'''







    return x.sum()



def lowerCamelCase						(     SCREAMING_SNAKE_CASE			):  # picklable for multiprocessing
    '''simple docstring'''







    return i + 1


@dataclass
class 			lowerCamelCase_  :






    '''simple docstring'''






    a__			:		int
    a__			:		str


class 			lowerCamelCase_  (     UpperCAmelCase_				):






    '''simple docstring'''






    def     UpperCamelCase__     (			self) ->			int:
        __UpperCamelCase  :Dict    						=	{}
        __UpperCamelCase  :int    						=	[]
        __UpperCamelCase  :int    						=	1
        __UpperCamelCase  :int    						=	[1, 2]
        __UpperCamelCase  :Optional[Any]    						=	{'''a''': 1, '''b''': 2}
        __UpperCamelCase  :Dict    						=	{'''a''': [1, 2], '''b''': [3, 4]}
        __UpperCamelCase  :Optional[int]    						=	{'''a''': {'''1''': 1}, '''b''': 2}
        __UpperCamelCase  :Optional[Any]    						=	{'''a''': 1, '''b''': 2, '''c''': 3, '''d''': 4}
        __UpperCamelCase  :str    						=	{}
        __UpperCamelCase  :Any    						=	[]
        __UpperCamelCase  :Optional[Any]    						=	2
        __UpperCamelCase  :Optional[int]    						=	[2, 3]
        __UpperCamelCase  :List[Any]    						=	{'''a''': 2, '''b''': 3}
        __UpperCamelCase  :List[str]    						=	{'''a''': [2, 3], '''b''': [4, 5]}
        __UpperCamelCase  :int    						=	{'''a''': {'''1''': 2}, '''b''': 3}
        __UpperCamelCase  :Union[str, Any]    						=	{'''a''': 2, '''b''': 3, '''c''': 4, '''d''': 5}
        self.assertEqual(map_nested(__lowercase				,				__lowercase)				,				__lowercase)
        self.assertEqual(map_nested(__lowercase				,				__lowercase)				,				__lowercase)
        self.assertEqual(map_nested(__lowercase				,				__lowercase)				,				__lowercase)
        self.assertEqual(map_nested(__lowercase				,				__lowercase)				,				__lowercase)
        self.assertEqual(map_nested(__lowercase				,				__lowercase)				,				__lowercase)
        self.assertEqual(map_nested(__lowercase				,				__lowercase)				,				__lowercase)
        self.assertEqual(map_nested(__lowercase				,				__lowercase)				,				__lowercase)
        self.assertEqual(map_nested(__lowercase				,				__lowercase)				,				__lowercase)

        __UpperCamelCase  :Optional[Any]    						=	2
        self.assertEqual(map_nested(__lowercase				,				__lowercase				,				num_proc=__lowercase)				,				__lowercase)
        self.assertEqual(map_nested(__lowercase				,				__lowercase				,				num_proc=__lowercase)				,				__lowercase)
        self.assertEqual(map_nested(__lowercase				,				__lowercase				,				num_proc=__lowercase)				,				__lowercase)
        self.assertEqual(map_nested(__lowercase				,				__lowercase				,				num_proc=__lowercase)				,				__lowercase)
        self.assertEqual(map_nested(__lowercase				,				__lowercase				,				num_proc=__lowercase)				,				__lowercase)
        self.assertEqual(map_nested(__lowercase				,				__lowercase				,				num_proc=__lowercase)				,				__lowercase)
        self.assertEqual(map_nested(__lowercase				,				__lowercase				,				num_proc=__lowercase)				,				__lowercase)
        self.assertEqual(map_nested(__lowercase				,				__lowercase				,				num_proc=__lowercase)				,				__lowercase)

        __UpperCamelCase  :Dict    						=	{'''a''': np.eye(2), '''b''': np.zeros(3), '''c''': np.ones(2)}
        __UpperCamelCase  :Optional[Any]    						=	{'''a''': 2, '''b''': 0, '''c''': 2}
        __UpperCamelCase  :Optional[int]    						=	{
            '''a''': np.eye(2).astype(__lowercase),
            '''b''': np.zeros(3).astype(__lowercase),
            '''c''': np.ones(2).astype(__lowercase),
        }
        self.assertEqual(map_nested(__lowercase				,				__lowercase				,				map_numpy=__lowercase)				,				__lowercase)
        self.assertEqual(
            {k: v.tolist() for k, v in map_nested(__lowercase				,				__lowercase				,				map_numpy=__lowercase).items()}				,				{k: v.tolist() for k, v in expected_map_nested_sna_int.items()}				,				)
        self.assertEqual(map_nested(__lowercase				,				__lowercase				,				map_numpy=__lowercase				,				num_proc=__lowercase)				,				__lowercase)
        self.assertEqual(
            {k: v.tolist() for k, v in map_nested(__lowercase				,				__lowercase				,				map_numpy=__lowercase				,				num_proc=__lowercase).items()}				,				{k: v.tolist() for k, v in expected_map_nested_sna_int.items()}				,				)
        with self.assertRaises(__lowercase):  # can't pickle a local lambda
            map_nested(lambda __lowercase: x + 1				,				__lowercase				,				num_proc=__lowercase)







    def     UpperCamelCase__     (			self) ->			List[Any]:
        __UpperCamelCase  :List[str]    						=	{'''a''': 1, '''b''': 2}
        __UpperCamelCase  :int    						=	{'''a''': 3, '''b''': 4}
        __UpperCamelCase  :List[str]    						=	{'''a''': 5, '''b''': 6}
        __UpperCamelCase  :List[Any]    						=	sorted([('''a''', (1, 3, 5)), ('''b''', (2, 4, 6))])
        self.assertEqual(sorted(zip_dict(__lowercase				,				__lowercase				,				__lowercase))				,				__lowercase)







    def     UpperCamelCase__     (			self) ->			List[Any]:


        class 			lowerCamelCase_  :






            '''simple docstring'''






            a__			:		int      			=   """bar"""
        __UpperCamelCase  :List[str]    						=	Foo()
        self.assertEqual(foo.my_attr				,				'''bar''')
        with temporary_assignment(__lowercase				,				'''my_attr'''				,				'''BAR'''):
            self.assertEqual(foo.my_attr				,				'''BAR''')
        self.assertEqual(foo.my_attr				,				'''bar''')



@pytest.mark.parametrize(
    '''iterable_length, num_proc, expected_num_proc'''						,    [
        (1, None, 1),
        (1, 1, 1),
        (2, None, 1),
        (2, 1, 1),
        (2, 2, 1),
        (2, 3, 1),
        (3, 2, 1),
        (16, 16, 16),
        (16, 17, 16),
        (17, 16, 16),
    ]						,    )
def lowerCamelCase						(     SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE			):
    '''simple docstring'''







    with patch('''datasets.utils.py_utils._single_map_nested'''			) as mock_single_map_nested, patch(
        '''datasets.parallel.parallel.Pool'''			) as mock_multiprocessing_pool:
        __UpperCamelCase  :Union[str, Any]    						=	{f"""{i}""": i for i in range(SCREAMING_SNAKE_CASE			)}
        __UpperCamelCase  :Dict    						=	map_nested(lambda SCREAMING_SNAKE_CASE			: x + 10						,    SCREAMING_SNAKE_CASE						,    num_proc=SCREAMING_SNAKE_CASE						,    parallel_min_length=16			)
        if expected_num_proc == 1:
            assert mock_single_map_nested.called
            assert not mock_multiprocessing_pool.called
        else:
            assert not mock_single_map_nested.called
            assert mock_multiprocessing_pool.called
            assert mock_multiprocessing_pool.call_args[0][0] == expected_num_proc


class 			lowerCamelCase_  (     UpperCAmelCase_				):






    '''simple docstring'''






    @require_tf
    def     UpperCamelCase__     (			self) ->			Optional[int]:
        import tensorflow as tf
        from tensorflow.keras import layers

        __UpperCamelCase  :Optional[Any]    						=	layers.Dense(2)

        def gen_random_output():
            __UpperCamelCase  :Optional[Any]    						=	tf.random.uniform((1, 3))
            return model(__lowercase).numpy()

        with temp_seed(42				,				set_tensorflow=__lowercase):
            __UpperCamelCase  :Tuple    						=	gen_random_output()
        with temp_seed(42				,				set_tensorflow=__lowercase):
            __UpperCamelCase  :int    						=	gen_random_output()
        __UpperCamelCase  :List[str]    						=	gen_random_output()

        np.testing.assert_equal(__lowercase				,				__lowercase)
        self.assertGreater(np.abs(outa - outa).sum()				,				0)







    @require_torch
    def     UpperCamelCase__     (			self) ->			List[str]:
        import torch

        def gen_random_output():
            __UpperCamelCase  :List[Any]    						=	torch.nn.Linear(3				,				2)
            __UpperCamelCase  :Optional[Any]    						=	torch.rand(1				,				3)
            return model(__lowercase).detach().numpy()

        with temp_seed(42				,				set_pytorch=__lowercase):
            __UpperCamelCase  :Union[str, Any]    						=	gen_random_output()
        with temp_seed(42				,				set_pytorch=__lowercase):
            __UpperCamelCase  :List[str]    						=	gen_random_output()
        __UpperCamelCase  :Dict    						=	gen_random_output()

        np.testing.assert_equal(__lowercase				,				__lowercase)
        self.assertGreater(np.abs(outa - outa).sum()				,				0)







    def     UpperCamelCase__     (			self) ->			str:
        def gen_random_output():
            return np.random.rand(1				,				3)

        with temp_seed(42):
            __UpperCamelCase  :List[Any]    						=	gen_random_output()
        with temp_seed(42):
            __UpperCamelCase  :Optional[Any]    						=	gen_random_output()
        __UpperCamelCase  :Optional[Any]    						=	gen_random_output()

        np.testing.assert_equal(__lowercase				,				__lowercase)
        self.assertGreater(np.abs(outa - outa).sum()				,				0)



@pytest.mark.parametrize('''input_data'''						,    [{}]			)
def lowerCamelCase						(     SCREAMING_SNAKE_CASE			):
    '''simple docstring'''







    __UpperCamelCase  :List[str]    						=	NestedDataStructure(SCREAMING_SNAKE_CASE			).data
    assert output_data == input_data



@pytest.mark.parametrize(
    '''data, expected_output'''						,    [
        ({}, []),
        ([], []),
        ('''foo''', ['''foo''']),
        (['''foo''', '''bar'''], ['''foo''', '''bar''']),
        ([['''foo''', '''bar''']], ['''foo''', '''bar''']),
        ([[['''foo'''], ['''bar''']]], ['''foo''', '''bar''']),
        ([[['''foo'''], '''bar''']], ['''foo''', '''bar''']),
        ({'''a''': 1, '''b''': 2}, [1, 2]),
        ({'''a''': [1, 2], '''b''': [3, 4]}, [1, 2, 3, 4]),
        ({'''a''': [[1, 2]], '''b''': [[3, 4]]}, [1, 2, 3, 4]),
        ({'''a''': [[1, 2]], '''b''': [3, 4]}, [1, 2, 3, 4]),
        ({'''a''': [[[1], [2]]], '''b''': [[[3], [4]]]}, [1, 2, 3, 4]),
        ({'''a''': [[[1], [2]]], '''b''': [[3, 4]]}, [1, 2, 3, 4]),
        ({'''a''': [[[1], [2]]], '''b''': [3, 4]}, [1, 2, 3, 4]),
        ({'''a''': [[[1], [2]]], '''b''': [3, [4]]}, [1, 2, 3, 4]),
        ({'''a''': {'''1''': 1}, '''b''': 2}, [1, 2]),
        ({'''a''': {'''1''': [1]}, '''b''': 2}, [1, 2]),
        ({'''a''': {'''1''': [1]}, '''b''': [2]}, [1, 2]),
    ]						,    )
def lowerCamelCase						(     SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE			):
    '''simple docstring'''







    __UpperCamelCase  :Dict    						=	NestedDataStructure(SCREAMING_SNAKE_CASE			).flatten()
    assert output == expected_output



def lowerCamelCase						(     ):
    '''simple docstring'''







    __UpperCamelCase  :Union[str, Any]    						=	A(x=1						,    y='''foobar'''			)
    __UpperCamelCase  :Dict    						=	{'''x''': 1, '''y''': '''foobar'''}
    assert asdict(SCREAMING_SNAKE_CASE			) == expected_output

    __UpperCamelCase  :str    						=	{'''a''': {'''b''': A(x=10						,    y='''foo'''			)}, '''c''': [A(x=20						,    y='''bar'''			)]}
    __UpperCamelCase  :List[Any]    						=	{'''a''': {'''b''': {'''x''': 10, '''y''': '''foo'''}}, '''c''': [{'''x''': 20, '''y''': '''bar'''}]}
    assert asdict(SCREAMING_SNAKE_CASE			) == expected_output

    with pytest.raises(SCREAMING_SNAKE_CASE			):
        asdict([1, A(x=10						,    y='''foo'''			)]			)



def lowerCamelCase						(     SCREAMING_SNAKE_CASE			):
    '''simple docstring'''







    return text.split()



def lowerCamelCase						(     SCREAMING_SNAKE_CASE			):
    '''simple docstring'''







    yield (time.time(), content)
    time.sleep(2			)
    yield (time.time(), content)



def lowerCamelCase						(     ):
    '''simple docstring'''







    with Pool(2			) as pool:
        __UpperCamelCase  :Tuple    						=	list(iflatmap_unordered(SCREAMING_SNAKE_CASE						,    _split_text						,    kwargs_iterable=[{'''text''': '''hello there'''}] * 10			)			)
        assert out.count('''hello'''			) == 10
        assert out.count('''there'''			) == 10
        assert len(SCREAMING_SNAKE_CASE			) == 20

    # check multiprocess from pathos (uses dill for pickling)
    with multiprocess.Pool(2			) as pool:
        __UpperCamelCase  :Any    						=	list(iflatmap_unordered(SCREAMING_SNAKE_CASE						,    _split_text						,    kwargs_iterable=[{'''text''': '''hello there'''}] * 10			)			)
        assert out.count('''hello'''			) == 10
        assert out.count('''there'''			) == 10
        assert len(SCREAMING_SNAKE_CASE			) == 20

    # check that we get items as fast as possible
    with Pool(2			) as pool:
        __UpperCamelCase  :Dict    						=	[]
        for yield_time, content in iflatmap_unordered(
            SCREAMING_SNAKE_CASE						,    _aseconds_generator_of_aitems_with_timing						,    kwargs_iterable=[{'''content''': '''a'''}, {'''content''': '''b'''}]			):
            assert yield_time < time.time() + 0.1, "we should each item directly after it was yielded"
            out.append(SCREAMING_SNAKE_CASE			)
        assert out.count('''a'''			) == 2
        assert out.count('''b'''			) == 2
        assert len(SCREAMING_SNAKE_CASE			) == 4





