import unittest

import numpy as np

from transformers import is_flax_available
from transformers.testing_utils import require_flax

from ..test_modeling_flax_common import ids_tensor


if is_flax_available():
			import jax
			import jax.numpy as jnp

			from transformers.generation import (
			    FlaxForcedBOSTokenLogitsProcessor,
			    FlaxForcedEOSTokenLogitsProcessor,
			    FlaxLogitsProcessorList,
			    FlaxMinLengthLogitsProcessor,
			    FlaxTemperatureLogitsWarper,
			    FlaxTopKLogitsWarper,
			    FlaxTopPLogitsWarper,
			)
@require_flax
class   _lowerCAmelCase		(     unittest.TestCase				):







							def 				snake_case  (				self     :    Optional[int]	,					__snake_case     :    int	,					__snake_case     :    int		):
										lowerCamelCase						:Dict						  =     jnp.ones((batch_size, length)		) / length
										return scores







							def 				snake_case  (				self     :    Optional[Any]		):
										lowerCamelCase						:Union[str, Any]						  =     None
										lowerCamelCase						:Dict						  =     20

										lowerCamelCase						:Any						  =     self._get_uniform_logits(batch_size=2	,					length=__snake_case		)

										# tweak scores to not be uniform anymore
										lowerCamelCase						:Dict						  =     scores.at[1, 5].set((1 / length) + 0.1		)  # peak, 1st batch
										lowerCamelCase						:Tuple						  =     scores.at[1, 10].set((1 / length) - 0.4		)  # valley, 1st batch

										# compute softmax
										lowerCamelCase						:Tuple						  =     jax.nn.softmax(__snake_case	,					axis=-1		)

										lowerCamelCase						:List[str]						  =     FlaxTemperatureLogitsWarper(temperature=0.5		)
										lowerCamelCase						:Optional[int]						  =     FlaxTemperatureLogitsWarper(temperature=1.3		)

										lowerCamelCase						:Optional[int]						  =     jax.nn.softmax(temp_dist_warper_sharper(__snake_case	,					scores.copy()	,					cur_len=__snake_case		)	,					axis=-1		)
										lowerCamelCase						:Any						  =     jax.nn.softmax(temp_dist_warper_smoother(__snake_case	,					scores.copy()	,					cur_len=__snake_case		)	,					axis=-1		)

										# uniform distribution stays uniform
										self.assertTrue(jnp.allclose(probs[0, :]	,					warped_prob_sharp[0, :]	,					atol=1e-3		)		)
										self.assertTrue(jnp.allclose(probs[0, :]	,					warped_prob_smooth[0, :]	,					atol=1e-3		)		)

										# sharp peaks get higher, valleys get lower
										self.assertLess(probs[1, :].max()	,					warped_prob_sharp[1, :].max()		)
										self.assertGreater(probs[1, :].min()	,					warped_prob_sharp[1, :].min()		)

										# smooth peaks get lower, valleys get higher
										self.assertGreater(probs[1, :].max()	,					warped_prob_smooth[1, :].max()		)
										self.assertLess(probs[1, :].min()	,					warped_prob_smooth[1, :].min()		)







							def 				snake_case  (				self     :    Optional[Any]		):
										lowerCamelCase						:int						  =     None
										lowerCamelCase						:Optional[int]						  =     10
										lowerCamelCase						:str						  =     2

										# create ramp distribution
										lowerCamelCase						:str						  =     np.broadcast_to(np.arange(__snake_case		)[None, :]	,					(batch_size, vocab_size)		).copy()
										lowerCamelCase						:List[str]						  =     ramp_logits[1:, : vocab_size // 2] + vocab_size

										lowerCamelCase						:Optional[int]						  =     FlaxTopKLogitsWarper(3		)

										lowerCamelCase						:int						  =     top_k_warp(__snake_case	,					__snake_case	,					cur_len=__snake_case		)

										# check that correct tokens are filtered
										self.assertListEqual(jnp.isinf(scores[0]		).tolist()	,					7 * [True] + 3 * [False]		)
										self.assertListEqual(jnp.isinf(scores[1]		).tolist()	,					2 * [True] + 3 * [False] + 5 * [True]		)

										# check special case
										lowerCamelCase						:Union[str, Any]						  =     5
										lowerCamelCase						:List[str]						  =     FlaxTopKLogitsWarper(top_k=1	,					filter_value=0.0	,					min_tokens_to_keep=3		)

										lowerCamelCase						:Union[str, Any]						  =     np.broadcast_to(np.arange(__snake_case		)[None, :]	,					(batch_size, length)		).copy()
										lowerCamelCase						:List[Any]						  =     top_k_warp_safety_check(__snake_case	,					__snake_case	,					cur_len=__snake_case		)

										# min_tokens overwrites k: 3 tokens are kept => 2 tokens are nullified
										self.assertListEqual((scores == 0.0).sum(axis=-1		).tolist()	,					[2, 2]		)







							def 				snake_case  (				self     :    Any		):
										lowerCamelCase						:List[str]						  =     None
										lowerCamelCase						:Any						  =     10
										lowerCamelCase						:Tuple						  =     2

										# create distribution and take log (inverse to Softmax as taken in TopPLogitsWarper)
										lowerCamelCase						:Optional[int]						  =     np.log(np.array([[0.3, 0.1, 0.1, 0.5], [0.1_5, 0.3, 0.3, 0.2_5]]		)		)

										lowerCamelCase						:int						  =     FlaxTopPLogitsWarper(0.8		)
										lowerCamelCase						:List[Any]						  =     np.exp(top_p_warp(__snake_case	,					__snake_case	,					cur_len=__snake_case		)		)

										# dist should be filtered to keep min num values so that sum is >= top_p
										# exp (-inf) => 0
										lowerCamelCase						:Any						  =     np.array([[0.3, 0.0, 0.0, 0.5], [0.0, 0.3, 0.3, 0.2_5]]		)
										self.assertTrue(np.allclose(__snake_case	,					__snake_case	,					atol=1e-3		)		)

										# check edge cases with negative and extreme logits
										lowerCamelCase						:int						  =     np.broadcast_to(np.arange(__snake_case		)[None, :]	,					(batch_size, vocab_size)		).copy() - (
										    vocab_size // 2
										)

										# make ramp_logits more extreme
										lowerCamelCase						:int						  =     ramp_logits[1] * 1_0_0.0

										# make sure at least 2 tokens are kept
										lowerCamelCase						:Dict						  =     FlaxTopPLogitsWarper(0.9	,					min_tokens_to_keep=2	,					filter_value=0.0		)
										lowerCamelCase						:List[Any]						  =     top_p_warp(__snake_case	,					__snake_case	,					cur_len=__snake_case		)

										# first batch should keep three tokens, second batch would keep only 1, but due to `min_tokens_to_keep=2` keeps 2.
										self.assertListEqual((filtered_dist != 0.0).sum(axis=-1		).tolist()	,					[3, 2]		)







							def 				snake_case  (				self     :    Optional[int]		):
										lowerCamelCase						:List[Any]						  =     20
										lowerCamelCase						:Optional[int]						  =     4
										lowerCamelCase						:List[str]						  =     0

										lowerCamelCase						:int						  =     FlaxMinLengthLogitsProcessor(min_length=10	,					eos_token_id=__snake_case		)

										# check that min length is applied at length 5
										lowerCamelCase						:int						  =     ids_tensor((batch_size, 20)	,					vocab_size=20		)
										lowerCamelCase						:int						  =     5
										lowerCamelCase						:List[Any]						  =     self._get_uniform_logits(__snake_case	,					__snake_case		)
										lowerCamelCase						:List[Any]						  =     min_dist_processor(__snake_case	,					__snake_case	,					cur_len=__snake_case		)
										self.assertListEqual(scores_before_min_length[:, eos_token_id].tolist()	,					4 * [-float('''inf'''		)]		)

										# check that min length is not applied anymore at length 15
										lowerCamelCase						:Optional[Any]						  =     self._get_uniform_logits(__snake_case	,					__snake_case		)
										lowerCamelCase						:Any						  =     15
										lowerCamelCase						:Any						  =     min_dist_processor(__snake_case	,					__snake_case	,					cur_len=__snake_case		)
										self.assertFalse(jnp.isinf(__snake_case		).any()		)







							def 				snake_case  (				self     :    str		):
										lowerCamelCase						:Any						  =     20
										lowerCamelCase						:List[Any]						  =     4
										lowerCamelCase						:List[str]						  =     0

										lowerCamelCase						:Any						  =     FlaxForcedBOSTokenLogitsProcessor(bos_token_id=__snake_case		)

										# check that all scores are -inf except the bos_token_id score
										lowerCamelCase						:Any						  =     ids_tensor((batch_size, 1)	,					vocab_size=20		)
										lowerCamelCase						:List[str]						  =     1
										lowerCamelCase						:Dict						  =     self._get_uniform_logits(__snake_case	,					__snake_case		)
										lowerCamelCase						:int						  =     logits_processor(__snake_case	,					__snake_case	,					cur_len=__snake_case		)
										self.assertTrue(jnp.isneginf(scores[:, bos_token_id + 1 :]		).all()		)
										self.assertListEqual(scores[:, bos_token_id].tolist()	,					4 * [0]		)  # score for bos_token_id shold be zero

										# check that bos_token_id is not forced if current length is greater than 1
										lowerCamelCase						:Optional[Any]						  =     3
										lowerCamelCase						:Union[str, Any]						  =     self._get_uniform_logits(__snake_case	,					__snake_case		)
										lowerCamelCase						:int						  =     logits_processor(__snake_case	,					__snake_case	,					cur_len=__snake_case		)
										self.assertFalse(jnp.isinf(__snake_case		).any()		)







							def 				snake_case  (				self     :    Dict		):
										lowerCamelCase						:Optional[Any]						  =     20
										lowerCamelCase						:List[str]						  =     4
										lowerCamelCase						:Any						  =     0
										lowerCamelCase						:Union[str, Any]						  =     5

										lowerCamelCase						:Dict						  =     FlaxForcedEOSTokenLogitsProcessor(max_length=__snake_case	,					eos_token_id=__snake_case		)

										# check that all scores are -inf except the eos_token_id when max_length is reached
										lowerCamelCase						:int						  =     ids_tensor((batch_size, 4)	,					vocab_size=20		)
										lowerCamelCase						:Tuple						  =     4
										lowerCamelCase						:List[str]						  =     self._get_uniform_logits(__snake_case	,					__snake_case		)
										lowerCamelCase						:List[str]						  =     logits_processor(__snake_case	,					__snake_case	,					cur_len=__snake_case		)
										self.assertTrue(jnp.isneginf(scores[:, eos_token_id + 1 :]		).all()		)
										self.assertListEqual(scores[:, eos_token_id].tolist()	,					4 * [0]		)  # score for eos_token_id should be zero

										# check that eos_token_id is not forced if max_length is not reached
										lowerCamelCase						:Optional[int]						  =     3
										lowerCamelCase						:str						  =     self._get_uniform_logits(__snake_case	,					__snake_case		)
										lowerCamelCase						:str						  =     logits_processor(__snake_case	,					__snake_case	,					cur_len=__snake_case		)
										self.assertFalse(jnp.isinf(__snake_case		).any()		)







							def 				snake_case  (				self     :    str		):
										lowerCamelCase						:List[Any]						  =     4
										lowerCamelCase						:Dict						  =     10
										lowerCamelCase						:int						  =     15
										lowerCamelCase						:Any						  =     2
										lowerCamelCase						:Dict						  =     1
										lowerCamelCase						:Union[str, Any]						  =     15

										# dummy input_ids and scores
										lowerCamelCase						:int						  =     ids_tensor((batch_size, sequence_length)	,					__snake_case		)
										lowerCamelCase						:int						  =     input_ids.copy()

										lowerCamelCase						:Tuple						  =     self._get_uniform_logits(__snake_case	,					__snake_case		)
										lowerCamelCase						:str						  =     scores.copy()

										# instantiate all dist processors
										lowerCamelCase						:List[Any]						  =     FlaxTemperatureLogitsWarper(temperature=0.5		)
										lowerCamelCase						:str						  =     FlaxTopKLogitsWarper(3		)
										lowerCamelCase						:Optional[int]						  =     FlaxTopPLogitsWarper(0.8		)

										# instantiate all logits processors
										lowerCamelCase						:int						  =     FlaxMinLengthLogitsProcessor(min_length=10	,					eos_token_id=__snake_case		)
										lowerCamelCase						:Any						  =     FlaxForcedBOSTokenLogitsProcessor(bos_token_id=__snake_case		)
										lowerCamelCase						:Any						  =     FlaxForcedEOSTokenLogitsProcessor(max_length=__snake_case	,					eos_token_id=__snake_case		)

										lowerCamelCase						:Tuple						  =     10

										# no processor list
										lowerCamelCase						:Dict						  =     temp_dist_warp(__snake_case	,					__snake_case	,					cur_len=__snake_case		)
										lowerCamelCase						:Tuple						  =     top_k_warp(__snake_case	,					__snake_case	,					cur_len=__snake_case		)
										lowerCamelCase						:Optional[Any]						  =     top_p_warp(__snake_case	,					__snake_case	,					cur_len=__snake_case		)
										lowerCamelCase						:Dict						  =     min_dist_proc(__snake_case	,					__snake_case	,					cur_len=__snake_case		)
										lowerCamelCase						:int						  =     bos_dist_proc(__snake_case	,					__snake_case	,					cur_len=__snake_case		)
										lowerCamelCase						:Union[str, Any]						  =     eos_dist_proc(__snake_case	,					__snake_case	,					cur_len=__snake_case		)

										# with processor list
										lowerCamelCase						:Dict						  =     FlaxLogitsProcessorList(
										    [temp_dist_warp, top_k_warp, top_p_warp, min_dist_proc, bos_dist_proc, eos_dist_proc]		)
										lowerCamelCase						:int						  =     processor(__snake_case	,					__snake_case	,					cur_len=__snake_case		)

										# scores should be equal
										self.assertTrue(jnp.allclose(__snake_case	,					__snake_case	,					atol=1e-3		)		)

										# input_ids should never be changed
										self.assertListEqual(input_ids.tolist()	,					input_ids_comp.tolist()		)







							def 				snake_case  (				self     :    Union[str, Any]		):
										lowerCamelCase						:Optional[int]						  =     4
										lowerCamelCase						:Optional[int]						  =     10
										lowerCamelCase						:int						  =     15
										lowerCamelCase						:List[str]						  =     2
										lowerCamelCase						:Optional[int]						  =     1
										lowerCamelCase						:Tuple						  =     15

										# dummy input_ids and scores
										lowerCamelCase						:Tuple						  =     ids_tensor((batch_size, sequence_length)	,					__snake_case		)
										lowerCamelCase						:List[str]						  =     input_ids.copy()

										lowerCamelCase						:List[str]						  =     self._get_uniform_logits(__snake_case	,					__snake_case		)
										lowerCamelCase						:Dict						  =     scores.copy()

										# instantiate all dist processors
										lowerCamelCase						:List[str]						  =     FlaxTemperatureLogitsWarper(temperature=0.5		)
										lowerCamelCase						:Union[str, Any]						  =     FlaxTopKLogitsWarper(3		)
										lowerCamelCase						:Dict						  =     FlaxTopPLogitsWarper(0.8		)

										# instantiate all logits processors
										lowerCamelCase						:Optional[Any]						  =     FlaxMinLengthLogitsProcessor(min_length=10	,					eos_token_id=__snake_case		)
										lowerCamelCase						:Dict						  =     FlaxForcedBOSTokenLogitsProcessor(bos_token_id=__snake_case		)
										lowerCamelCase						:int						  =     FlaxForcedEOSTokenLogitsProcessor(max_length=__snake_case	,					eos_token_id=__snake_case		)

										lowerCamelCase						:Any						  =     10

										# no processor list
										def run_no_processor_list(__snake_case     :    int	,					__snake_case     :    Any	,					__snake_case     :    Optional[Any]		):
													lowerCamelCase						:Dict						  =     temp_dist_warp(__snake_case	,					__snake_case	,					cur_len=__snake_case		)
													lowerCamelCase						:int						  =     top_k_warp(__snake_case	,					__snake_case	,					cur_len=__snake_case		)
													lowerCamelCase						:str						  =     top_p_warp(__snake_case	,					__snake_case	,					cur_len=__snake_case		)
													lowerCamelCase						:List[Any]						  =     min_dist_proc(__snake_case	,					__snake_case	,					cur_len=__snake_case		)
													lowerCamelCase						:Dict						  =     bos_dist_proc(__snake_case	,					__snake_case	,					cur_len=__snake_case		)
													lowerCamelCase						:Optional[int]						  =     eos_dist_proc(__snake_case	,					__snake_case	,					cur_len=__snake_case		)
													return scores

										# with processor list
										def run_processor_list(__snake_case     :    Union[str, Any]	,					__snake_case     :    int	,					__snake_case     :    Tuple		):
													lowerCamelCase						:Union[str, Any]						  =     FlaxLogitsProcessorList(
													    [temp_dist_warp, top_k_warp, top_p_warp, min_dist_proc, bos_dist_proc, eos_dist_proc]		)
													lowerCamelCase						:List[str]						  =     processor(__snake_case	,					__snake_case	,					cur_len=__snake_case		)
													return scores

										lowerCamelCase						:Optional[Any]						  =     jax.jit(__snake_case		)
										lowerCamelCase						:Tuple						  =     jax.jit(__snake_case		)

										lowerCamelCase						:List[str]						  =     jitted_run_no_processor_list(__snake_case	,					__snake_case	,					__snake_case		)
										lowerCamelCase						:int						  =     jitted_run_processor_list(__snake_case	,					__snake_case	,					__snake_case		)

										# scores should be equal
										self.assertTrue(jnp.allclose(__snake_case	,					__snake_case	,					atol=1e-3		)		)

										# input_ids should never be changed
										self.assertListEqual(input_ids.tolist()	,					input_ids_comp.tolist()		)




import ast
import os
import re
import shutil
import tempfile
import unittest
from unittest import mock

import torch

from accelerate.test_utils.examples import compare_against_test
from accelerate.test_utils.testing import TempDirTestCase, require_trackers, run_command, slow
from accelerate.utils import write_basic_config


# DataLoaders built from `test_samples/MRPC` for quick testing
# Should mock `{script_name}.get_dataloaders` via:
# @mock.patch("{script_name}.get_dataloaders", mocked_dataloaders)

A__           =							[
    """cross_validation.py""",
    """gradient_accumulation.py""",
    """local_sgd.py""",
    """multi_process_metrics.py""",
    """memory.py""",
    """automatic_gradient_accumulation.py""",
    """fsdp_with_peak_mem_tracking.py""",
    """deepspeed_with_config_support.py""",
    """megatron_lm_gpt_pretraining.py""",
]
class   _lowerCAmelCase		(     unittest.TestCase				):







							def 				snake_case  (				self     :    Any	,					__snake_case     :    str	,					__snake_case     :    bool	,					__snake_case     :    str = None	,					__snake_case     :    list = None		):
										lowerCamelCase						:Tuple						  =     None
										lowerCamelCase						:Tuple						  =     os.path.abspath(os.path.join('''examples'''	,					'''by_feature'''		)		)
										lowerCamelCase						:Optional[int]						  =     os.path.abspath('''examples'''		)
										for item in os.listdir(__snake_case		):
													if item not in EXCLUDE_EXAMPLES:
																lowerCamelCase						:Optional[int]						  =     os.path.join(__snake_case	,					__snake_case		)
																if os.path.isfile(__snake_case		) and ".py" in item_path:
																			with self.subTest(
																			    tested_script=__snake_case	,					feature_script=__snake_case	,					tested_section='''main()''' if parser_only else '''training_function()'''	,					):
																						lowerCamelCase						:Union[str, Any]						  =     compare_against_test(
																						    os.path.join(__snake_case	,					__snake_case		)	,					__snake_case	,					__snake_case	,					__snake_case		)
																						lowerCamelCase						:int						  =     '''\n'''.join(__snake_case		)
																						if special_strings is not None:
																									for string in special_strings:
																												lowerCamelCase						:int						  =     diff.replace(__snake_case	,					''''''		)
																						self.assertEqual(__snake_case	,					''''''		)







							def 				snake_case  (				self     :    Dict		):
										self.one_complete_example('''complete_nlp_example.py'''	,					__snake_case		)
										self.one_complete_example('''complete_nlp_example.py'''	,					__snake_case		)







							def 				snake_case  (				self     :    Optional[Any]		):
										lowerCamelCase						:Union[str, Any]						  =     os.path.abspath(os.path.join('''examples'''	,					'''cv_example.py'''		)		)
										lowerCamelCase						:Optional[int]						  =     [
										    ''' ''' * 16 + '''{\n\n''',
										    ''' ''' * 20 + '''"accuracy": eval_metric["accuracy"],\n\n''',
										    ''' ''' * 20 + '''"f1": eval_metric["f1"],\n\n''',
										    ''' ''' * 20 + '''"train_loss": total_loss.item() / len(train_dataloader),\n\n''',
										    ''' ''' * 20 + '''"epoch": epoch,\n\n''',
										    ''' ''' * 16 + '''},\n\n''',
										    ''' ''' * 16 + '''step=epoch,\n''',
										    ''' ''' * 12,
										    ''' ''' * 8 + '''for step, batch in enumerate(active_dataloader):\n''',
										]
										self.one_complete_example('''complete_cv_example.py'''	,					__snake_case	,					__snake_case	,					__snake_case		)
										self.one_complete_example('''complete_cv_example.py'''	,					__snake_case	,					__snake_case	,					__snake_case		)


@mock.patch.dict(os.environ      ,   {'TESTING_MOCKED_DATALOADERS': '1'}				)
class   _lowerCAmelCase		(     __SCREAMING_SNAKE_CASE				):
							_UpperCAmelCase						    =     False







							@classmethod
							def 				snake_case  (				cls     :    Optional[Any]		):
										super().setUpClass()
										lowerCamelCase						:Any						  =     tempfile.mkdtemp()
										lowerCamelCase						:Optional[int]						  =     os.path.join(cls._tmpdir	,					'''default_config.yml'''		)

										write_basic_config(save_location=cls.configPath		)
										lowerCamelCase						:List[str]						  =     ['''accelerate''', '''launch''', '''--config_file''', cls.configPath]







							@classmethod
							def 				snake_case  (				cls     :    Dict		):
										super().tearDownClass()
										shutil.rmtree(cls._tmpdir		)







							def 				snake_case  (				self     :    int		):
										lowerCamelCase						:Any						  =     F"\n        examples/by_feature/checkpointing.py\n        --checkpointing_steps epoch\n        --output_dir {self.tmpdir}\n        ".split()
										run_command(self._launch_args + testargs		)
										self.assertTrue(os.path.exists(os.path.join(self.tmpdir	,					'''epoch_0'''		)		)		)







							def 				snake_case  (				self     :    List[Any]		):
										lowerCamelCase						:Tuple						  =     F"\n        examples/by_feature/checkpointing.py\n        --checkpointing_steps 1\n        --output_dir {self.tmpdir}\n        ".split()
										lowerCamelCase						:List[Any]						  =     run_command(self._launch_args + testargs		)
										self.assertTrue(os.path.exists(os.path.join(self.tmpdir	,					'''step_2'''		)		)		)







							def 				snake_case  (				self     :    List[str]		):
										lowerCamelCase						:Dict						  =     F"\n        examples/by_feature/checkpointing.py\n        --resume_from_checkpoint {os.path.join(self.tmpdir	,					'epoch_0'		)}\n        ".split()
										lowerCamelCase						:Dict						  =     run_command(self._launch_args + testargs	,					return_stdout=__snake_case		)
										self.assertNotIn('''epoch 0:'''	,					__snake_case		)
										self.assertIn('''epoch 1:'''	,					__snake_case		)







							def 				snake_case  (				self     :    str		):
										lowerCamelCase						:List[Any]						  =     F"\n        examples/by_feature/checkpointing.py\n        --resume_from_checkpoint {os.path.join(self.tmpdir	,					'step_2'		)}\n        ".split()
										lowerCamelCase						:Optional[int]						  =     run_command(self._launch_args + testargs	,					return_stdout=__snake_case		)
										if torch.cuda.is_available():
													lowerCamelCase						:Union[str, Any]						  =     torch.cuda.device_count()
										else:
													lowerCamelCase						:Dict						  =     1
										if num_processes > 1:
													self.assertNotIn('''epoch 0:'''	,					__snake_case		)
													self.assertIn('''epoch 1:'''	,					__snake_case		)
										else:
													self.assertIn('''epoch 0:'''	,					__snake_case		)
													self.assertIn('''epoch 1:'''	,					__snake_case		)







							@slow
							def 				snake_case  (				self     :    Any		):
										lowerCamelCase						:Tuple						  =     '''
        examples/by_feature/cross_validation.py
        --num_folds 2
        '''.split()
										with mock.patch.dict(os.environ	,					{'''TESTING_MOCKED_DATALOADERS''': '''0'''}		):
													lowerCamelCase						:Dict						  =     run_command(self._launch_args + testargs	,					return_stdout=__snake_case		)
													lowerCamelCase						:Tuple						  =     re.findall('''({.+})'''	,					__snake_case		)
													lowerCamelCase						:Optional[Any]						  =     [r for r in results if '''accuracy''' in r][-1]
													lowerCamelCase						:List[str]						  =     ast.literal_eval(__snake_case		)
													self.assertGreaterEqual(results['''accuracy''']	,					0.7_5		)







							def 				snake_case  (				self     :    int		):
										lowerCamelCase						:Dict						  =     ['''examples/by_feature/multi_process_metrics.py''']
										run_command(self._launch_args + testargs		)







							@require_trackers
							@mock.patch.dict(os.environ	,					{'''WANDB_MODE''': '''offline'''}		)
							def 				snake_case  (				self     :    Any		):
										with tempfile.TemporaryDirectory() as tmpdir:
													lowerCamelCase						:Tuple						  =     F"\n            examples/by_feature/tracking.py\n            --with_tracking\n            --project_dir {tmpdir}\n            ".split()
													run_command(self._launch_args + testargs		)
													self.assertTrue(os.path.exists(os.path.join(__snake_case	,					'''tracking'''		)		)		)







							def 				snake_case  (				self     :    Tuple		):
										lowerCamelCase						:Tuple						  =     ['''examples/by_feature/gradient_accumulation.py''']
										run_command(self._launch_args + testargs		)







							def 				snake_case  (				self     :    Optional[Any]		):
										lowerCamelCase						:int						  =     ['''examples/by_feature/local_sgd.py''']
										run_command(self._launch_args + testargs		)



