





'''simple docstring'''







import ast
import os
import re
import shutil
import tempfile
import unittest
from unittest import mock

import torch

from accelerate.test_utils.examples import compare_against_test
from accelerate.test_utils.testing import TempDirTestCase, require_trackers, run_command, slow
from accelerate.utils import write_basic_config


# DataLoaders built from `test_samples/MRPC` for quick testing
# Should mock `{script_name}.get_dataloaders` via:
# @mock.patch("{script_name}.get_dataloaders", mocked_dataloaders)

_lowercase       						=			[
    'cross_validation.py',
    'gradient_accumulation.py',
    'local_sgd.py',
    'multi_process_metrics.py',
    'memory.py',
    'automatic_gradient_accumulation.py',
    'fsdp_with_peak_mem_tracking.py',
    'deepspeed_with_config_support.py',
    'megatron_lm_gpt_pretraining.py',
]
class 			_lowercase						(       unittest.TestCase					):







     def     UpperCamelCase			( self    ,   A__    ,   A__    ,   A__ = None    ,   A__ = None    ) ->	List[str]:
      snake_case				=			None
      snake_case				=			os.path.abspath(os.path.join('''examples'''    ,   '''by_feature'''    )    )
      snake_case				=			os.path.abspath('''examples'''    )
      for item in os.listdir(A__    ):
       if item not in EXCLUDE_EXAMPLES:
        snake_case				=			os.path.join(A__    ,   A__    )
        if os.path.isfile(A__    ) and ".py" in item_path:
         with self.subTest(
             tested_script=A__    ,   feature_script=A__    ,   tested_section='''main()''' if parser_only else '''training_function()'''    ,   ):
          snake_case				=			compare_against_test(
              os.path.join(A__    ,   A__    )    ,   A__    ,   A__    ,   A__    )
          snake_case				=			'''\n'''.join(A__    )
          if special_strings is not None:
           for string in special_strings:
            snake_case				=			diff.replace(A__    ,   ''''''    )
          self.assertEqual(A__    ,   ''''''    )







     def     UpperCamelCase			( self    ) ->	Optional[int]:
      self.one_complete_example('''complete_nlp_example.py'''    ,   A__    )
      self.one_complete_example('''complete_nlp_example.py'''    ,   A__    )







     def     UpperCamelCase			( self    ) ->	Optional[int]:
      snake_case				=			os.path.abspath(os.path.join('''examples'''    ,   '''cv_example.py'''    )    )
      snake_case				=			[
          ''' ''' * 16 + '''{\n\n''',
          ''' ''' * 20 + '''"accuracy": eval_metric["accuracy"],\n\n''',
          ''' ''' * 20 + '''"f1": eval_metric["f1"],\n\n''',
          ''' ''' * 20 + '''"train_loss": total_loss.item() / len(train_dataloader),\n\n''',
          ''' ''' * 20 + '''"epoch": epoch,\n\n''',
          ''' ''' * 16 + '''},\n\n''',
          ''' ''' * 16 + '''step=epoch,\n''',
          ''' ''' * 12,
          ''' ''' * 8 + '''for step, batch in enumerate(active_dataloader):\n''',
      ]
      self.one_complete_example('''complete_cv_example.py'''    ,   A__    ,   A__    ,   A__    )
      self.one_complete_example('''complete_cv_example.py'''    ,   A__    ,   A__    ,   A__    )




@mock.patch.dict(os.environ				,		{'''TESTING_MOCKED_DATALOADERS''': '''1'''}					)
class 			_lowercase						(       __a					):
     _UpperCAmelCase										=					False







     @classmethod
     def     UpperCamelCase			( cls    ) ->	Union[str, Any]:
      super().setUpClass()
      snake_case				=			tempfile.mkdtemp()
      snake_case				=			os.path.join(cls._tmpdir    ,   '''default_config.yml'''    )

      write_basic_config(save_location=cls.configPath    )
      snake_case				=			['''accelerate''', '''launch''', '''--config_file''', cls.configPath]







     @classmethod
     def     UpperCamelCase			( cls    ) ->	str:
      super().tearDownClass()
      shutil.rmtree(cls._tmpdir    )







     def     UpperCamelCase			( self    ) ->	List[Any]:
      snake_case				=			F"""
        examples/by_feature/checkpointing.py
        --checkpointing_steps epoch
        --output_dir {self.tmpdir}
        """.split()
      run_command(self._launch_args + testargs    )
      self.assertTrue(os.path.exists(os.path.join(self.tmpdir    ,   '''epoch_0'''    )    )    )







     def     UpperCamelCase			( self    ) ->	Any:
      snake_case				=			F"""
        examples/by_feature/checkpointing.py
        --checkpointing_steps 1
        --output_dir {self.tmpdir}
        """.split()
      snake_case				=			run_command(self._launch_args + testargs    )
      self.assertTrue(os.path.exists(os.path.join(self.tmpdir    ,   '''step_2'''    )    )    )







     def     UpperCamelCase			( self    ) ->	List[str]:
      snake_case				=			F"""
        examples/by_feature/checkpointing.py
        --resume_from_checkpoint {os.path.join(self.tmpdir    ,   'epoch_0'    )}
        """.split()
      snake_case				=			run_command(self._launch_args + testargs    ,   return_stdout=A__    )
      self.assertNotIn('''epoch 0:'''    ,   A__    )
      self.assertIn('''epoch 1:'''    ,   A__    )







     def     UpperCamelCase			( self    ) ->	Optional[Any]:
      snake_case				=			F"""
        examples/by_feature/checkpointing.py
        --resume_from_checkpoint {os.path.join(self.tmpdir    ,   'step_2'    )}
        """.split()
      snake_case				=			run_command(self._launch_args + testargs    ,   return_stdout=A__    )
      if torch.cuda.is_available():
       snake_case				=			torch.cuda.device_count()
      else:
       snake_case				=			1
      if num_processes > 1:
       self.assertNotIn('''epoch 0:'''    ,   A__    )
       self.assertIn('''epoch 1:'''    ,   A__    )
      else:
       self.assertIn('''epoch 0:'''    ,   A__    )
       self.assertIn('''epoch 1:'''    ,   A__    )







     @slow
     def     UpperCamelCase			( self    ) ->	str:
      snake_case				=			'''
        examples/by_feature/cross_validation.py
        --num_folds 2
        '''.split()
      with mock.patch.dict(os.environ    ,   {'''TESTING_MOCKED_DATALOADERS''': '''0'''}    ):
       snake_case				=			run_command(self._launch_args + testargs    ,   return_stdout=A__    )
       snake_case				=			re.findall('''({.+})'''    ,   A__    )
       snake_case				=			[r for r in results if '''accuracy''' in r][-1]
       snake_case				=			ast.literal_eval(A__    )
       self.assertGreaterEqual(results['''accuracy''']    ,   0.7_5    )







     def     UpperCamelCase			( self    ) ->	List[Any]:
      snake_case				=			['''examples/by_feature/multi_process_metrics.py''']
      run_command(self._launch_args + testargs    )







     @require_trackers
     @mock.patch.dict(os.environ    ,   {'''WANDB_MODE''': '''offline'''}    )
     def     UpperCamelCase			( self    ) ->	Tuple:
      with tempfile.TemporaryDirectory() as tmpdir:
       snake_case				=			F"""
            examples/by_feature/tracking.py
            --with_tracking
            --project_dir {tmpdir}
            """.split()
       run_command(self._launch_args + testargs    )
       self.assertTrue(os.path.exists(os.path.join(A__    ,   '''tracking'''    )    )    )







     def     UpperCamelCase			( self    ) ->	Dict:
      snake_case				=			['''examples/by_feature/gradient_accumulation.py''']
      run_command(self._launch_args + testargs    )







     def     UpperCamelCase			( self    ) ->	Union[str, Any]:
      snake_case				=			['''examples/by_feature/local_sgd.py''']
      run_command(self._launch_args + testargs    )








'''simple docstring'''







import warnings

from transformers import AutoTokenizer
from transformers.utils import is_torch_available
from transformers.utils.generic import ExplicitEnum

from ...processing_utils import ProcessorMixin


if is_torch_available():
   import torch
class 			_lowercase						(       __a					):
     _UpperCAmelCase										=					'''char'''
     _UpperCAmelCase										=					'''bpe'''
     _UpperCAmelCase										=					'''wp'''




_lowercase       						=			(DecodeType.CHARACTER, DecodeType.BPE, DecodeType.WORDPIECE)
class 			_lowercase						(       __a					):
     _UpperCAmelCase										=					['''image_processor''', '''char_tokenizer''']
     _UpperCAmelCase										=					'''ViTImageProcessor'''
     _UpperCAmelCase										=					'''MgpstrTokenizer'''







     def __init__( self    ,   A__=None    ,   A__=None    ,   **A__    ) ->	List[Any]:
      snake_case				=			None
      if "feature_extractor" in kwargs:
       warnings.warn(
           '''The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'''
           ''' instead.'''    ,   A__    ,   )
       snake_case				=			kwargs.pop('''feature_extractor'''    )

      snake_case				=			image_processor if image_processor is not None else feature_extractor
      if image_processor is None:
       raise ValueError('''You need to specify an `image_processor`.'''    )
      if tokenizer is None:
       raise ValueError('''You need to specify a `tokenizer`.'''    )

      snake_case				=			tokenizer
      snake_case				=			AutoTokenizer.from_pretrained('''gpt2'''    )
      snake_case				=			AutoTokenizer.from_pretrained('''bert-base-uncased'''    )

      super().__init__(A__    ,   A__    )







     def __call__( self    ,   A__=None    ,   A__=None    ,   A__=None    ,   **A__    ) ->	List[str]:
      if images is None and text is None:
       raise ValueError('''You need to specify either an `images` or `text` input to process.'''    )

      if images is not None:
       snake_case				=			self.image_processor(A__    ,   return_tensors=A__    ,   **A__    )
      if text is not None:
       snake_case				=			self.char_tokenizer(A__    ,   return_tensors=A__    ,   **A__    )

      if text is None:
       return inputs
      elif images is None:
       return encodings
      else:
       snake_case				=			encodings['''input_ids''']
       return inputs







     def     UpperCamelCase			( self    ,   A__    ) ->	Dict:
      snake_case				,		snake_case				,		snake_case				=			sequences
      snake_case				=			char_preds.size(0    )

      snake_case				,		snake_case				=			self._decode_helper(A__    ,   '''char'''    )
      snake_case				,		snake_case				=			self._decode_helper(A__    ,   '''bpe'''    )
      snake_case				,		snake_case				=			self._decode_helper(A__    ,   '''wp'''    )

      snake_case				=			[]
      snake_case				=			[]
      for i in range(A__    ):
       snake_case				=			[char_scores[i], bpe_scores[i], wp_scores[i]]
       snake_case				=			[char_strs[i], bpe_strs[i], wp_strs[i]]
       snake_case				=			scores.index(max(A__    )    )
       final_strs.append(strs[max_score_index]    )
       final_scores.append(scores[max_score_index]    )

      snake_case				=			{}
      snake_case				=			final_strs
      snake_case				=			final_scores
      snake_case				=			char_strs
      snake_case				=			bpe_strs
      snake_case				=			wp_strs
      return out







     def     UpperCamelCase			( self    ,   A__    ,   A__    ) ->	Optional[Any]:
      if format == DecodeType.CHARACTER:
       snake_case				=			self.char_decode
       snake_case				=			1
       snake_case				=			'''[s]'''
      elif format == DecodeType.BPE:
       snake_case				=			self.bpe_decode
       snake_case				=			2
       snake_case				=			'''#'''
      elif format == DecodeType.WORDPIECE:
       snake_case				=			self.wp_decode
       snake_case				=			1_02
       snake_case				=			'''[SEP]'''
      else:
       raise ValueError(F"""Format {format} is not supported."""    )

      snake_case				,		snake_case				=			[], []
      snake_case				=			pred_logits.size(0    )
      snake_case				=			pred_logits.size(1    )
      snake_case				,		snake_case				=			pred_logits.topk(1    ,   dim=-1    ,   largest=A__    ,   sorted=A__    )
      snake_case				=			preds_index.view(-1    ,   A__    )[:, 1:]
      snake_case				=			decoder(A__    )
      snake_case				,		snake_case				=			torch.nn.functional.softmax(A__    ,   dim=2    ).max(dim=2    )
      snake_case				=			preds_max_prob[:, 1:]

      for index in range(A__    ):
       snake_case				=			preds_str[index].find(A__    )
       snake_case				=			preds_str[index][:pred_eos]
       snake_case				=			preds_index[index].cpu().tolist()
       snake_case				=			pred_index.index(A__    ) if eos_token in pred_index else -1
       snake_case				=			preds_max_prob[index][: pred_eos_index + 1]
       snake_case				=			pred_max_prob.cumprod(dim=0    )[-1] if pred_max_prob.nelement() != 0 else 0.0
       dec_strs.append(A__    )
       conf_scores.append(A__    )

      return dec_strs, conf_scores







     def     UpperCamelCase			( self    ,   A__    ) ->	int:
      snake_case				=			[seq.replace(''' '''    ,   ''''''    ) for seq in self.char_tokenizer.batch_decode(A__    )]
      return decode_strs







     def     UpperCamelCase			( self    ,   A__    ) ->	List[str]:
      return self.bpe_tokenizer.batch_decode(A__    )







     def     UpperCamelCase			( self    ,   A__    ) ->	Union[str, Any]:
      snake_case				=			[seq.replace(''' '''    ,   ''''''    ) for seq in self.wp_tokenizer.batch_decode(A__    )]
      return decode_strs

