
'''simple docstring'''

import os
from pathlib import Path
from unittest.mock import patch

import pytest
import zstandard as zstd

from datasets.download.download_config import DownloadConfig
from datasets.utils.file_utils import (
    OfflineModeIsEnabled,
    cached_path,
    fsspec_get,
    fsspec_head,
    ftp_get,
    ftp_head,
    get_from_cache,
    http_get,
    http_head,
)


__snake_case:  str								=			"\\n    Text data.\n    Second line of data."

__snake_case:  List[Any]								=			"file"






@pytest.fixture(scope="""session"""   )
def 	_snake_case						(      A_       :    Optional[int]   ):
  """simple docstring"""







  a_      :	Optional[Any]      =							tmp_path_factory.mktemp("""data"""   ) / (FILE_PATH + """.zstd""")
  a_      :	Optional[Any]      =							bytes(__lowerCAmelCase     ,		"""utf-8"""   )
  with zstd.open(__lowerCAmelCase     ,		"""wb"""   ) as f:
    f.write(__lowerCAmelCase   )
  return path






@pytest.fixture
def 	_snake_case						(      A_       :    Tuple   ):
  """simple docstring"""







  with open(os.path.join(tmpfs.local_root_dir     ,		__lowerCAmelCase   )     ,		"""w"""   ) as f:
    f.write(__lowerCAmelCase   )
  return FILE_PATH






@pytest.mark.parametrize("""compression_format"""     ,		["""gzip""", """xz""", """zstd"""]   )
def 	_snake_case						(      A_       :    Any     ,		A_       :    Optional[Any]     ,		A_       :    List[str]     ,		A_       :    int     ,		A_       :    List[str]     ,		A_       :    Union[str, Any]   ):
  """simple docstring"""







  a_      :	Dict      =							{"""gzip""": gz_file, """xz""": xz_file, """zstd""": zstd_path}
  a_      :	Dict      =							input_paths[compression_format]
  a_      :	Dict      =							tmp_path / """cache"""
  a_      :	Any      =							DownloadConfig(cache_dir=__lowerCAmelCase     ,		extract_compressed_file=__lowerCAmelCase   )
  a_      :	Union[str, Any]      =							cached_path(__lowerCAmelCase     ,		download_config=__lowerCAmelCase   )
  with open(__lowerCAmelCase   ) as f:
    a_      :	Optional[int]      =							f.read()
  with open(__lowerCAmelCase   ) as f:
    a_      :	Dict      =							f.read()
  assert extracted_file_content == expected_file_content






@pytest.mark.parametrize("""default_extracted"""     ,		[True, False]   )
@pytest.mark.parametrize("""default_cache_dir"""     ,		[True, False]   )
def 	_snake_case						(      A_       :    Optional[Any]     ,		A_       :    List[Any]     ,		A_       :    List[str]     ,		A_       :    Optional[Any]     ,		A_       :    Any   ):
  """simple docstring"""







  a_      :	Union[str, Any]      =							"""custom_cache"""
  a_      :	Dict      =							"""custom_extracted_dir"""
  a_      :	List[Any]      =							tmp_path / """custom_extracted_path"""
  if default_extracted:
    a_      :	List[Any]      =							("""downloads""" if default_cache_dir else custom_cache_dir, """extracted""")
  else:
    monkeypatch.setattr("""datasets.config.EXTRACTED_DATASETS_DIR"""     ,		__lowerCAmelCase   )
    monkeypatch.setattr("""datasets.config.EXTRACTED_DATASETS_PATH"""     ,		str(__lowerCAmelCase   )   )
    a_      :	List[str]      =							custom_extracted_path.parts[-2:] if default_cache_dir else (custom_cache_dir, custom_extracted_dir)

  a_      :	Optional[int]      =							xz_file
  a_      :	str      =							(
      DownloadConfig(extract_compressed_file=__lowerCAmelCase   )
      if default_cache_dir
      else DownloadConfig(cache_dir=tmp_path / custom_cache_dir     ,		extract_compressed_file=__lowerCAmelCase   )
  )
  a_      :	List[str]      =							cached_path(__lowerCAmelCase     ,		download_config=__lowerCAmelCase   )
  assert Path(__lowerCAmelCase   ).parent.parts[-2:] == expected






def 	_snake_case						(      A_       :    Optional[int]   ):
  """simple docstring"""







  a_      :	Optional[int]      =							str(Path(__lowerCAmelCase   ).resolve()   )
  assert cached_path(__lowerCAmelCase   ) == text_file
  # relative path
  a_      :	List[str]      =							str(Path(__lowerCAmelCase   ).resolve().relative_to(Path(os.getcwd()   )   )   )
  assert cached_path(__lowerCAmelCase   ) == text_file






def 	_snake_case						(      A_       :    Optional[int]   ):
  """simple docstring"""







  a_      :	Optional[Any]      =							str(tmp_path.resolve() / """__missing_file__.txt"""   )
  with pytest.raises(__lowerCAmelCase   ):
    cached_path(__lowerCAmelCase   )
  # relative path
  a_      :	Union[str, Any]      =							"""./__missing_file__.txt"""
  with pytest.raises(__lowerCAmelCase   ):
    cached_path(__lowerCAmelCase   )






def 	_snake_case						(      A_       :    int   ):
  """simple docstring"""







  a_      :	Union[str, Any]      =							get_from_cache(f'''tmp://{tmpfs_file}'''   )
  with open(__lowerCAmelCase   ) as f:
    a_      :	Optional[int]      =							f.read()
  assert output_file_content == FILE_CONTENT






@patch("""datasets.config.HF_DATASETS_OFFLINE"""     ,		__lowerCAmelCase   )
def 	_snake_case						(      ):
  """simple docstring"""







  with pytest.raises(__lowerCAmelCase   ):
    cached_path("""https://huggingface.co"""   )






@patch("""datasets.config.HF_DATASETS_OFFLINE"""     ,		__lowerCAmelCase   )
def 	_snake_case						(      A_       :    Optional[int]   ):
  """simple docstring"""







  a_      :	List[Any]      =							tmp_path_factory.mktemp("""data"""   ) / """file.html"""
  with pytest.raises(__lowerCAmelCase   ):
    http_get("""https://huggingface.co"""     ,		temp_file=__lowerCAmelCase   )
  with pytest.raises(__lowerCAmelCase   ):
    http_head("""https://huggingface.co"""   )






@patch("""datasets.config.HF_DATASETS_OFFLINE"""     ,		__lowerCAmelCase   )
def 	_snake_case						(      A_       :    Optional[int]   ):
  """simple docstring"""







  a_      :	str      =							tmp_path_factory.mktemp("""data"""   ) / """file.html"""
  with pytest.raises(__lowerCAmelCase   ):
    ftp_get("""ftp://huggingface.co"""     ,		temp_file=__lowerCAmelCase   )
  with pytest.raises(__lowerCAmelCase   ):
    ftp_head("""ftp://huggingface.co"""   )






@patch("""datasets.config.HF_DATASETS_OFFLINE"""     ,		__lowerCAmelCase   )
def 	_snake_case						(      A_       :    List[str]   ):
  """simple docstring"""







  a_      :	Union[str, Any]      =							tmp_path_factory.mktemp("""data"""   ) / """file.html"""
  with pytest.raises(__lowerCAmelCase   ):
    fsspec_get("""s3://huggingface.co"""     ,		temp_file=__lowerCAmelCase   )
  with pytest.raises(__lowerCAmelCase   ):
    fsspec_head("""s3://huggingface.co"""   )




'''simple docstring'''

import os
import tempfile
import unittest
from pathlib import Path

from transformers import AutoConfig, is_torch_available
from transformers.testing_utils import require_torch, torch_device


if is_torch_available():
    from transformers import PyTorchBenchmark, PyTorchBenchmarkArguments


@require_torch
class  _UpperCAmelCase ( unittest.TestCase    ):

     """simple docstring"""

     def 							_lowerCAmelCase  (      self							,		lowerCAmelCase_ ):

       '''simple docstring'''

       for model_result in results.values():
         for batch_size, sequence_length in zip(model_result["""bs"""]							,		model_result["""ss"""] ):
           a_      :	str      =							model_result["""result"""][batch_size][sequence_length]
           self.assertIsNotNone(lowerCAmelCase_ )


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	Optional[int]      =							"""sshleifer/tiny-gpt2"""
       a_      :	Any      =							PyTorchBenchmarkArguments(
           models=[MODEL_ID]							,		training=lowerCAmelCase_							,		inference=lowerCAmelCase_							,		sequence_lengths=[8]							,		batch_sizes=[1]							,		multi_process=lowerCAmelCase_							,		)
       a_      :	List[Any]      =							PyTorchBenchmark(lowerCAmelCase_ )
       a_      :	List[Any]      =							benchmark.run()
       self.check_results_dict_not_empty(results.time_inference_result )
       self.check_results_dict_not_empty(results.memory_inference_result )


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	Optional[int]      =							"""sgugger/tiny-distilbert-classification"""
       a_      :	str      =							PyTorchBenchmarkArguments(
           models=[MODEL_ID]							,		training=lowerCAmelCase_							,		inference=lowerCAmelCase_							,		sequence_lengths=[8]							,		batch_sizes=[1]							,		multi_process=lowerCAmelCase_							,		only_pretrain_model=lowerCAmelCase_							,		)
       a_      :	Optional[Any]      =							PyTorchBenchmark(lowerCAmelCase_ )
       a_      :	Optional[Any]      =							benchmark.run()
       self.check_results_dict_not_empty(results.time_inference_result )
       self.check_results_dict_not_empty(results.memory_inference_result )


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	List[str]      =							"""sshleifer/tiny-gpt2"""
       a_      :	Any      =							PyTorchBenchmarkArguments(
           models=[MODEL_ID]							,		training=lowerCAmelCase_							,		inference=lowerCAmelCase_							,		torchscript=lowerCAmelCase_							,		sequence_lengths=[8]							,		batch_sizes=[1]							,		multi_process=lowerCAmelCase_							,		)
       a_      :	Optional[Any]      =							PyTorchBenchmark(lowerCAmelCase_ )
       a_      :	Optional[Any]      =							benchmark.run()
       self.check_results_dict_not_empty(results.time_inference_result )
       self.check_results_dict_not_empty(results.memory_inference_result )


     @unittest.skipIf(torch_device == """cpu"""							,		"""Cant do half precision""" )
     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	int      =							"""sshleifer/tiny-gpt2"""
       a_      :	Tuple      =							PyTorchBenchmarkArguments(
           models=[MODEL_ID]							,		training=lowerCAmelCase_							,		inference=lowerCAmelCase_							,		fpaa=lowerCAmelCase_							,		sequence_lengths=[8]							,		batch_sizes=[1]							,		multi_process=lowerCAmelCase_							,		)
       a_      :	List[Any]      =							PyTorchBenchmark(lowerCAmelCase_ )
       a_      :	Dict      =							benchmark.run()
       self.check_results_dict_not_empty(results.time_inference_result )
       self.check_results_dict_not_empty(results.memory_inference_result )


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	Dict      =							"""sshleifer/tiny-gpt2"""
       a_      :	Dict      =							AutoConfig.from_pretrained(lowerCAmelCase_ )
       # set architectures equal to `None`
       a_      :	Any      =							None
       a_      :	List[Any]      =							PyTorchBenchmarkArguments(
           models=[MODEL_ID]							,		training=lowerCAmelCase_							,		inference=lowerCAmelCase_							,		sequence_lengths=[8]							,		batch_sizes=[1]							,		multi_process=lowerCAmelCase_							,		)
       a_      :	Optional[Any]      =							PyTorchBenchmark(lowerCAmelCase_							,		configs=[config] )
       a_      :	Any      =							benchmark.run()
       self.check_results_dict_not_empty(results.time_inference_result )
       self.check_results_dict_not_empty(results.memory_inference_result )


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	Dict      =							"""sshleifer/tiny-gpt2"""
       a_      :	Tuple      =							PyTorchBenchmarkArguments(
           models=[MODEL_ID]							,		training=lowerCAmelCase_							,		inference=lowerCAmelCase_							,		sequence_lengths=[8]							,		batch_sizes=[1]							,		multi_process=lowerCAmelCase_							,		)
       a_      :	int      =							PyTorchBenchmark(lowerCAmelCase_ )
       a_      :	Tuple      =							benchmark.run()
       self.check_results_dict_not_empty(results.time_train_result )
       self.check_results_dict_not_empty(results.memory_train_result )


     @unittest.skipIf(torch_device == """cpu"""							,		"""Can't do half precision""" )
     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	List[str]      =							"""sshleifer/tiny-gpt2"""
       a_      :	List[Any]      =							PyTorchBenchmarkArguments(
           models=[MODEL_ID]							,		training=lowerCAmelCase_							,		inference=lowerCAmelCase_							,		sequence_lengths=[8]							,		batch_sizes=[1]							,		fpaa=lowerCAmelCase_							,		multi_process=lowerCAmelCase_							,		)
       a_      :	List[str]      =							PyTorchBenchmark(lowerCAmelCase_ )
       a_      :	Optional[int]      =							benchmark.run()
       self.check_results_dict_not_empty(results.time_train_result )
       self.check_results_dict_not_empty(results.memory_train_result )


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	List[Any]      =							"""sshleifer/tiny-gpt2"""
       a_      :	List[str]      =							AutoConfig.from_pretrained(lowerCAmelCase_ )
       a_      :	Union[str, Any]      =							PyTorchBenchmarkArguments(
           models=[MODEL_ID]							,		training=lowerCAmelCase_							,		inference=lowerCAmelCase_							,		sequence_lengths=[8]							,		batch_sizes=[1]							,		multi_process=lowerCAmelCase_							,		)
       a_      :	Tuple      =							PyTorchBenchmark(lowerCAmelCase_							,		configs=[config] )
       a_      :	List[str]      =							benchmark.run()
       self.check_results_dict_not_empty(results.time_inference_result )
       self.check_results_dict_not_empty(results.memory_inference_result )


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	Dict      =							"""sshleifer/tinier_bart"""
       a_      :	Union[str, Any]      =							AutoConfig.from_pretrained(lowerCAmelCase_ )
       a_      :	Dict      =							PyTorchBenchmarkArguments(
           models=[MODEL_ID]							,		training=lowerCAmelCase_							,		inference=lowerCAmelCase_							,		sequence_lengths=[8]							,		batch_sizes=[1]							,		multi_process=lowerCAmelCase_							,		)
       a_      :	Union[str, Any]      =							PyTorchBenchmark(lowerCAmelCase_							,		configs=[config] )
       a_      :	Optional[Any]      =							benchmark.run()
       self.check_results_dict_not_empty(results.time_inference_result )
       self.check_results_dict_not_empty(results.memory_inference_result )


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	Union[str, Any]      =							"""sshleifer/tiny-gpt2"""
       a_      :	Tuple      =							AutoConfig.from_pretrained(lowerCAmelCase_ )
       a_      :	Tuple      =							PyTorchBenchmarkArguments(
           models=[MODEL_ID]							,		training=lowerCAmelCase_							,		inference=lowerCAmelCase_							,		sequence_lengths=[8]							,		batch_sizes=[1]							,		multi_process=lowerCAmelCase_							,		)
       a_      :	Union[str, Any]      =							PyTorchBenchmark(lowerCAmelCase_							,		configs=[config] )
       a_      :	Tuple      =							benchmark.run()
       self.check_results_dict_not_empty(results.time_train_result )
       self.check_results_dict_not_empty(results.memory_train_result )


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	Optional[int]      =							"""sshleifer/tinier_bart"""
       a_      :	int      =							AutoConfig.from_pretrained(lowerCAmelCase_ )
       a_      :	List[Any]      =							PyTorchBenchmarkArguments(
           models=[MODEL_ID]							,		training=lowerCAmelCase_							,		inference=lowerCAmelCase_							,		sequence_lengths=[8]							,		batch_sizes=[1]							,		multi_process=lowerCAmelCase_							,		)
       a_      :	int      =							PyTorchBenchmark(lowerCAmelCase_							,		configs=[config] )
       a_      :	Dict      =							benchmark.run()
       self.check_results_dict_not_empty(results.time_train_result )
       self.check_results_dict_not_empty(results.memory_train_result )


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	Optional[Any]      =							"""sshleifer/tiny-gpt2"""
       with tempfile.TemporaryDirectory() as tmp_dir:
         a_      :	List[str]      =							PyTorchBenchmarkArguments(
             models=[MODEL_ID]							,		training=lowerCAmelCase_							,		inference=lowerCAmelCase_							,		save_to_csv=lowerCAmelCase_							,		sequence_lengths=[8]							,		batch_sizes=[1]							,		inference_time_csv_file=os.path.join(lowerCAmelCase_							,		"""inf_time.csv""" )							,		train_memory_csv_file=os.path.join(lowerCAmelCase_							,		"""train_mem.csv""" )							,		inference_memory_csv_file=os.path.join(lowerCAmelCase_							,		"""inf_mem.csv""" )							,		train_time_csv_file=os.path.join(lowerCAmelCase_							,		"""train_time.csv""" )							,		env_info_csv_file=os.path.join(lowerCAmelCase_							,		"""env.csv""" )							,		multi_process=lowerCAmelCase_							,		)
         a_      :	Tuple      =							PyTorchBenchmark(lowerCAmelCase_ )
         benchmark.run()
         self.assertTrue(Path(os.path.join(lowerCAmelCase_							,		"""inf_time.csv""" ) ).exists() )
         self.assertTrue(Path(os.path.join(lowerCAmelCase_							,		"""train_time.csv""" ) ).exists() )
         self.assertTrue(Path(os.path.join(lowerCAmelCase_							,		"""inf_mem.csv""" ) ).exists() )
         self.assertTrue(Path(os.path.join(lowerCAmelCase_							,		"""train_mem.csv""" ) ).exists() )
         self.assertTrue(Path(os.path.join(lowerCAmelCase_							,		"""env.csv""" ) ).exists() )






     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	Dict      =							"""sshleifer/tiny-gpt2"""

       def _check_summary_is_not_empty(lowerCAmelCase_ ):
         self.assertTrue(hasattr(lowerCAmelCase_							,		"""sequential""" ) )
         self.assertTrue(hasattr(lowerCAmelCase_							,		"""cumulative""" ) )
         self.assertTrue(hasattr(lowerCAmelCase_							,		"""current""" ) )
         self.assertTrue(hasattr(lowerCAmelCase_							,		"""total""" ) )

       with tempfile.TemporaryDirectory() as tmp_dir:
         a_      :	List[Any]      =							PyTorchBenchmarkArguments(
             models=[MODEL_ID]							,		training=lowerCAmelCase_							,		inference=lowerCAmelCase_							,		sequence_lengths=[8]							,		batch_sizes=[1]							,		log_filename=os.path.join(lowerCAmelCase_							,		"""log.txt""" )							,		log_print=lowerCAmelCase_							,		trace_memory_line_by_line=lowerCAmelCase_							,		multi_process=lowerCAmelCase_							,		)
         a_      :	Dict      =							PyTorchBenchmark(lowerCAmelCase_ )
         a_      :	Tuple      =							benchmark.run()
         _check_summary_is_not_empty(result.inference_summary )
         _check_summary_is_not_empty(result.train_summary )
         self.assertTrue(Path(os.path.join(lowerCAmelCase_							,		"""log.txt""" ) ).exists() )


