






'''simple docstring'''

import copy
import tempfile
import unittest

from huggingface_hub import HfFolder, delete_repo
from parameterized import parameterized
from requests.exceptions import HTTPError

from transformers import AutoConfig, GenerationConfig
from transformers.testing_utils import TOKEN, USER, is_staging_test






class 							__magic_name__     (	unittest.TestCase				):







	"""simple docstring"""



	@parameterized.expand([(None,), ('foo.json',)]       )
	def        lowerCamelCase__		(							self  ,      _lowercase       )						->  Any:
							lowercase_      :   Tuple        =			GenerationConfig(
							    do_sample=_lowercase  ,      temperature=0.7  ,      length_penalty=1.0  ,      bad_words_ids=[[1, 2, 3], [4, 5]]  ,      )
							with tempfile.TemporaryDirectory() as tmp_dir:
													config.save_pretrained(_lowercase  ,      config_name=_lowercase       )
													lowercase_      :   Any        =			GenerationConfig.from_pretrained(_lowercase  ,      config_name=_lowercase       )

							# Checks parameters that were specified
							self.assertEqual(loaded_config.do_sample  ,      _lowercase       )
							self.assertEqual(loaded_config.temperature  ,      0.7       )
							self.assertEqual(loaded_config.length_penalty  ,      1.0       )
							self.assertEqual(loaded_config.bad_words_ids  ,      [[1, 2, 3], [4, 5]]       )

							# Checks parameters that were not specified (defaults)
							self.assertEqual(loaded_config.top_k  ,      50       )
							self.assertEqual(loaded_config.max_length  ,      20       )
							self.assertEqual(loaded_config.max_time  ,      _lowercase       )
	def        lowerCamelCase__		(							self       )						->  Optional[Any]:
							lowercase_      :   Optional[Any]        =			AutoConfig.from_pretrained('gpt2'       )
							lowercase_      :   Optional[Any]        =			GenerationConfig.from_model_config(_lowercase       )
							lowercase_      :   Tuple        =			GenerationConfig()

							# The generation config has loaded a few non-default parameters from the model config
							self.assertNotEqual(_lowercase  ,      _lowercase       )

							# One of those parameters is eos_token_id -- check if it matches
							self.assertNotEqual(generation_config_from_model.eos_token_id  ,      default_generation_config.eos_token_id       )
							self.assertEqual(generation_config_from_model.eos_token_id  ,      model_config.eos_token_id       )
	def        lowerCamelCase__		(							self       )						->  Tuple:
							lowercase_      :   str        =			GenerationConfig()
							lowercase_      :   Any        =			{
							    'max_new_tokens': 1024,
							    'foo': 'bar',
							}
							lowercase_      :   str        =			copy.deepcopy(_lowercase       )
							lowercase_      :   Any        =			generation_config.update(**_lowercase       )

							# update_kwargs was not modified (no side effects)
							self.assertEqual(_lowercase  ,      _lowercase       )

							# update_kwargs was used to update the config on valid attributes
							self.assertEqual(generation_config.max_new_tokens  ,      1024       )

							# `.update()` returns a dictionary of unused kwargs
							self.assertEqual(_lowercase  ,      {'foo': 'bar'}       )
	def        lowerCamelCase__		(							self       )						->  Union[str, Any]:
							lowercase_      :   Optional[Any]        =			GenerationConfig()
							lowercase_      :   Any        =			'bar'

							with tempfile.TemporaryDirectory('test-generation-config'       ) as tmp_dir:
													generation_config.save_pretrained(_lowercase       )

													lowercase_      :   Dict        =			GenerationConfig.from_pretrained(_lowercase       )
							# update_kwargs was used to update the config on valid attributes
							self.assertEqual(new_config.foo  ,      'bar'       )

							lowercase_      :   Tuple        =			GenerationConfig.from_model_config(_lowercase       )
							assert not hasattr(_lowercase  ,      'foo'       )  # no new kwargs should be initialized if from config







	def        lowerCamelCase__		(							self       )						->  List[str]:
							lowercase_      :   int        =			GenerationConfig()
							self.assertEqual(default_config.temperature  ,      1.0       )
							self.assertEqual(default_config.do_sample  ,      _lowercase       )
							self.assertEqual(default_config.num_beams  ,      1       )

							lowercase_      :   int        =			GenerationConfig(
							    do_sample=_lowercase  ,      temperature=0.7  ,      length_penalty=1.0  ,      bad_words_ids=[[1, 2, 3], [4, 5]]  ,      )
							self.assertEqual(config.temperature  ,      0.7       )
							self.assertEqual(config.do_sample  ,      _lowercase       )
							self.assertEqual(config.num_beams  ,      1       )

							with tempfile.TemporaryDirectory() as tmp_dir:
													config.save_pretrained(_lowercase       )
													lowercase_      :   Optional[int]        =			GenerationConfig.from_pretrained(_lowercase  ,      temperature=1.0       )

							self.assertEqual(loaded_config.temperature  ,      1.0       )
							self.assertEqual(loaded_config.do_sample  ,      _lowercase       )
							self.assertEqual(loaded_config.num_beams  ,      1       )  # default value






@is_staging_test
class 							__magic_name__     (	unittest.TestCase				):







	"""simple docstring"""



	@classmethod
	def        lowerCamelCase__		(							cls       )						->  Optional[int]:
							lowercase_      :   List[Any]        =			TOKEN
							HfFolder.save_token(_lowercase       )
	@classmethod
	def        lowerCamelCase__		(							cls       )						->  Tuple:
							try:
													delete_repo(token=cls._token  ,      repo_id='test-generation-config'       )
							except HTTPError:
													pass

							try:
													delete_repo(token=cls._token  ,      repo_id='valid_org/test-generation-config-org'       )
							except HTTPError:
													pass
	def        lowerCamelCase__		(							self       )						->  List[str]:
							lowercase_      :   Tuple        =			GenerationConfig(
							    do_sample=_lowercase  ,      temperature=0.7  ,      length_penalty=1.0  ,      )
							config.push_to_hub('test-generation-config'  ,      use_auth_token=self._token       )

							lowercase_      :   Union[str, Any]        =			GenerationConfig.from_pretrained(f"{USER}/test-generation-config"       )
							for k, v in config.to_dict().items():
													if k != "transformers_version":
																			self.assertEqual(_lowercase  ,      getattr(_lowercase  ,      _lowercase       )       )

        # Reset repo
							delete_repo(token=self._token  ,      repo_id='test-generation-config'       )

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
													config.save_pretrained(
													    _lowercase  ,      repo_id='test-generation-config'  ,      push_to_hub=_lowercase  ,      use_auth_token=self._token       )

							lowercase_      :   Tuple        =			GenerationConfig.from_pretrained(f"{USER}/test-generation-config"       )
							for k, v in config.to_dict().items():
													if k != "transformers_version":
																			self.assertEqual(_lowercase  ,      getattr(_lowercase  ,      _lowercase       )       )







	def        lowerCamelCase__		(							self       )						->  Dict:
							lowercase_      :   Tuple        =			GenerationConfig(
							    do_sample=_lowercase  ,      temperature=0.7  ,      length_penalty=1.0  ,      )
							config.push_to_hub('valid_org/test-generation-config-org'  ,      use_auth_token=self._token       )

							lowercase_      :   Optional[int]        =			GenerationConfig.from_pretrained('valid_org/test-generation-config-org'       )
							for k, v in config.to_dict().items():
													if k != "transformers_version":
																			self.assertEqual(_lowercase  ,      getattr(_lowercase  ,      _lowercase       )       )

        # Reset repo
							delete_repo(token=self._token  ,      repo_id='valid_org/test-generation-config-org'       )

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
													config.save_pretrained(
													    _lowercase  ,      repo_id='valid_org/test-generation-config-org'  ,      push_to_hub=_lowercase  ,      use_auth_token=self._token       )

							lowercase_      :   Union[str, Any]        =			GenerationConfig.from_pretrained('valid_org/test-generation-config-org'       )
							for k, v in config.to_dict().items():
													if k != "transformers_version":
																			self.assertEqual(_lowercase  ,      getattr(_lowercase  ,      _lowercase       )       )








'''simple docstring'''

from pathlib import PurePosixPath
from typing import Optional

import fsspec
from fsspec import AbstractFileSystem
from huggingface_hub.hf_api import DatasetInfo

from ..utils.file_utils import get_authentication_headers_for_url
from ..utils.hub import hf_hub_url






class 							__magic_name__     (	UpperCAmelCase_				):







	"""simple docstring"""



	SCREAMING_SNAKE_CASE_					:      Union[str, Any]        =  ''
	SCREAMING_SNAKE_CASE_					:      Union[str, Any]        =  'hf-legacy'  # "hf://"" is reserved for hffs
	def __init__(							self  ,      _lowercase = None  ,      _lowercase = None  ,      **_lowercase  ,      )						->  Optional[Any]:
							super().__init__(self  ,      **_lowercase       )
							lowercase_      :   int        =			repo_info
							lowercase_      :   List[Any]        =			token
							lowercase_      :   Union[str, Any]        =			None
	def        lowerCamelCase__		(							self       )						->  Optional[Any]:
							if self.dir_cache is None:
													lowercase_      :   Optional[Any]        =			{}
													for hf_file in self.repo_info.siblings:
																			# TODO(QL): add sizes
																			lowercase_      :   str        =			{
																			    'name': hf_file.rfilename,
																			    'size': None,
																			    'type': 'file',
																			}
																			self.dir_cache.update(
																			    {
																			        str(_lowercase       ): {'name': str(_lowercase       ), 'size': None, 'type': 'directory'}
																			        for d in list(PurePosixPath(hf_file.rfilename       ).parents       )[:-1]
																			    }       )
	def        lowerCamelCase__		(							self  ,      _lowercase  ,      _lowercase = "rb"  ,      **_lowercase  ,      )						->  Dict:
							if not isinstance(self.repo_info  ,      _lowercase       ):
													raise NotImplementedError(f"Open is only implemented for dataset repositories, but got {self.repo_info}"       )
							lowercase_      :   Optional[int]        =			hf_hub_url(self.repo_info.id  ,      _lowercase  ,      revision=self.repo_info.sha       )
							return fsspec.open(
							    _lowercase  ,      mode=_lowercase  ,      headers=get_authentication_headers_for_url(_lowercase  ,      use_auth_token=self.token       )  ,      client_kwargs={'trust_env': True}  ,      ).open()
	def        lowerCamelCase__		(							self  ,      _lowercase  ,      **_lowercase       )						->  Tuple:
							self._get_dirs()
							lowercase_      :   str        =			self._strip_protocol(_lowercase       )
							if path in self.dir_cache:
													return self.dir_cache[path]
							else:
													raise FileNotFoundError(_lowercase       )







	def        lowerCamelCase__		(							self  ,      _lowercase  ,      _lowercase=False  ,      **_lowercase       )						->  List[str]:
							self._get_dirs()
							lowercase_      :   List[str]        =			PurePosixPath(path.strip('/'       )       )
							lowercase_      :   List[str]        =			{}
							for p, f in self.dir_cache.items():
													lowercase_      :   Tuple        =			PurePosixPath(p.strip('/'       )       )
													lowercase_      :   Optional[int]        =			p.parent
													if root == path:
																			lowercase_      :   List[str]        =			f
							lowercase_      :   List[str]        =			list(paths.values()       )
							if detail:
													return out
							else:
													return sorted(f['name'] for f in out       )
