






'''simple docstring'''


import os
from pathlib import Path
from unittest.mock import patch

import pytest
import zstandard as zstd

from datasets.download.download_config import DownloadConfig
from datasets.utils.file_utils import (
    OfflineModeIsEnabled,
    cached_path,
    fsspec_get,
    fsspec_head,
    ftp_get,
    ftp_head,
    get_from_cache,
    http_get,
    http_head,
)


lowerCAmelCase__										= '''\
    Text data.
    Second line of data.'''

lowerCAmelCase__										= '''file'''







@pytest.fixture(scope='session'  )
def 						SCREAMING_SNAKE_CASE(  UpperCamelCase  )    ->    Dict:
	UpperCAmelCase_		:      Union[str, Any]       	=					tmp_path_factory.mktemp('data'  ) / (FILE_PATH + '.zstd')
	UpperCAmelCase_		:      Any       	=					bytes(lowerCAmelCase__   ,'utf-8'  )
	with zstd.open(lowerCAmelCase__   ,'wb'  ) as f:
		f.write(lowerCAmelCase__  )
	return path







@pytest.fixture
def 						SCREAMING_SNAKE_CASE(  UpperCamelCase  )    ->    Any:
	with open(os.path.join(tmpfs.local_root_dir   ,lowerCAmelCase__  )   ,'w'  ) as f:
		f.write(lowerCAmelCase__  )
	return FILE_PATH







@pytest.mark.parametrize('compression_format'   ,['gzip', 'xz', 'zstd']  )
def 						SCREAMING_SNAKE_CASE(  UpperCamelCase   ,UpperCamelCase   ,UpperCamelCase   ,UpperCamelCase   ,UpperCamelCase   ,UpperCamelCase  )    ->    Any:
	UpperCAmelCase_		:      int       	=					{'gzip': gz_file, 'xz': xz_file, 'zstd': zstd_path}
	UpperCAmelCase_		:      List[str]       	=					input_paths[compression_format]
	UpperCAmelCase_		:      Dict       	=					tmp_path / 'cache'
	UpperCAmelCase_		:      str       	=					DownloadConfig(cache_dir=lowerCAmelCase__   ,extract_compressed_file=lowerCAmelCase__  )
	UpperCAmelCase_		:      Optional[int]       	=					cached_path(lowerCAmelCase__   ,download_config=lowerCAmelCase__  )
	with open(lowerCAmelCase__  ) as f:
		UpperCAmelCase_		:      Union[str, Any]       	=					f.read()
	with open(lowerCAmelCase__  ) as f:
		UpperCAmelCase_		:      str       	=					f.read()
	assert extracted_file_content == expected_file_content







@pytest.mark.parametrize('default_extracted'   ,[True, False]  )
@pytest.mark.parametrize('default_cache_dir'   ,[True, False]  )
def 						SCREAMING_SNAKE_CASE(  UpperCamelCase   ,UpperCamelCase   ,UpperCamelCase   ,UpperCamelCase   ,UpperCamelCase  )    ->    Optional[int]:
	UpperCAmelCase_		:      Tuple       	=					'custom_cache'
	UpperCAmelCase_		:      Optional[int]       	=					'custom_extracted_dir'
	UpperCAmelCase_		:      List[str]       	=					tmp_path / 'custom_extracted_path'
	if default_extracted:
		UpperCAmelCase_		:      Dict       	=					('downloads' if default_cache_dir else custom_cache_dir, 'extracted')
	else:
		monkeypatch.setattr('datasets.config.EXTRACTED_DATASETS_DIR'   ,lowerCAmelCase__  )
		monkeypatch.setattr('datasets.config.EXTRACTED_DATASETS_PATH'   ,str(lowerCAmelCase__  )  )
		UpperCAmelCase_		:      str       	=					custom_extracted_path.parts[-2:] if default_cache_dir else (custom_cache_dir, custom_extracted_dir)

	UpperCAmelCase_		:      Tuple       	=					xz_file
	UpperCAmelCase_		:      List[Any]       	=					(
	    DownloadConfig(extract_compressed_file=lowerCAmelCase__  )
	    if default_cache_dir
	    else DownloadConfig(cache_dir=tmp_path / custom_cache_dir   ,extract_compressed_file=lowerCAmelCase__  )
	)
	UpperCAmelCase_		:      Union[str, Any]       	=					cached_path(lowerCAmelCase__   ,download_config=lowerCAmelCase__  )
	assert Path(lowerCAmelCase__  ).parent.parts[-2:] == expected







def 						SCREAMING_SNAKE_CASE(  UpperCamelCase  )    ->    List[str]:
	# absolute path
	UpperCAmelCase_		:      Tuple       	=					str(Path(lowerCAmelCase__  ).resolve()  )
	assert cached_path(lowerCAmelCase__  ) == text_file
	# relative path
	UpperCAmelCase_		:      int       	=					str(Path(lowerCAmelCase__  ).resolve().relative_to(Path(os.getcwd()  )  )  )
	assert cached_path(lowerCAmelCase__  ) == text_file







def 						SCREAMING_SNAKE_CASE(  UpperCamelCase  )    ->    Optional[Any]:
	# absolute path
	UpperCAmelCase_		:      Optional[int]       	=					str(tmp_path.resolve() / '__missing_file__.txt'  )
	with pytest.raises(lowerCAmelCase__  ):
		cached_path(lowerCAmelCase__  )
	# relative path
	UpperCAmelCase_		:      Any       	=					'./__missing_file__.txt'
	with pytest.raises(lowerCAmelCase__  ):
		cached_path(lowerCAmelCase__  )







def 						SCREAMING_SNAKE_CASE(  UpperCamelCase  )    ->    List[Any]:
	UpperCAmelCase_		:      List[Any]       	=					get_from_cache(f"""tmp://{tmpfs_file}"""  )
	with open(lowerCAmelCase__  ) as f:
		UpperCAmelCase_		:      str       	=					f.read()
	assert output_file_content == FILE_CONTENT







@patch('datasets.config.HF_DATASETS_OFFLINE'   ,lowerCAmelCase__  )
def 						SCREAMING_SNAKE_CASE(  )    ->    Any:
	with pytest.raises(lowerCAmelCase__  ):
		cached_path('https://huggingface.co'  )







@patch('datasets.config.HF_DATASETS_OFFLINE'   ,lowerCAmelCase__  )
def 						SCREAMING_SNAKE_CASE(  UpperCamelCase  )    ->    str:
	UpperCAmelCase_		:      Union[str, Any]       	=					tmp_path_factory.mktemp('data'  ) / 'file.html'
	with pytest.raises(lowerCAmelCase__  ):
		http_get('https://huggingface.co'   ,temp_file=lowerCAmelCase__  )
	with pytest.raises(lowerCAmelCase__  ):
		http_head('https://huggingface.co'  )







@patch('datasets.config.HF_DATASETS_OFFLINE'   ,lowerCAmelCase__  )
def 						SCREAMING_SNAKE_CASE(  UpperCamelCase  )    ->    int:
	UpperCAmelCase_		:      Optional[Any]       	=					tmp_path_factory.mktemp('data'  ) / 'file.html'
	with pytest.raises(lowerCAmelCase__  ):
		ftp_get('ftp://huggingface.co'   ,temp_file=lowerCAmelCase__  )
	with pytest.raises(lowerCAmelCase__  ):
		ftp_head('ftp://huggingface.co'  )







@patch('datasets.config.HF_DATASETS_OFFLINE'   ,lowerCAmelCase__  )
def 						SCREAMING_SNAKE_CASE(  UpperCamelCase  )    ->    Tuple:
	UpperCAmelCase_		:      Optional[Any]       	=					tmp_path_factory.mktemp('data'  ) / 'file.html'
	with pytest.raises(lowerCAmelCase__  ):
		fsspec_get('s3://huggingface.co'   ,temp_file=lowerCAmelCase__  )
	with pytest.raises(lowerCAmelCase__  ):
		fsspec_head('s3://huggingface.co'  )









'''simple docstring'''


from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase__										= logging.get_logger(__name__)

lowerCAmelCase__										= {
    "tanreinama/GPTSAN-2.8B-spout_is_uniform": (
        "https://huggingface.co/tanreinama/GPTSAN-2.8B-spout_is_uniform/resolve/main/config.json"
    ),
}



class 					lowercase						(  a_ ):
							_lowerCamelCase		:      Any=   "gptsan-japanese"
							_lowerCamelCase		:      List[Any]=   [
							    "past_key_values",
							]
							_lowerCamelCase		:      Tuple=   {
							    "hidden_size": "d_model",
							    "num_attention_heads": "num_heads",
							    "num_hidden_layers": "num_layers",
							}


							def __init__(			self    ,			_snake_case=3_6000    ,			_snake_case=1280    ,			_snake_case=1024    ,			_snake_case=8192    ,			_snake_case=4096    ,			_snake_case=128    ,			_snake_case=10    ,			_snake_case=0    ,			_snake_case=16    ,			_snake_case=16    ,			_snake_case=128    ,			_snake_case=0.0    ,			_snake_case=1e-5    ,			_snake_case=False    ,			_snake_case=0.0    ,			_snake_case="float32"    ,			_snake_case=False    ,			_snake_case=False    ,			_snake_case=False    ,			_snake_case=0.002    ,			_snake_case=False    ,			_snake_case=True    ,			_snake_case=3_5998    ,			_snake_case=3_5995    ,			_snake_case=3_5999    ,			**_snake_case    ,			)							->				Optional[int]:
								UpperCAmelCase_		:      Dict       	=					vocab_size
								UpperCAmelCase_		:      Optional[int]       	=					max_position_embeddings
								UpperCAmelCase_		:      Any       	=					d_model
								UpperCAmelCase_		:      int       	=					d_ff
								UpperCAmelCase_		:      Tuple       	=					d_ext
								UpperCAmelCase_		:      Tuple       	=					d_spout
								UpperCAmelCase_		:      List[Any]       	=					num_switch_layers
								UpperCAmelCase_		:      Tuple       	=					num_ext_layers
								UpperCAmelCase_		:      Optional[int]       	=					num_switch_layers + num_ext_layers
								UpperCAmelCase_		:      Tuple       	=					num_heads
								UpperCAmelCase_		:      List[str]       	=					num_experts
								UpperCAmelCase_		:      List[str]       	=					expert_capacity
								UpperCAmelCase_		:      Optional[int]       	=					dropout_rate
								UpperCAmelCase_		:      List[Any]       	=					layer_norm_epsilon
								UpperCAmelCase_		:      Any       	=					router_bias
								UpperCAmelCase_		:      Optional[Any]       	=					router_jitter_noise
								UpperCAmelCase_		:      str       	=					router_dtype
								UpperCAmelCase_		:      List[str]       	=					router_ignore_padding_tokens
								UpperCAmelCase_		:      Optional[int]       	=					output_hidden_states
								UpperCAmelCase_		:      Union[str, Any]       	=					output_attentions
								UpperCAmelCase_		:      Optional[int]       	=					initializer_factor
								UpperCAmelCase_		:      Optional[int]       	=					output_router_logits
								UpperCAmelCase_		:      List[str]       	=					use_cache

								super().__init__(
								    separator_token_id=_snake_case    ,			pad_token_id=_snake_case    ,			eos_token_id=_snake_case    ,			**_snake_case    ,			)






