





import tempfile
import unittest

import numpy as np
from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import BertConfig, is_flax_available
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_flax


if is_flax_available():
	import os

	from flax.core.frozen_dict import unfreeze
	from flax.traverse_util import flatten_dict

	from transformers import FlaxBertModel

	_lowerCamelCase   				=			'0.12'  # assumed parallelism: 8




@require_flax
@is_staging_test
class   UpperCamelCase_       (     unittest.TestCase    ):


							@classmethod
							def      _snake_case  (							cls    :Dict		)   ->   Optional[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      TOKEN
									HfFolder.save_token(__A		)


							@classmethod
							def      _snake_case  (							cls    :str		)   ->   List[Any]:



									"""simple docstring"""



									try:
											delete_repo(token=cls._token							,				repo_id="""test-model-flax"""		)
									except HTTPError:
											pass

									try:
											delete_repo(token=cls._token							,				repo_id="""valid_org/test-model-flax-org"""		)
									except HTTPError:
											pass


							def      _snake_case  (							self    :List[Any]		)   ->   Tuple:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      BertConfig(
									    vocab_size=99							,				hidden_size=32							,				num_hidden_layers=5							,				num_attention_heads=4							,				intermediate_size=37		)
									SCREAMING_SNAKE_CASE__			=      FlaxBertModel(__A		)
									model.push_to_hub("""test-model-flax"""							,				use_auth_token=self._token		)

									SCREAMING_SNAKE_CASE__			=      FlaxBertModel.from_pretrained(f'''{USER}/test-model-flax'''		)

									SCREAMING_SNAKE_CASE__			=      flatten_dict(unfreeze(model.params		)		)
									SCREAMING_SNAKE_CASE__			=      flatten_dict(unfreeze(new_model.params		)		)

									for key in base_params.keys():
											SCREAMING_SNAKE_CASE__			=      (base_params[key] - new_params[key]).sum().item()
											self.assertLessEqual(__A							,				1E-3							,				msg=f'''{key} not identical'''		)

									# Reset repo
									delete_repo(token=self._token							,				repo_id="""test-model-flax"""		)

									# Push to hub via save_pretrained
									with tempfile.TemporaryDirectory() as tmp_dir:
											model.save_pretrained(__A							,				repo_id="""test-model-flax"""							,				push_to_hub=__A							,				use_auth_token=self._token		)

									SCREAMING_SNAKE_CASE__			=      FlaxBertModel.from_pretrained(f'''{USER}/test-model-flax'''		)

									SCREAMING_SNAKE_CASE__			=      flatten_dict(unfreeze(model.params		)		)
									SCREAMING_SNAKE_CASE__			=      flatten_dict(unfreeze(new_model.params		)		)

									for key in base_params.keys():
											SCREAMING_SNAKE_CASE__			=      (base_params[key] - new_params[key]).sum().item()
											self.assertLessEqual(__A							,				1E-3							,				msg=f'''{key} not identical'''		)


							def      _snake_case  (							self    :Optional[Any]		)   ->   str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      BertConfig(
									    vocab_size=99							,				hidden_size=32							,				num_hidden_layers=5							,				num_attention_heads=4							,				intermediate_size=37		)
									SCREAMING_SNAKE_CASE__			=      FlaxBertModel(__A		)
									model.push_to_hub("""valid_org/test-model-flax-org"""							,				use_auth_token=self._token		)

									SCREAMING_SNAKE_CASE__			=      FlaxBertModel.from_pretrained("""valid_org/test-model-flax-org"""		)

									SCREAMING_SNAKE_CASE__			=      flatten_dict(unfreeze(model.params		)		)
									SCREAMING_SNAKE_CASE__			=      flatten_dict(unfreeze(new_model.params		)		)

									for key in base_params.keys():
											SCREAMING_SNAKE_CASE__			=      (base_params[key] - new_params[key]).sum().item()
											self.assertLessEqual(__A							,				1E-3							,				msg=f'''{key} not identical'''		)

									# Reset repo
									delete_repo(token=self._token							,				repo_id="""valid_org/test-model-flax-org"""		)

									# Push to hub via save_pretrained
									with tempfile.TemporaryDirectory() as tmp_dir:
											model.save_pretrained(
											    __A							,				repo_id="""valid_org/test-model-flax-org"""							,				push_to_hub=__A							,				use_auth_token=self._token		)

									SCREAMING_SNAKE_CASE__			=      FlaxBertModel.from_pretrained("""valid_org/test-model-flax-org"""		)

									SCREAMING_SNAKE_CASE__			=      flatten_dict(unfreeze(model.params		)		)
									SCREAMING_SNAKE_CASE__			=      flatten_dict(unfreeze(new_model.params		)		)

									for key in base_params.keys():
											SCREAMING_SNAKE_CASE__			=      (base_params[key] - new_params[key]).sum().item()
											self.assertLessEqual(__A							,				1E-3							,				msg=f'''{key} not identical'''		)






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							str					,   UpperCamelCase__:							Optional[Any]    ):
		SCREAMING_SNAKE_CASE__			=      True
		SCREAMING_SNAKE_CASE__			=      flatten_dict(modela.params    )
		SCREAMING_SNAKE_CASE__			=      flatten_dict(modela.params    )
		for key in flat_params_a.keys():
				if np.sum(np.abs(flat_params_a[key] - flat_params_a[key]    )    ) > 1e-4:
						SCREAMING_SNAKE_CASE__			=      False

		return models_are_equal




@require_flax
class   UpperCamelCase_       (     unittest.TestCase    ):


							def      _snake_case  (							self    :List[str]		)   ->   Optional[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      BertConfig.from_pretrained("""hf-internal-testing/tiny-bert-flax-only"""		)
									SCREAMING_SNAKE_CASE__			=      FlaxBertModel(__A		)

									SCREAMING_SNAKE_CASE__			=      """bert"""
									with tempfile.TemporaryDirectory() as tmp_dir:
											model.save_pretrained(os.path.join(__A							,				__A		)		)

											with self.assertRaises(__A		):
													SCREAMING_SNAKE_CASE__			=      FlaxBertModel.from_pretrained(__A		)

											SCREAMING_SNAKE_CASE__			=      FlaxBertModel.from_pretrained(__A							,				subfolder=__A		)

									self.assertTrue(check_models_equal(__A							,				__A		)		)


							def      _snake_case  (							self    :Union[str, Any]		)   ->   Dict:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      BertConfig.from_pretrained("""hf-internal-testing/tiny-bert-flax-only"""		)
									SCREAMING_SNAKE_CASE__			=      FlaxBertModel(__A		)

									SCREAMING_SNAKE_CASE__			=      """bert"""
									with tempfile.TemporaryDirectory() as tmp_dir:
											model.save_pretrained(os.path.join(__A							,				__A		)							,				max_shard_size="""10KB"""		)

											with self.assertRaises(__A		):
													SCREAMING_SNAKE_CASE__			=      FlaxBertModel.from_pretrained(__A		)

											SCREAMING_SNAKE_CASE__			=      FlaxBertModel.from_pretrained(__A							,				subfolder=__A		)

									self.assertTrue(check_models_equal(__A							,				__A		)		)


							def      _snake_case  (							self    :List[Any]		)   ->   str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      """bert"""
									SCREAMING_SNAKE_CASE__			=      """hf-internal-testing/tiny-random-bert-subfolder"""

									with self.assertRaises(__A		):
											SCREAMING_SNAKE_CASE__			=      FlaxBertModel.from_pretrained(__A		)

									SCREAMING_SNAKE_CASE__			=      FlaxBertModel.from_pretrained(__A							,				subfolder=__A		)

									self.assertIsNotNone(__A		)


							def      _snake_case  (							self    :List[str]		)   ->   Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      """bert"""
									SCREAMING_SNAKE_CASE__			=      """hf-internal-testing/tiny-random-bert-sharded-subfolder"""
									with self.assertRaises(__A		):
											SCREAMING_SNAKE_CASE__			=      FlaxBertModel.from_pretrained(__A		)

									SCREAMING_SNAKE_CASE__			=      FlaxBertModel.from_pretrained(__A							,				subfolder=__A		)

									self.assertIsNotNone(__A		)






import os
from pathlib import Path
from unittest.mock import patch

import pytest
import zstandard as zstd

from datasets.download.download_config import DownloadConfig
from datasets.utils.file_utils import (
    OfflineModeIsEnabled,
    cached_path,
    fsspec_get,
    fsspec_head,
    ftp_get,
    ftp_head,
    get_from_cache,
    http_get,
    http_head,
)


_lowerCamelCase   				=			'\\n    Text data.\n    Second line of data.'

_lowerCamelCase   				=			'file'






@pytest.fixture(scope="""session"""    )
def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							Any    ):
		SCREAMING_SNAKE_CASE__			=      tmp_path_factory.mktemp("""data"""    ) / (FILE_PATH + """.zstd""")
		SCREAMING_SNAKE_CASE__			=      bytes(UpperCamelCase__					,   """utf-8"""    )
		with zstd.open(UpperCamelCase__					,   """wb"""    ) as f:
				f.write(UpperCamelCase__    )
		return path






@pytest.fixture
def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							List[str]    ):
		with open(os.path.join(tmpfs.local_root_dir					,   UpperCamelCase__    )					,   """w"""    ) as f:
				f.write(UpperCamelCase__    )
		return FILE_PATH






@pytest.mark.parametrize("""compression_format"""					,   ["""gzip""", """xz""", """zstd"""]    )
def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							int					,   UpperCamelCase__:							Dict					,   UpperCamelCase__:							int					,   UpperCamelCase__:							str					,   UpperCamelCase__:							Optional[int]					,   UpperCamelCase__:							Optional[Any]    ):
		SCREAMING_SNAKE_CASE__			=      {"""gzip""": gz_file, """xz""": xz_file, """zstd""": zstd_path}
		SCREAMING_SNAKE_CASE__			=      input_paths[compression_format]
		SCREAMING_SNAKE_CASE__			=      tmp_path / """cache"""
		SCREAMING_SNAKE_CASE__			=      DownloadConfig(cache_dir=UpperCamelCase__					,   extract_compressed_file=UpperCamelCase__    )
		SCREAMING_SNAKE_CASE__			=      cached_path(UpperCamelCase__					,   download_config=UpperCamelCase__    )
		with open(UpperCamelCase__    ) as f:
				SCREAMING_SNAKE_CASE__			=      f.read()
		with open(UpperCamelCase__    ) as f:
				SCREAMING_SNAKE_CASE__			=      f.read()
		assert extracted_file_content == expected_file_content






@pytest.mark.parametrize("""default_extracted"""					,   [True, False]    )
@pytest.mark.parametrize("""default_cache_dir"""					,   [True, False]    )
def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							Tuple					,   UpperCamelCase__:							List[str]					,   UpperCamelCase__:							Optional[int]					,   UpperCamelCase__:							Any					,   UpperCamelCase__:							Union[str, Any]    ):
		SCREAMING_SNAKE_CASE__			=      """custom_cache"""
		SCREAMING_SNAKE_CASE__			=      """custom_extracted_dir"""
		SCREAMING_SNAKE_CASE__			=      tmp_path / """custom_extracted_path"""
		if default_extracted:
				SCREAMING_SNAKE_CASE__			=      ("""downloads""" if default_cache_dir else custom_cache_dir, """extracted""")
		else:
				monkeypatch.setattr("""datasets.config.EXTRACTED_DATASETS_DIR"""					,   UpperCamelCase__    )
				monkeypatch.setattr("""datasets.config.EXTRACTED_DATASETS_PATH"""					,   str(UpperCamelCase__    )    )
				SCREAMING_SNAKE_CASE__			=      custom_extracted_path.parts[-2:] if default_cache_dir else (custom_cache_dir, custom_extracted_dir)

		SCREAMING_SNAKE_CASE__			=      xz_file
		SCREAMING_SNAKE_CASE__			=      (
		    DownloadConfig(extract_compressed_file=UpperCamelCase__    )
		    if default_cache_dir
		    else DownloadConfig(cache_dir=tmp_path / custom_cache_dir					,   extract_compressed_file=UpperCamelCase__    )
		)
		SCREAMING_SNAKE_CASE__			=      cached_path(UpperCamelCase__					,   download_config=UpperCamelCase__    )
		assert Path(UpperCamelCase__    ).parent.parts[-2:] == expected






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							Optional[int]    ):
		# absolute path
		SCREAMING_SNAKE_CASE__			=      str(Path(UpperCamelCase__    ).resolve()    )
		assert cached_path(UpperCamelCase__    ) == text_file
		# relative path
		SCREAMING_SNAKE_CASE__			=      str(Path(UpperCamelCase__    ).resolve().relative_to(Path(os.getcwd()    )    )    )
		assert cached_path(UpperCamelCase__    ) == text_file






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							List[str]    ):
		# absolute path
		SCREAMING_SNAKE_CASE__			=      str(tmp_path.resolve() / """__missing_file__.txt"""    )
		with pytest.raises(UpperCamelCase__    ):
				cached_path(UpperCamelCase__    )
		# relative path
		SCREAMING_SNAKE_CASE__			=      """./__missing_file__.txt"""
		with pytest.raises(UpperCamelCase__    ):
				cached_path(UpperCamelCase__    )






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							List[str]    ):
		SCREAMING_SNAKE_CASE__			=      get_from_cache(f'''tmp://{tmpfs_file}'''    )
		with open(UpperCamelCase__    ) as f:
				SCREAMING_SNAKE_CASE__			=      f.read()
		assert output_file_content == FILE_CONTENT






@patch("""datasets.config.HF_DATASETS_OFFLINE"""					,   UpperCamelCase__    )
def 				SCREAMING_SNAKE_CASE__							(       ):
		with pytest.raises(UpperCamelCase__    ):
				cached_path("""https://huggingface.co"""    )






@patch("""datasets.config.HF_DATASETS_OFFLINE"""					,   UpperCamelCase__    )
def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							Optional[Any]    ):
		SCREAMING_SNAKE_CASE__			=      tmp_path_factory.mktemp("""data"""    ) / """file.html"""
		with pytest.raises(UpperCamelCase__    ):
				http_get("""https://huggingface.co"""					,   temp_file=UpperCamelCase__    )
		with pytest.raises(UpperCamelCase__    ):
				http_head("""https://huggingface.co"""    )






@patch("""datasets.config.HF_DATASETS_OFFLINE"""					,   UpperCamelCase__    )
def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							List[Any]    ):
		SCREAMING_SNAKE_CASE__			=      tmp_path_factory.mktemp("""data"""    ) / """file.html"""
		with pytest.raises(UpperCamelCase__    ):
				ftp_get("""ftp://huggingface.co"""					,   temp_file=UpperCamelCase__    )
		with pytest.raises(UpperCamelCase__    ):
				ftp_head("""ftp://huggingface.co"""    )






@patch("""datasets.config.HF_DATASETS_OFFLINE"""					,   UpperCamelCase__    )
def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							int    ):
		SCREAMING_SNAKE_CASE__			=      tmp_path_factory.mktemp("""data"""    ) / """file.html"""
		with pytest.raises(UpperCamelCase__    ):
				fsspec_get("""s3://huggingface.co"""					,   temp_file=UpperCamelCase__    )
		with pytest.raises(UpperCamelCase__    ):
				fsspec_head("""s3://huggingface.co"""    )