


"""simple docstring"""

import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import AutoFeatureExtractor, WavaVecaFeatureExtractor
from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402


UpperCAmelCase__  							=get_tests_dir("fixtures")





class    lowerCamelCase__						(		unittest.TestCase    ):
      def SCREAMING_SNAKE_CASE_	(		self	:	List[Any]							):






       '''simple docstring'''


       __lowercase									=	mock.Mock()
       __lowercase									=	5_0_0
       __lowercase									=	{}
       __lowercase									=	HTTPError
       __lowercase									=	{}

       # Download this model to make sure it's in the cache.
       __lowercase									=	WavaVecaFeatureExtractor.from_pretrained("""hf-internal-testing/tiny-random-wav2vec2"""							)
       # Under the mock environment we get a 500 error when trying to reach the model.
       with mock.patch("""requests.Session.request"""      ,       return_value=A_							) as mock_head:
        __lowercase									=	WavaVecaFeatureExtractor.from_pretrained("""hf-internal-testing/tiny-random-wav2vec2"""							)
        # This check we did call the fake head request
        mock_head.assert_called()






      def SCREAMING_SNAKE_CASE_	(		self	:	Tuple							):






       '''simple docstring'''


       __lowercase									=	WavaVecaFeatureExtractor.from_pretrained(
           """https://huggingface.co/hf-internal-testing/tiny-random-wav2vec2/resolve/main/preprocessor_config.json"""							)






@is_staging_test
class    lowerCamelCase__						(		unittest.TestCase    ):
      @classmethod
      def SCREAMING_SNAKE_CASE_	(		cls	:	Union[str, Any]							):






       '''simple docstring'''


       __lowercase									=	TOKEN
       HfFolder.save_token(A_							)
      @classmethod
      def SCREAMING_SNAKE_CASE_	(		cls	:	str							):






       '''simple docstring'''


       try:
        delete_repo(token=cls._token      ,       repo_id="""test-feature-extractor"""							)
       except HTTPError:
        pass

       try:
        delete_repo(token=cls._token      ,       repo_id="""valid_org/test-feature-extractor-org"""							)
       except HTTPError:
        pass

       try:
        delete_repo(token=cls._token      ,       repo_id="""test-dynamic-feature-extractor"""							)
       except HTTPError:
        pass
      def SCREAMING_SNAKE_CASE_	(		self	:	Tuple							):






       '''simple docstring'''


       __lowercase									=	WavaVecaFeatureExtractor.from_pretrained(A_							)
       feature_extractor.push_to_hub("""test-feature-extractor"""      ,       use_auth_token=self._token							)

       __lowercase									=	WavaVecaFeatureExtractor.from_pretrained(F'''{USER}/test-feature-extractor'''							)
       for k, v in feature_extractor.__dict__.items():
        self.assertEqual(A_      ,       getattr(A_      ,       A_							)							)

       # Reset repo
       delete_repo(token=self._token      ,       repo_id="""test-feature-extractor"""							)

       # Push to hub via save_pretrained
       with tempfile.TemporaryDirectory() as tmp_dir:
        feature_extractor.save_pretrained(
            A_      ,       repo_id="""test-feature-extractor"""      ,       push_to_hub=A_      ,       use_auth_token=self._token							)

       __lowercase									=	WavaVecaFeatureExtractor.from_pretrained(F'''{USER}/test-feature-extractor'''							)
       for k, v in feature_extractor.__dict__.items():
        self.assertEqual(A_      ,       getattr(A_      ,       A_							)							)
      def SCREAMING_SNAKE_CASE_	(		self	:	int							):






       '''simple docstring'''


       __lowercase									=	WavaVecaFeatureExtractor.from_pretrained(A_							)
       feature_extractor.push_to_hub("""valid_org/test-feature-extractor"""      ,       use_auth_token=self._token							)

       __lowercase									=	WavaVecaFeatureExtractor.from_pretrained("""valid_org/test-feature-extractor"""							)
       for k, v in feature_extractor.__dict__.items():
        self.assertEqual(A_      ,       getattr(A_      ,       A_							)							)

       # Reset repo
       delete_repo(token=self._token      ,       repo_id="""valid_org/test-feature-extractor"""							)

       # Push to hub via save_pretrained
       with tempfile.TemporaryDirectory() as tmp_dir:
        feature_extractor.save_pretrained(
            A_      ,       repo_id="""valid_org/test-feature-extractor-org"""      ,       push_to_hub=A_      ,       use_auth_token=self._token							)

       __lowercase									=	WavaVecaFeatureExtractor.from_pretrained("""valid_org/test-feature-extractor-org"""							)
       for k, v in feature_extractor.__dict__.items():
        self.assertEqual(A_      ,       getattr(A_      ,       A_							)							)






      def SCREAMING_SNAKE_CASE_	(		self	:	Tuple							):






       '''simple docstring'''


       CustomFeatureExtractor.register_for_auto_class()
       __lowercase									=	CustomFeatureExtractor.from_pretrained(A_							)

       feature_extractor.push_to_hub("""test-dynamic-feature-extractor"""      ,       use_auth_token=self._token							)

       # This has added the proper auto_map field to the config
       self.assertDictEqual(
           feature_extractor.auto_map      ,       {"""AutoFeatureExtractor""": """custom_feature_extraction.CustomFeatureExtractor"""}      ,       )

       __lowercase									=	AutoFeatureExtractor.from_pretrained(
           F'''{USER}/test-dynamic-feature-extractor'''      ,       trust_remote_code=A_							)
       # Can't make an isinstance check because the new_feature_extractor is from the CustomFeatureExtractor class of a dynamic module
       self.assertEqual(new_feature_extractor.__class__.__name__      ,       """CustomFeatureExtractor"""							)




"""simple docstring"""

import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.text import TextDatasetReader

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases


def lowerCAmelCase_      (				UpperCamelCase__	: List[str]				,   UpperCamelCase__	: Any ):






 """simple docstring"""







 assert isinstance(UpperCamelCase__				,   UpperCamelCase__ )
 assert dataset.num_rows == 4
 assert dataset.num_columns == 1
 assert dataset.column_names == ["text"]
 for feature, expected_dtype in expected_features.items():
  assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize("""keep_in_memory"""				,   [False, True] )
def lowerCAmelCase_      (				UpperCamelCase__	: Dict				,   UpperCamelCase__	: Optional[int]				,   UpperCamelCase__	: List[str] ):






 """simple docstring"""







 __lowercase									=	tmp_path / """cache"""
 __lowercase									=	{"""text""": """string"""}
 with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
  __lowercase									=	TextDatasetReader(UpperCamelCase__				,   cache_dir=UpperCamelCase__				,   keep_in_memory=UpperCamelCase__ ).read()
 _check_text_dataset(UpperCamelCase__				,   UpperCamelCase__ )


@pytest.mark.parametrize(
    """features"""				,   [
        None,
        {"""text""": """string"""},
        {"""text""": """int32"""},
        {"""text""": """float32"""},
    ]				,   )
def lowerCAmelCase_      (				UpperCamelCase__	: List[Any]				,   UpperCamelCase__	: List[str]				,   UpperCamelCase__	: Union[str, Any] ):






 """simple docstring"""







 __lowercase									=	tmp_path / """cache"""
 __lowercase									=	{"""text""": """string"""}
 __lowercase									=	features.copy() if features else default_expected_features
 __lowercase									=	(
     Features({feature: Value(UpperCamelCase__ ) for feature, dtype in features.items()} ) if features is not None else None
 )
 __lowercase									=	TextDatasetReader(UpperCamelCase__				,   features=UpperCamelCase__				,   cache_dir=UpperCamelCase__ ).read()
 _check_text_dataset(UpperCamelCase__				,   UpperCamelCase__ )


@pytest.mark.parametrize("""split"""				,   [None, NamedSplit("""train""" ), """train""", """test"""] )
def lowerCAmelCase_      (				UpperCamelCase__	: Union[str, Any]				,   UpperCamelCase__	: str				,   UpperCamelCase__	: Tuple ):






 """simple docstring"""







 __lowercase									=	tmp_path / """cache"""
 __lowercase									=	{"""text""": """string"""}
 __lowercase									=	TextDatasetReader(UpperCamelCase__				,   cache_dir=UpperCamelCase__				,   split=UpperCamelCase__ ).read()
 _check_text_dataset(UpperCamelCase__				,   UpperCamelCase__ )
 assert dataset.split == split if split else "train"


@pytest.mark.parametrize("""path_type"""				,   [str, list] )
def lowerCAmelCase_      (				UpperCamelCase__	: Union[str, Any]				,   UpperCamelCase__	: List[str]				,   UpperCamelCase__	: Any ):






 """simple docstring"""







 if issubclass(UpperCamelCase__				,   UpperCamelCase__ ):
  __lowercase									=	text_path
 elif issubclass(UpperCamelCase__				,   UpperCamelCase__ ):
  __lowercase									=	[text_path]
 __lowercase									=	tmp_path / """cache"""
 __lowercase									=	{"""text""": """string"""}
 __lowercase									=	TextDatasetReader(UpperCamelCase__				,   cache_dir=UpperCamelCase__ ).read()
 _check_text_dataset(UpperCamelCase__				,   UpperCamelCase__ )


def lowerCAmelCase_      (				UpperCamelCase__	: str				,   UpperCamelCase__	: str				,   UpperCamelCase__	: Any=("train",) ):






 """simple docstring"""







 assert isinstance(UpperCamelCase__				,   UpperCamelCase__ )
 for split in splits:
  __lowercase									=	dataset_dict[split]
  assert dataset.num_rows == 4
  assert dataset.num_columns == 1
  assert dataset.column_names == ["text"]
  for feature, expected_dtype in expected_features.items():
   assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize("""keep_in_memory"""				,   [False, True] )
def lowerCAmelCase_      (				UpperCamelCase__	: int				,   UpperCamelCase__	: List[str]				,   UpperCamelCase__	: List[Any] ):






 """simple docstring"""







 __lowercase									=	tmp_path / """cache"""
 __lowercase									=	{"""text""": """string"""}
 with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
  __lowercase									=	TextDatasetReader({"""train""": text_path}				,   cache_dir=UpperCamelCase__				,   keep_in_memory=UpperCamelCase__ ).read()
 _check_text_datasetdict(UpperCamelCase__				,   UpperCamelCase__ )


@pytest.mark.parametrize(
    """features"""				,   [
        None,
        {"""text""": """string"""},
        {"""text""": """int32"""},
        {"""text""": """float32"""},
    ]				,   )
def lowerCAmelCase_      (				UpperCamelCase__	: Optional[Any]				,   UpperCamelCase__	: Dict				,   UpperCamelCase__	: Union[str, Any] ):






 """simple docstring"""







 __lowercase									=	tmp_path / """cache"""
 # CSV file loses col_1 string dtype information: default now is "int64" instead of "string"
 __lowercase									=	{"""text""": """string"""}
 __lowercase									=	features.copy() if features else default_expected_features
 __lowercase									=	(
     Features({feature: Value(UpperCamelCase__ ) for feature, dtype in features.items()} ) if features is not None else None
 )
 __lowercase									=	TextDatasetReader({"""train""": text_path}				,   features=UpperCamelCase__				,   cache_dir=UpperCamelCase__ ).read()
 _check_text_datasetdict(UpperCamelCase__				,   UpperCamelCase__ )


@pytest.mark.parametrize("""split"""				,   [None, NamedSplit("""train""" ), """train""", """test"""] )
def lowerCAmelCase_      (				UpperCamelCase__	: Dict				,   UpperCamelCase__	: int				,   UpperCamelCase__	: int ):






 """simple docstring"""







 if split:
  __lowercase									=	{split: text_path}
 else:
  __lowercase									=	"""train"""
  __lowercase									=	{"""train""": text_path, """test""": text_path}
 __lowercase									=	tmp_path / """cache"""
 __lowercase									=	{"""text""": """string"""}
 __lowercase									=	TextDatasetReader(UpperCamelCase__				,   cache_dir=UpperCamelCase__ ).read()
 _check_text_datasetdict(UpperCamelCase__				,   UpperCamelCase__				,   splits=list(path.keys() ) )
 assert all(dataset[split].split == split for split in path.keys() )
