

# tests directory-specific settings - this file is run automatically
# by pytest before any tests are run

import doctest
import sys
import warnings
from os.path import abspath, dirname, join

import _pytest

from transformers.testing_utils import HfDoctestModule, HfDocTestParser


# allow having multiple repository checkouts and not needing to remember to rerun
# 'pip install -e .[dev]' when switching between checkouts and running tests.
A_ :				Optional[int]						 =    abspath(join(dirname(__file__), 'src'))
sys.path.insert(1, git_repo_path)

# silence FutureWarning warnings in tests since often we can't act on them until
# they become normal warnings - i.e. the tests still need to test the current functionality
warnings.simplefilter(action='ignore', category=FutureWarning)

def 		UpperCamelCase      (lowercase_:				List[str]      )  ->   Any:
	config.addinivalue_line(
	    """markers"""					,   """is_pt_tf_cross_test: mark test to run only when PT and TF interactions are tested"""      )
	config.addinivalue_line(
	    """markers"""					,   """is_pt_flax_cross_test: mark test to run only when PT and FLAX interactions are tested"""      )
	config.addinivalue_line("""markers"""					,   """is_pipeline_test: mark test to run only when pipelines are tested"""      )
	config.addinivalue_line("""markers"""					,   """is_staging_test: mark test to run only in the staging environment"""      )
	config.addinivalue_line("""markers"""					,   """accelerate_tests: mark test that require accelerate"""      )
	config.addinivalue_line("""markers"""					,   """tool_tests: mark the tool tests that are run on their specific schedule"""      )

def 		UpperCamelCase      (lowercase_:				Optional[int]      )  ->   Optional[Any]:
	from transformers.testing_utils import pytest_addoption_shared

	pytest_addoption_shared(lowercase_      )

def 		UpperCamelCase      (lowercase_:				List[str]      )  ->   Optional[Any]:
	from transformers.testing_utils import pytest_terminal_summary_main

	A__							:					List[Any]       =	terminalreporter.config.getoption("""--make-reports"""      )
	if make_reports:
		pytest_terminal_summary_main(lowercase_					,   id=lowercase_      )

def 		UpperCamelCase      (lowercase_:				Union[str, Any]					,   lowercase_:				int      )  ->   List[str]:
	# If no tests are collected, pytest exists with code 5, which makes the CI fail.
	if exitstatus == 5:
		A__							:					Tuple       =	0


# Doctest custom flag to ignore output.
A_ :				Tuple						 =    doctest.register_optionflag('IGNORE_RESULT')

A_ :				Dict						 =    doctest.OutputChecker



class 					_a		(__magic_name__       ):


					'''simple docstring'''






					def 						__A    (				self					, A__					, A__					, A__    ):
						if IGNORE_RESULT & optionflags:
							return True
						return OutputChecker.check_output(self					, A__					, A__					, A__    )



A_ :				str						 =    CustomOutputChecker
A_ :				Dict						 =    HfDoctestModule


A_ :				Optional[int]						 =    HfDocTestParser






import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam



class 					_a		(datasets.BeamBasedBuilder       ):


					'''simple docstring'''






					def 						__A    (				self    ):
						return datasets.DatasetInfo(
						    features=datasets.Features({"""content""": datasets.Value("""string"""    )}    )					, supervised_keys=A__					, )






					def 						__A    (				self					, A__					, A__    ):
						return [datasets.SplitGenerator(name=datasets.Split.TRAIN					, gen_kwargs={"""examples""": get_test_dummy_examples()}    )]






					def 						__A    (				self					, A__					, A__    ):
						import apache_beam as beam

						return pipeline | "Load Examples" >> beam.Create(A__    )



class 					_a		(datasets.BeamBasedBuilder       ):


					'''simple docstring'''






					def 						__A    (				self    ):
						return datasets.DatasetInfo(
						    features=datasets.Features({"""a""": datasets.Sequence({"""b""": datasets.Value("""string"""    )}    )}    )					, supervised_keys=A__					, )






					def 						__A    (				self					, A__					, A__    ):
						return [
						    datasets.SplitGenerator(name=datasets.Split.TRAIN					, gen_kwargs={"""examples""": get_test_nested_examples()}    )
						]






					def 						__A    (				self					, A__					, A__    ):
						import apache_beam as beam

						return pipeline | "Load Examples" >> beam.Create(A__    )



def 		UpperCamelCase      ()  ->   Dict:
	return [(i, {"content": content}) for i, content in enumerate(["""foo""", """bar""", """foobar"""]      )]

def 		UpperCamelCase      ()  ->   Tuple:
	return [(i, {"a": {"b": [content]}}) for i, content in enumerate(["""foo""", """bar""", """foobar"""]      )]



class 					_a		(__magic_name__       ):


					'''simple docstring'''






					@require_beam
					def 						__A    (				self    ):
						A__							:					Dict       =	len(get_test_dummy_examples()    )
						with tempfile.TemporaryDirectory() as tmp_cache_dir:
							A__							:					int       =	DummyBeamDataset(cache_dir=A__					, beam_runner="""DirectRunner"""    )
							builder.download_and_prepare()
							self.assertTrue(
							    os.path.exists(
							        os.path.join(A__					, builder.name					, """default"""					, """0.0.0"""					, F"""{builder.name}-train.arrow"""    )    )    )
							self.assertDictEqual(builder.info.features					, datasets.Features({"""content""": datasets.Value("""string"""    )}    )    )
							A__							:					int       =	builder.as_dataset()
							self.assertEqual(dset["""train"""].num_rows					, A__    )
							self.assertEqual(dset["""train"""].info.splits["""train"""].num_examples					, A__    )
							self.assertDictEqual(dset["""train"""][0]					, get_test_dummy_examples()[0][1]    )
							self.assertDictEqual(
							    dset["""train"""][expected_num_examples - 1]					, get_test_dummy_examples()[expected_num_examples - 1][1]    )
							self.assertTrue(
							    os.path.exists(os.path.join(A__					, builder.name					, """default"""					, """0.0.0"""					, """dataset_info.json"""    )    )    )
							del dset






					@require_beam
					def 						__A    (				self    ):
						import apache_beam as beam

						A__							:					int       =	beam.io.parquetio.WriteToParquet

						A__							:					List[str]       =	len(get_test_dummy_examples()    )
						with tempfile.TemporaryDirectory() as tmp_cache_dir:
							A__							:					str       =	DummyBeamDataset(cache_dir=A__					, beam_runner="""DirectRunner"""    )
							with patch("""apache_beam.io.parquetio.WriteToParquet"""    ) as write_parquet_mock:
								A__							:					Optional[Any]       =	partial(A__					, num_shards=2    )
								builder.download_and_prepare()
							self.assertTrue(
							    os.path.exists(
							        os.path.join(
							            A__					, builder.name					, """default"""					, """0.0.0"""					, F"""{builder.name}-train-00000-of-00002.arrow"""    )    )    )
							self.assertTrue(
							    os.path.exists(
							        os.path.join(
							            A__					, builder.name					, """default"""					, """0.0.0"""					, F"""{builder.name}-train-00000-of-00002.arrow"""    )    )    )
							self.assertDictEqual(builder.info.features					, datasets.Features({"""content""": datasets.Value("""string"""    )}    )    )
							A__							:					Optional[int]       =	builder.as_dataset()
							self.assertEqual(dset["""train"""].num_rows					, A__    )
							self.assertEqual(dset["""train"""].info.splits["""train"""].num_examples					, A__    )
							# Order is not preserved when sharding, so we just check that all the elements are there
							self.assertListEqual(sorted(dset["""train"""]["""content"""]    )					, sorted(["""foo""", """bar""", """foobar"""]    )    )
							self.assertTrue(
							    os.path.exists(os.path.join(A__					, builder.name					, """default"""					, """0.0.0"""					, """dataset_info.json"""    )    )    )
							del dset






					@require_beam
					def 						__A    (				self    ):
						with tempfile.TemporaryDirectory() as tmp_cache_dir:
							A__							:					int       =	DummyBeamDataset(cache_dir=A__    )
							self.assertRaises(datasets.builder.MissingBeamOptions					, builder.download_and_prepare    )






					@require_beam
					def 						__A    (				self    ):
						A__							:					List[Any]       =	len(get_test_nested_examples()    )
						with tempfile.TemporaryDirectory() as tmp_cache_dir:
							A__							:					Optional[int]       =	NestedBeamDataset(cache_dir=A__					, beam_runner="""DirectRunner"""    )
							builder.download_and_prepare()
							self.assertTrue(
							    os.path.exists(
							        os.path.join(A__					, builder.name					, """default"""					, """0.0.0"""					, F"""{builder.name}-train.arrow"""    )    )    )
							self.assertDictEqual(
							    builder.info.features					, datasets.Features({"""a""": datasets.Sequence({"""b""": datasets.Value("""string"""    )}    )}    )    )
							A__							:					Optional[int]       =	builder.as_dataset()
							self.assertEqual(dset["""train"""].num_rows					, A__    )
							self.assertEqual(dset["""train"""].info.splits["""train"""].num_examples					, A__    )
							self.assertDictEqual(dset["""train"""][0]					, get_test_nested_examples()[0][1]    )
							self.assertDictEqual(
							    dset["""train"""][expected_num_examples - 1]					, get_test_nested_examples()[expected_num_examples - 1][1]    )
							self.assertTrue(
							    os.path.exists(os.path.join(A__					, builder.name					, """default"""					, """0.0.0"""					, """dataset_info.json"""    )    )    )
							del dset



