
import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				)     ->     int:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				10
   _SCREAMING_SNAKE_CASE      =				datasets.Features(
       {
           """tokens""": datasets.Sequence(datasets.Value("""string"""						)						),
           """labels""": datasets.Sequence(datasets.ClassLabel(names=["""negative""", """positive"""]						)						),
           """answers""": datasets.Sequence(
               {
                   """text""": datasets.Value("""string"""						),
                   """answer_start""": datasets.Value("""int32"""						),
               }						),
           """id""": datasets.Value("""int64"""						),
       }						)
   _SCREAMING_SNAKE_CASE      =				datasets.Dataset.from_dict(
       {
           """tokens""": [["""foo"""] * 5] * n,
           """labels""": [[1] * 5] * n,
           """answers""": [{"""answer_start""": [97], """text""": ["""1976"""]}] * 10,
           """id""": list(range(SCREAMING_SNAKE_CASE_						)						),
       }    ,      features=SCREAMING_SNAKE_CASE_    ,      )
   return dataset
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)     ->     Optional[int]:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				str(tmp_path_factory.mktemp("""data"""						) / """file.arrow"""						)
   dataset.map(cache_file_name=SCREAMING_SNAKE_CASE_						)
   return filename


# FILE_CONTENT + files


UpperCamelCase__      :		Tuple       							=  """\
    Text data.
    Second line of data."""
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_						)     ->     List[Any]:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				tmp_path_factory.mktemp("""data"""						) / """file.txt"""
   _SCREAMING_SNAKE_CASE      =				FILE_CONTENT
   with open(SCREAMING_SNAKE_CASE_    ,      """w"""						) as f:
      f.write(SCREAMING_SNAKE_CASE_						)
   return filename
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_						)     ->     Optional[Any]:




   """simple docstring"""
   import bza

   _SCREAMING_SNAKE_CASE      =				tmp_path_factory.mktemp("""data"""						) / """file.txt.bz2"""
   _SCREAMING_SNAKE_CASE      =				bytes(SCREAMING_SNAKE_CASE_    ,      """utf-8"""						)
   with bza.open(SCREAMING_SNAKE_CASE_    ,      """wb"""						) as f:
      f.write(SCREAMING_SNAKE_CASE_						)
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_						)     ->     Tuple:




   """simple docstring"""
   import gzip

   _SCREAMING_SNAKE_CASE      =				str(tmp_path_factory.mktemp("""data"""						) / """file.txt.gz"""						)
   _SCREAMING_SNAKE_CASE      =				bytes(SCREAMING_SNAKE_CASE_    ,      """utf-8"""						)
   with gzip.open(SCREAMING_SNAKE_CASE_    ,      """wb"""						) as f:
      f.write(SCREAMING_SNAKE_CASE_						)
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_						)     ->     Union[str, Any]:




   """simple docstring"""

   if datasets.config.LZ4_AVAILABLE:
      import lza.frame

      _SCREAMING_SNAKE_CASE      =				tmp_path_factory.mktemp("""data"""						) / """file.txt.lz4"""
      _SCREAMING_SNAKE_CASE      =				bytes(SCREAMING_SNAKE_CASE_    ,      """utf-8"""						)
      with lza.frame.open(SCREAMING_SNAKE_CASE_    ,      """wb"""						) as f:
         f.write(SCREAMING_SNAKE_CASE_						)
      return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)     ->     Tuple:




   """simple docstring"""

   if datasets.config.PY7ZR_AVAILABLE:
      import pyazr

      _SCREAMING_SNAKE_CASE      =				tmp_path_factory.mktemp("""data"""						) / """file.txt.7z"""
      with pyazr.SevenZipFile(SCREAMING_SNAKE_CASE_    ,      """w"""						) as archive:
         archive.write(SCREAMING_SNAKE_CASE_    ,      arcname=os.path.basename(SCREAMING_SNAKE_CASE_						)						)
      return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)     ->     int:




   """simple docstring"""
   import tarfile

   _SCREAMING_SNAKE_CASE      =				tmp_path_factory.mktemp("""data"""						) / """file.txt.tar"""
   with tarfile.TarFile(SCREAMING_SNAKE_CASE_    ,      """w"""						) as f:
      f.add(SCREAMING_SNAKE_CASE_    ,      arcname=os.path.basename(SCREAMING_SNAKE_CASE_						)						)
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_						)     ->     str:




   """simple docstring"""
   import lzma

   _SCREAMING_SNAKE_CASE      =				tmp_path_factory.mktemp("""data"""						) / """file.txt.xz"""
   _SCREAMING_SNAKE_CASE      =				bytes(SCREAMING_SNAKE_CASE_    ,      """utf-8"""						)
   with lzma.open(SCREAMING_SNAKE_CASE_    ,      """wb"""						) as f:
      f.write(SCREAMING_SNAKE_CASE_						)
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)     ->     int:




   """simple docstring"""
   import zipfile

   _SCREAMING_SNAKE_CASE      =				tmp_path_factory.mktemp("""data"""						) / """file.txt.zip"""
   with zipfile.ZipFile(SCREAMING_SNAKE_CASE_    ,      """w"""						) as f:
      f.write(SCREAMING_SNAKE_CASE_    ,      arcname=os.path.basename(SCREAMING_SNAKE_CASE_						)						)
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_						)     ->     Optional[Any]:




   """simple docstring"""

   if datasets.config.ZSTANDARD_AVAILABLE:
      import zstandard as zstd

      _SCREAMING_SNAKE_CASE      =				tmp_path_factory.mktemp("""data"""						) / """file.txt.zst"""
      _SCREAMING_SNAKE_CASE      =				bytes(SCREAMING_SNAKE_CASE_    ,      """utf-8"""						)
      with zstd.open(SCREAMING_SNAKE_CASE_    ,      """wb"""						) as f:
         f.write(SCREAMING_SNAKE_CASE_						)
      return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_						)     ->     Dict:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				tmp_path_factory.mktemp("""data"""						) / """file.xml"""
   _SCREAMING_SNAKE_CASE      =				textwrap.dedent(
       """\\n    <?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n    <tmx version=\"1.4\">\n      <header segtype=\"sentence\" srclang=\"ca\" />\n      <body>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 1</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 1</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 2</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 2</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 3</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 3</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 4</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 4</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 5</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 5</seg></tuv>\n        </tu>\n      </body>\n    </tmx>"""						)
   with open(SCREAMING_SNAKE_CASE_    ,      """w"""						) as f:
      f.write(SCREAMING_SNAKE_CASE_						)
   return filename


UpperCamelCase__      :		Tuple       							=  [
    {"""col_1""": """0""", """col_2""": 0, """col_3""": 0.0},
    {"""col_1""": """1""", """col_2""": 1, """col_3""": 1.0},
    {"""col_1""": """2""", """col_2""": 2, """col_3""": 2.0},
    {"""col_1""": """3""", """col_2""": 3, """col_3""": 3.0},
]
UpperCamelCase__      :		Optional[Any]       							=  [
    {"""col_1""": """4""", """col_2""": 4, """col_3""": 4.0},
    {"""col_1""": """5""", """col_2""": 5, """col_3""": 5.0},
]
UpperCamelCase__      :		Any       							=  {
    """col_1""": ["""0""", """1""", """2""", """3"""],
    """col_2""": [0, 1, 2, 3],
    """col_3""": [0.0, 1.0, 2.0, 3.0],
}

UpperCamelCase__      :		List[Any]       							=  [
    {"""col_3""": 0.0, """col_1""": """0""", """col_2""": 0},
    {"""col_3""": 1.0, """col_1""": """1""", """col_2""": 1},
]

UpperCamelCase__      :		List[str]       							=  [
    {"""col_1""": """s0""", """col_2""": 0, """col_3""": 0.0},
    {"""col_1""": """s1""", """col_2""": 1, """col_3""": 1.0},
    {"""col_1""": """s2""", """col_2""": 2, """col_3""": 2.0},
    {"""col_1""": """s3""", """col_2""": 3, """col_3""": 3.0},
]
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				)     ->     int:




   """simple docstring"""
   return DATA_DICT_OF_LISTS
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_						)     ->     Optional[int]:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				datasets.Dataset.from_dict(SCREAMING_SNAKE_CASE_						)
   _SCREAMING_SNAKE_CASE      =				str(tmp_path_factory.mktemp("""data"""						) / """dataset.arrow"""						)
   dataset.map(cache_file_name=SCREAMING_SNAKE_CASE_						)
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_						)     ->     Optional[int]:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				str(tmp_path_factory.mktemp("""data"""						) / """dataset.sqlite"""						)
   with contextlib.closing(sqlitea.connect(SCREAMING_SNAKE_CASE_						)						) as con:
      _SCREAMING_SNAKE_CASE      =				con.cursor()
      cur.execute("""CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)"""						)
      for item in DATA:
         cur.execute("""INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)"""    ,      tuple(item.values()						)						)
      con.commit()
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_						)     ->     Dict:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				str(tmp_path_factory.mktemp("""data"""						) / """dataset.csv"""						)
   with open(SCREAMING_SNAKE_CASE_    ,      """w"""    ,      newline=""""""						) as f:
      _SCREAMING_SNAKE_CASE      =				csv.DictWriter(SCREAMING_SNAKE_CASE_    ,      fieldnames=["""col_1""", """col_2""", """col_3"""]						)
      writer.writeheader()
      for item in DATA:
         writer.writerow(SCREAMING_SNAKE_CASE_						)
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_						)     ->     Any:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				str(tmp_path_factory.mktemp("""data"""						) / """dataset2.csv"""						)
   with open(SCREAMING_SNAKE_CASE_    ,      """w"""    ,      newline=""""""						) as f:
      _SCREAMING_SNAKE_CASE      =				csv.DictWriter(SCREAMING_SNAKE_CASE_    ,      fieldnames=["""col_1""", """col_2""", """col_3"""]						)
      writer.writeheader()
      for item in DATA:
         writer.writerow(SCREAMING_SNAKE_CASE_						)
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)     ->     Tuple:




   """simple docstring"""
   import bza

   _SCREAMING_SNAKE_CASE      =				tmp_path_factory.mktemp("""data"""						) / """dataset.csv.bz2"""
   with open(SCREAMING_SNAKE_CASE_    ,      """rb"""						) as f:
      _SCREAMING_SNAKE_CASE      =				f.read()
   # data = bytes(FILE_CONTENT, "utf-8")
   with bza.open(SCREAMING_SNAKE_CASE_    ,      """wb"""						) as f:
      f.write(SCREAMING_SNAKE_CASE_						)
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)     ->     Any:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				tmp_path_factory.mktemp("""data"""						) / """dataset.csv.zip"""
   with zipfile.ZipFile(SCREAMING_SNAKE_CASE_    ,      """w"""						) as f:
      f.write(SCREAMING_SNAKE_CASE_    ,      arcname=os.path.basename(SCREAMING_SNAKE_CASE_						)						)
      f.write(SCREAMING_SNAKE_CASE_    ,      arcname=os.path.basename(SCREAMING_SNAKE_CASE_						)						)
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)     ->     str:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				tmp_path_factory.mktemp("""data"""						) / """dataset.csv.zip"""
   with zipfile.ZipFile(SCREAMING_SNAKE_CASE_    ,      """w"""						) as f:
      f.write(SCREAMING_SNAKE_CASE_    ,      arcname=os.path.basename(csv_path.replace(""".csv"""    ,      """.CSV"""						)						)						)
      f.write(SCREAMING_SNAKE_CASE_    ,      arcname=os.path.basename(csva_path.replace(""".csv"""    ,      """.CSV"""						)						)						)
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)     ->     Union[str, Any]:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				tmp_path_factory.mktemp("""data"""						) / """dataset_with_dir.csv.zip"""
   with zipfile.ZipFile(SCREAMING_SNAKE_CASE_    ,      """w"""						) as f:
      f.write(SCREAMING_SNAKE_CASE_    ,      arcname=os.path.join("""main_dir"""    ,      os.path.basename(SCREAMING_SNAKE_CASE_						)						)						)
      f.write(SCREAMING_SNAKE_CASE_    ,      arcname=os.path.join("""main_dir"""    ,      os.path.basename(SCREAMING_SNAKE_CASE_						)						)						)
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_						)     ->     Dict:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				str(tmp_path_factory.mktemp("""data"""						) / """dataset.parquet"""						)
   _SCREAMING_SNAKE_CASE      =				pa.schema(
       {
           """col_1""": pa.string(),
           """col_2""": pa.intaa(),
           """col_3""": pa.floataa(),
       }						)
   with open(SCREAMING_SNAKE_CASE_    ,      """wb"""						) as f:
      _SCREAMING_SNAKE_CASE      =				pq.ParquetWriter(SCREAMING_SNAKE_CASE_    ,      schema=SCREAMING_SNAKE_CASE_						)
      _SCREAMING_SNAKE_CASE      =				pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(SCREAMING_SNAKE_CASE_						)						)] for k in DATA[0]}    ,      schema=SCREAMING_SNAKE_CASE_						)
      writer.write_table(SCREAMING_SNAKE_CASE_						)
      writer.close()
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_						)     ->     Optional[int]:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				str(tmp_path_factory.mktemp("""data"""						) / """dataset.json"""						)
   _SCREAMING_SNAKE_CASE      =				{"""data""": DATA}
   with open(SCREAMING_SNAKE_CASE_    ,      """w"""						) as f:
      json.dump(SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_						)     ->     Optional[int]:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				str(tmp_path_factory.mktemp("""data"""						) / """dataset.json"""						)
   _SCREAMING_SNAKE_CASE      =				{"""data""": DATA_DICT_OF_LISTS}
   with open(SCREAMING_SNAKE_CASE_    ,      """w"""						) as f:
      json.dump(SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_						)     ->     Tuple:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				str(tmp_path_factory.mktemp("""data"""						) / """dataset.jsonl"""						)
   with open(SCREAMING_SNAKE_CASE_    ,      """w"""						) as f:
      for item in DATA:
         f.write(json.dumps(SCREAMING_SNAKE_CASE_						) + """\n"""						)
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_						)     ->     Optional[Any]:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				str(tmp_path_factory.mktemp("""data"""						) / """dataset2.jsonl"""						)
   with open(SCREAMING_SNAKE_CASE_    ,      """w"""						) as f:
      for item in DATA:
         f.write(json.dumps(SCREAMING_SNAKE_CASE_						) + """\n"""						)
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_						)     ->     List[str]:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				str(tmp_path_factory.mktemp("""data"""						) / """dataset_312.jsonl"""						)
   with open(SCREAMING_SNAKE_CASE_    ,      """w"""						) as f:
      for item in DATA_312:
         f.write(json.dumps(SCREAMING_SNAKE_CASE_						) + """\n"""						)
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_						)     ->     Any:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				str(tmp_path_factory.mktemp("""data"""						) / """dataset-str.jsonl"""						)
   with open(SCREAMING_SNAKE_CASE_    ,      """w"""						) as f:
      for item in DATA_STR:
         f.write(json.dumps(SCREAMING_SNAKE_CASE_						) + """\n"""						)
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)     ->     Optional[int]:




   """simple docstring"""
   import gzip

   _SCREAMING_SNAKE_CASE      =				str(tmp_path_factory.mktemp("""data"""						) / """dataset.txt.gz"""						)
   with open(SCREAMING_SNAKE_CASE_    ,      """rb"""						) as orig_file:
      with gzip.open(SCREAMING_SNAKE_CASE_    ,      """wb"""						) as zipped_file:
         zipped_file.writelines(SCREAMING_SNAKE_CASE_						)
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)     ->     Dict:




   """simple docstring"""
   import gzip

   _SCREAMING_SNAKE_CASE      =				str(tmp_path_factory.mktemp("""data"""						) / """dataset.jsonl.gz"""						)
   with open(SCREAMING_SNAKE_CASE_    ,      """rb"""						) as orig_file:
      with gzip.open(SCREAMING_SNAKE_CASE_    ,      """wb"""						) as zipped_file:
         zipped_file.writelines(SCREAMING_SNAKE_CASE_						)
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)     ->     Union[str, Any]:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				tmp_path_factory.mktemp("""data"""						) / """dataset.jsonl.zip"""
   with zipfile.ZipFile(SCREAMING_SNAKE_CASE_    ,      """w"""						) as f:
      f.write(SCREAMING_SNAKE_CASE_    ,      arcname=os.path.basename(SCREAMING_SNAKE_CASE_						)						)
      f.write(SCREAMING_SNAKE_CASE_    ,      arcname=os.path.basename(SCREAMING_SNAKE_CASE_						)						)
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)     ->     List[str]:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				tmp_path_factory.mktemp("""data"""						) / """dataset_nested.jsonl.zip"""
   with zipfile.ZipFile(SCREAMING_SNAKE_CASE_    ,      """w"""						) as f:
      f.write(SCREAMING_SNAKE_CASE_    ,      arcname=os.path.join("""nested"""    ,      os.path.basename(SCREAMING_SNAKE_CASE_						)						)						)
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)     ->     List[str]:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				tmp_path_factory.mktemp("""data"""						) / """dataset_with_dir.jsonl.zip"""
   with zipfile.ZipFile(SCREAMING_SNAKE_CASE_    ,      """w"""						) as f:
      f.write(SCREAMING_SNAKE_CASE_    ,      arcname=os.path.join("""main_dir"""    ,      os.path.basename(SCREAMING_SNAKE_CASE_						)						)						)
      f.write(SCREAMING_SNAKE_CASE_    ,      arcname=os.path.join("""main_dir"""    ,      os.path.basename(SCREAMING_SNAKE_CASE_						)						)						)
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)     ->     Optional[int]:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				tmp_path_factory.mktemp("""data"""						) / """dataset.jsonl.tar"""
   with tarfile.TarFile(SCREAMING_SNAKE_CASE_    ,      """w"""						) as f:
      f.add(SCREAMING_SNAKE_CASE_    ,      arcname=os.path.basename(SCREAMING_SNAKE_CASE_						)						)
      f.add(SCREAMING_SNAKE_CASE_    ,      arcname=os.path.basename(SCREAMING_SNAKE_CASE_						)						)
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)     ->     Optional[int]:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				tmp_path_factory.mktemp("""data"""						) / """dataset_nested.jsonl.tar"""
   with tarfile.TarFile(SCREAMING_SNAKE_CASE_    ,      """w"""						) as f:
      f.add(SCREAMING_SNAKE_CASE_    ,      arcname=os.path.join("""nested"""    ,      os.path.basename(SCREAMING_SNAKE_CASE_						)						)						)
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_						)     ->     Any:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				["""0""", """1""", """2""", """3"""]
   _SCREAMING_SNAKE_CASE      =				str(tmp_path_factory.mktemp("""data"""						) / """dataset.txt"""						)
   with open(SCREAMING_SNAKE_CASE_    ,      """w"""						) as f:
      for item in data:
         f.write(item + """\n"""						)
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_						)     ->     Dict:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				["""0""", """1""", """2""", """3"""]
   _SCREAMING_SNAKE_CASE      =				str(tmp_path_factory.mktemp("""data"""						) / """dataset2.txt"""						)
   with open(SCREAMING_SNAKE_CASE_    ,      """w"""						) as f:
      for item in data:
         f.write(item + """\n"""						)
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_						)     ->     List[str]:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				["""0""", """1""", """2""", """3"""]
   _SCREAMING_SNAKE_CASE      =				tmp_path_factory.mktemp("""data"""						) / """dataset.abc"""
   with open(SCREAMING_SNAKE_CASE_    ,      """w"""						) as f:
      for item in data:
         f.write(item + """\n"""						)
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)     ->     Tuple:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				tmp_path_factory.mktemp("""data"""						) / """dataset.text.zip"""
   with zipfile.ZipFile(SCREAMING_SNAKE_CASE_    ,      """w"""						) as f:
      f.write(SCREAMING_SNAKE_CASE_    ,      arcname=os.path.basename(SCREAMING_SNAKE_CASE_						)						)
      f.write(SCREAMING_SNAKE_CASE_    ,      arcname=os.path.basename(SCREAMING_SNAKE_CASE_						)						)
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)     ->     Dict:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				tmp_path_factory.mktemp("""data"""						) / """dataset_with_dir.text.zip"""
   with zipfile.ZipFile(SCREAMING_SNAKE_CASE_    ,      """w"""						) as f:
      f.write(SCREAMING_SNAKE_CASE_    ,      arcname=os.path.join("""main_dir"""    ,      os.path.basename(SCREAMING_SNAKE_CASE_						)						)						)
      f.write(SCREAMING_SNAKE_CASE_    ,      arcname=os.path.join("""main_dir"""    ,      os.path.basename(SCREAMING_SNAKE_CASE_						)						)						)
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)     ->     Tuple:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				tmp_path_factory.mktemp("""data"""						) / """dataset.ext.zip"""
   with zipfile.ZipFile(SCREAMING_SNAKE_CASE_    ,      """w"""						) as f:
      f.write(SCREAMING_SNAKE_CASE_    ,      arcname=os.path.basename("""unsupported.ext"""						)						)
      f.write(SCREAMING_SNAKE_CASE_    ,      arcname=os.path.basename("""unsupported_2.ext"""						)						)
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_						)     ->     List[Any]:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				"""\n""".join(["""First""", """Second\u2029with Unicode new line""", """Third"""]						)
   _SCREAMING_SNAKE_CASE      =				str(tmp_path_factory.mktemp("""data"""						) / """dataset_with_unicode_new_lines.txt"""						)
   with open(SCREAMING_SNAKE_CASE_    ,      """w"""    ,      encoding="""utf-8"""						) as f:
      f.write(SCREAMING_SNAKE_CASE_						)
   return path
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				)     ->     Tuple:




   """simple docstring"""
   return os.path.join("""tests"""    ,      """features"""    ,      """data"""    ,      """test_image_rgb.jpg"""						)
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				)     ->     Any:




   """simple docstring"""
   return os.path.join("""tests"""    ,      """features"""    ,      """data"""    ,      """test_audio_44100.wav"""						)
@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)     ->     Dict:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				tmp_path_factory.mktemp("""data"""						) / """dataset.img.zip"""
   with zipfile.ZipFile(SCREAMING_SNAKE_CASE_    ,      """w"""						) as f:
      f.write(SCREAMING_SNAKE_CASE_    ,      arcname=os.path.basename(SCREAMING_SNAKE_CASE_						)						)
      f.write(SCREAMING_SNAKE_CASE_    ,      arcname=os.path.basename(SCREAMING_SNAKE_CASE_						).replace(""".jpg"""    ,      """2.jpg"""						)						)
   return path







@pytest.fixture(scope="""session"""						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_						)     ->     List[Any]:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				tmp_path_factory.mktemp("""data_dir"""						)

   (data_dir / "subdir").mkdir()
   with open(data_dir / """subdir""" / """train.txt"""    ,      """w"""						) as f:
      f.write("""foo\n""" * 10						)
   with open(data_dir / """subdir""" / """test.txt"""    ,      """w"""						) as f:
      f.write("""bar\n""" * 10						)
   # hidden file
   with open(data_dir / """subdir""" / """.test.txt"""    ,      """w"""						) as f:
      f.write("""bar\n""" * 10						)

   # hidden directory
   (data_dir / ".subdir").mkdir()
   with open(data_dir / """.subdir""" / """train.txt"""    ,      """w"""						) as f:
      f.write("""foo\n""" * 10						)
   with open(data_dir / """.subdir""" / """test.txt"""    ,      """w"""						) as f:
      f.write("""bar\n""" * 10						)

   return data_dir








'''simple docstring'''



import pyarrow.parquet as pq
import pytest

from datasets import Audio, Dataset, DatasetDict, Features, NamedSplit, Sequence, Value, config
from datasets.features.image import Image
from datasets.io.parquet import ParquetDatasetReader, ParquetDatasetWriter, get_writer_batch_size

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)     ->     Optional[int]:




   """simple docstring"""
   assert isinstance(SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)
   assert dataset.num_rows == 4
   assert dataset.num_columns == 3
   assert dataset.column_names == ["col_1", "col_2", "col_3"]
   for feature, expected_dtype in expected_features.items():
      assert dataset.features[feature].dtype == expected_dtype
@pytest.mark.parametrize("""keep_in_memory"""    ,      [False, True]						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)     ->     List[Any]:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				tmp_path / """cache"""
   _SCREAMING_SNAKE_CASE      =				{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
   with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
      _SCREAMING_SNAKE_CASE      =				ParquetDatasetReader(SCREAMING_SNAKE_CASE_    ,      cache_dir=SCREAMING_SNAKE_CASE_    ,      keep_in_memory=SCREAMING_SNAKE_CASE_						).read()
   _check_parquet_dataset(SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)
@pytest.mark.parametrize(
    """features"""    ,      [
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]    ,      )
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)     ->     Optional[int]:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				tmp_path / """cache"""
   _SCREAMING_SNAKE_CASE      =				{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
   _SCREAMING_SNAKE_CASE      =				features.copy() if features else default_expected_features
   _SCREAMING_SNAKE_CASE      =				(
       Features({feature: Value(SCREAMING_SNAKE_CASE_						) for feature, dtype in features.items()}						) if features is not None else None
   )
   _SCREAMING_SNAKE_CASE      =				ParquetDatasetReader(SCREAMING_SNAKE_CASE_    ,      features=SCREAMING_SNAKE_CASE_    ,      cache_dir=SCREAMING_SNAKE_CASE_						).read()
   _check_parquet_dataset(SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)
@pytest.mark.parametrize("""split"""    ,      [None, NamedSplit("""train"""						), """train""", """test"""]						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)     ->     List[str]:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				tmp_path / """cache"""
   _SCREAMING_SNAKE_CASE      =				{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
   _SCREAMING_SNAKE_CASE      =				ParquetDatasetReader(SCREAMING_SNAKE_CASE_    ,      cache_dir=SCREAMING_SNAKE_CASE_    ,      split=SCREAMING_SNAKE_CASE_						).read()
   _check_parquet_dataset(SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)
   assert dataset.split == split if split else "train"
@pytest.mark.parametrize("""path_type"""    ,      [str, list]						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)     ->     Optional[int]:




   """simple docstring"""
   if issubclass(SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						):
      _SCREAMING_SNAKE_CASE      =				parquet_path
   elif issubclass(SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						):
      _SCREAMING_SNAKE_CASE      =				[parquet_path]
   _SCREAMING_SNAKE_CASE      =				tmp_path / """cache"""
   _SCREAMING_SNAKE_CASE      =				{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
   _SCREAMING_SNAKE_CASE      =				ParquetDatasetReader(SCREAMING_SNAKE_CASE_    ,      cache_dir=SCREAMING_SNAKE_CASE_						).read()
   _check_parquet_dataset(SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_=("train",)						)     ->     List[str]:




   """simple docstring"""
   assert isinstance(SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)
   for split in splits:
      _SCREAMING_SNAKE_CASE      =				dataset_dict[split]
      assert dataset.num_rows == 4
      assert dataset.num_columns == 3
      assert dataset.column_names == ["col_1", "col_2", "col_3"]
      for feature, expected_dtype in expected_features.items():
         assert dataset.features[feature].dtype == expected_dtype
@pytest.mark.parametrize("""keep_in_memory"""    ,      [False, True]						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)     ->     List[Any]:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				tmp_path / """cache"""
   _SCREAMING_SNAKE_CASE      =				{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
   with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
      _SCREAMING_SNAKE_CASE      =				ParquetDatasetReader(
          {"""train""": parquet_path}    ,      cache_dir=SCREAMING_SNAKE_CASE_    ,      keep_in_memory=SCREAMING_SNAKE_CASE_						).read()
   _check_parquet_datasetdict(SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)
@pytest.mark.parametrize(
    """features"""    ,      [
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]    ,      )
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)     ->     Optional[Any]:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				tmp_path / """cache"""
   _SCREAMING_SNAKE_CASE      =				{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
   _SCREAMING_SNAKE_CASE      =				features.copy() if features else default_expected_features
   _SCREAMING_SNAKE_CASE      =				(
       Features({feature: Value(SCREAMING_SNAKE_CASE_						) for feature, dtype in features.items()}						) if features is not None else None
   )
   _SCREAMING_SNAKE_CASE      =				ParquetDatasetReader({"""train""": parquet_path}    ,      features=SCREAMING_SNAKE_CASE_    ,      cache_dir=SCREAMING_SNAKE_CASE_						).read()
   _check_parquet_datasetdict(SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)
@pytest.mark.parametrize("""split"""    ,      [None, NamedSplit("""train"""						), """train""", """test"""]						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)     ->     Optional[Any]:




   """simple docstring"""
   if split:
      _SCREAMING_SNAKE_CASE      =				{split: parquet_path}
   else:
      _SCREAMING_SNAKE_CASE      =				"""train"""
      _SCREAMING_SNAKE_CASE      =				{"""train""": parquet_path, """test""": parquet_path}
   _SCREAMING_SNAKE_CASE      =				tmp_path / """cache"""
   _SCREAMING_SNAKE_CASE      =				{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
   _SCREAMING_SNAKE_CASE      =				ParquetDatasetReader(SCREAMING_SNAKE_CASE_    ,      cache_dir=SCREAMING_SNAKE_CASE_						).read()
   _check_parquet_datasetdict(SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      splits=list(path.keys()						)						)
   assert all(dataset[split].split == split for split in path.keys()						)
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)     ->     Dict:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				ParquetDatasetWriter(SCREAMING_SNAKE_CASE_    ,      tmp_path / """foo.parquet"""						)
   assert writer.write() > 0
   _SCREAMING_SNAKE_CASE      =				pq.ParquetFile(tmp_path / """foo.parquet"""						)
   _SCREAMING_SNAKE_CASE      =				pf.read()
   assert dataset.data.table == output_table
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)     ->     Optional[Any]:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE      =				str(shared_datadir / """test_image_rgb.jpg"""						)
   _SCREAMING_SNAKE_CASE      =				{"""image""": [image_path]}
   _SCREAMING_SNAKE_CASE      =				Features({"""image""": Image()}						)
   _SCREAMING_SNAKE_CASE      =				Dataset.from_dict(SCREAMING_SNAKE_CASE_    ,      features=SCREAMING_SNAKE_CASE_						)
   _SCREAMING_SNAKE_CASE      =				ParquetDatasetWriter(SCREAMING_SNAKE_CASE_    ,      tmp_path / """foo.parquet"""						)
   assert writer.write() > 0

   _SCREAMING_SNAKE_CASE      =				Dataset.from_parquet(str(tmp_path / """foo.parquet"""						)						)
   assert dataset.features == reloaded_dataset.features

   _SCREAMING_SNAKE_CASE      =				ParquetDatasetReader(str(tmp_path / """foo.parquet"""						)    ,      streaming=SCREAMING_SNAKE_CASE_						).read()
   assert dataset.features == reloaded_iterable_dataset.features







@pytest.mark.parametrize(
    """feature, expected"""    ,      [
        (Features({"""foo""": Value("""int32"""						)}						), None),
        (Features({"""image""": Image(), """foo""": Value("""int32"""						)}						), config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS),
        (Features({"""nested""": Sequence(Audio()						)}						), config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS),
    ]    ,      )
def    lowerCAmelCase_	(				SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_						)     ->     str:




   """simple docstring"""
   assert get_writer_batch_size(SCREAMING_SNAKE_CASE_						) == expected






