



import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     ):
			lowercase__					  =				10
			lowercase__					  =				datasets.Features(
			    {
			        "tokens": datasets.Sequence(datasets.Value("string"  )  ),
			        "labels": datasets.Sequence(datasets.ClassLabel(names=["negative", "positive"]  )  ),
			        "answers": datasets.Sequence(
			            {
			                "text": datasets.Value("string"  ),
			                "answer_start": datasets.Value("int32"  ),
			            }  ),
			        "id": datasets.Value("int64"  ),
			    }  )
			lowercase__					  =				datasets.Dataset.from_dict(
			    {
			        "tokens": [["foo"] * 5] * n,
			        "labels": [[1] * 5] * n,
			        "answers": [{"answer_start": [97], "text": ["1976"]}] * 10,
			        "id": list(range(SCREAMING_SNAKE_CASE_  )  ),
			    }						,  features=SCREAMING_SNAKE_CASE_						,  )
			return dataset



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				str(tmp_path_factory.mktemp("data"  ) / "file.arrow"  )
			dataset.map(cache_file_name=SCREAMING_SNAKE_CASE_  )
			return filename


# FILE_CONTENT + files


lowercase_								=     """\
    Text data.
    Second line of data."""



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				tmp_path_factory.mktemp("data"  ) / "file.txt"
			lowercase__					  =				FILE_CONTENT
			with open(SCREAMING_SNAKE_CASE_						,  "w"  ) as f:
						f.write(SCREAMING_SNAKE_CASE_  )
			return filename



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_  ):
			import bza

			lowercase__					  =				tmp_path_factory.mktemp("data"  ) / "file.txt.bz2"
			lowercase__					  =				bytes(SCREAMING_SNAKE_CASE_						,  "utf-8"  )
			with bza.open(SCREAMING_SNAKE_CASE_						,  "wb"  ) as f:
						f.write(SCREAMING_SNAKE_CASE_  )
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_  ):
			import gzip

			lowercase__					  =				str(tmp_path_factory.mktemp("data"  ) / "file.txt.gz"  )
			lowercase__					  =				bytes(SCREAMING_SNAKE_CASE_						,  "utf-8"  )
			with gzip.open(SCREAMING_SNAKE_CASE_						,  "wb"  ) as f:
						f.write(SCREAMING_SNAKE_CASE_  )
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_  ):
			if datasets.config.LZ4_AVAILABLE:
						import lza.frame

						lowercase__					  =				tmp_path_factory.mktemp("data"  ) / "file.txt.lz4"
						lowercase__					  =				bytes(SCREAMING_SNAKE_CASE_						,  "utf-8"  )
						with lza.frame.open(SCREAMING_SNAKE_CASE_						,  "wb"  ) as f:
									f.write(SCREAMING_SNAKE_CASE_  )
						return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			if datasets.config.PY7ZR_AVAILABLE:
						import pyazr

						lowercase__					  =				tmp_path_factory.mktemp("data"  ) / "file.txt.7z"
						with pyazr.SevenZipFile(SCREAMING_SNAKE_CASE_						,  "w"  ) as archive:
									archive.write(SCREAMING_SNAKE_CASE_						,  arcname=os.path.basename(SCREAMING_SNAKE_CASE_  )  )
						return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			import tarfile

			lowercase__					  =				tmp_path_factory.mktemp("data"  ) / "file.txt.tar"
			with tarfile.TarFile(SCREAMING_SNAKE_CASE_						,  "w"  ) as f:
						f.add(SCREAMING_SNAKE_CASE_						,  arcname=os.path.basename(SCREAMING_SNAKE_CASE_  )  )
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_  ):
			import lzma

			lowercase__					  =				tmp_path_factory.mktemp("data"  ) / "file.txt.xz"
			lowercase__					  =				bytes(SCREAMING_SNAKE_CASE_						,  "utf-8"  )
			with lzma.open(SCREAMING_SNAKE_CASE_						,  "wb"  ) as f:
						f.write(SCREAMING_SNAKE_CASE_  )
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			import zipfile

			lowercase__					  =				tmp_path_factory.mktemp("data"  ) / "file.txt.zip"
			with zipfile.ZipFile(SCREAMING_SNAKE_CASE_						,  "w"  ) as f:
						f.write(SCREAMING_SNAKE_CASE_						,  arcname=os.path.basename(SCREAMING_SNAKE_CASE_  )  )
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_  ):
			if datasets.config.ZSTANDARD_AVAILABLE:
						import zstandard as zstd

						lowercase__					  =				tmp_path_factory.mktemp("data"  ) / "file.txt.zst"
						lowercase__					  =				bytes(SCREAMING_SNAKE_CASE_						,  "utf-8"  )
						with zstd.open(SCREAMING_SNAKE_CASE_						,  "wb"  ) as f:
									f.write(SCREAMING_SNAKE_CASE_  )
						return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				tmp_path_factory.mktemp("data"  ) / "file.xml"
			lowercase__					  =				textwrap.dedent(
			    "\\n    <?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n    <tmx version=\"1.4\">\n      <header segtype=\"sentence\" srclang=\"ca\" />\n      <body>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 1</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 1</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 2</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 2</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 3</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 3</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 4</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 4</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 5</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 5</seg></tuv>\n        </tu>\n      </body>\n    </tmx>"  )
			with open(SCREAMING_SNAKE_CASE_						,  "w"  ) as f:
						f.write(SCREAMING_SNAKE_CASE_  )
			return filename


lowercase_								=     [
    {"""col_1""": """0""", """col_2""": 0, """col_3""": 0.0},
    {"""col_1""": """1""", """col_2""": 1, """col_3""": 1.0},
    {"""col_1""": """2""", """col_2""": 2, """col_3""": 2.0},
    {"""col_1""": """3""", """col_2""": 3, """col_3""": 3.0},
]
lowercase_								=     [
    {"""col_1""": """4""", """col_2""": 4, """col_3""": 4.0},
    {"""col_1""": """5""", """col_2""": 5, """col_3""": 5.0},
]
lowercase_								=     {
    """col_1""": ["""0""", """1""", """2""", """3"""],
    """col_2""": [0, 1, 2, 3],
    """col_3""": [0.0, 1.0, 2.0, 3.0],
}

lowercase_								=     [
    {"""col_3""": 0.0, """col_1""": """0""", """col_2""": 0},
    {"""col_3""": 1.0, """col_1""": """1""", """col_2""": 1},
]

lowercase_								=     [
    {"""col_1""": """s0""", """col_2""": 0, """col_3""": 0.0},
    {"""col_1""": """s1""", """col_2""": 1, """col_3""": 1.0},
    {"""col_1""": """s2""", """col_2""": 2, """col_3""": 2.0},
    {"""col_1""": """s3""", """col_2""": 3, """col_3""": 3.0},
]



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     ):
			return DATA_DICT_OF_LISTS



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				datasets.Dataset.from_dict(SCREAMING_SNAKE_CASE_  )
			lowercase__					  =				str(tmp_path_factory.mktemp("data"  ) / "dataset.arrow"  )
			dataset.map(cache_file_name=SCREAMING_SNAKE_CASE_  )
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				str(tmp_path_factory.mktemp("data"  ) / "dataset.sqlite"  )
			with contextlib.closing(sqlitea.connect(SCREAMING_SNAKE_CASE_  )  ) as con:
						lowercase__					  =				con.cursor()
						cur.execute("CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)"  )
						for item in DATA:
									cur.execute("INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)"						,  tuple(item.values()  )  )
						con.commit()
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				str(tmp_path_factory.mktemp("data"  ) / "dataset.csv"  )
			with open(SCREAMING_SNAKE_CASE_						,  "w"						,  newline=""  ) as f:
						lowercase__					  =				csv.DictWriter(SCREAMING_SNAKE_CASE_						,  fieldnames=["col_1", "col_2", "col_3"]  )
						writer.writeheader()
						for item in DATA:
									writer.writerow(SCREAMING_SNAKE_CASE_  )
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				str(tmp_path_factory.mktemp("data"  ) / "dataset2.csv"  )
			with open(SCREAMING_SNAKE_CASE_						,  "w"						,  newline=""  ) as f:
						lowercase__					  =				csv.DictWriter(SCREAMING_SNAKE_CASE_						,  fieldnames=["col_1", "col_2", "col_3"]  )
						writer.writeheader()
						for item in DATA:
									writer.writerow(SCREAMING_SNAKE_CASE_  )
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			import bza

			lowercase__					  =				tmp_path_factory.mktemp("data"  ) / "dataset.csv.bz2"
			with open(SCREAMING_SNAKE_CASE_						,  "rb"  ) as f:
						lowercase__					  =				f.read()
			# data = bytes(FILE_CONTENT, "utf-8")
			with bza.open(SCREAMING_SNAKE_CASE_						,  "wb"  ) as f:
						f.write(SCREAMING_SNAKE_CASE_  )
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				tmp_path_factory.mktemp("data"  ) / "dataset.csv.zip"
			with zipfile.ZipFile(SCREAMING_SNAKE_CASE_						,  "w"  ) as f:
						f.write(SCREAMING_SNAKE_CASE_						,  arcname=os.path.basename(SCREAMING_SNAKE_CASE_  )  )
						f.write(SCREAMING_SNAKE_CASE_						,  arcname=os.path.basename(SCREAMING_SNAKE_CASE_  )  )
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				tmp_path_factory.mktemp("data"  ) / "dataset.csv.zip"
			with zipfile.ZipFile(SCREAMING_SNAKE_CASE_						,  "w"  ) as f:
						f.write(SCREAMING_SNAKE_CASE_						,  arcname=os.path.basename(csv_path.replace(".csv"						,  ".CSV"  )  )  )
						f.write(SCREAMING_SNAKE_CASE_						,  arcname=os.path.basename(csva_path.replace(".csv"						,  ".CSV"  )  )  )
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				tmp_path_factory.mktemp("data"  ) / "dataset_with_dir.csv.zip"
			with zipfile.ZipFile(SCREAMING_SNAKE_CASE_						,  "w"  ) as f:
						f.write(SCREAMING_SNAKE_CASE_						,  arcname=os.path.join("main_dir"						,  os.path.basename(SCREAMING_SNAKE_CASE_  )  )  )
						f.write(SCREAMING_SNAKE_CASE_						,  arcname=os.path.join("main_dir"						,  os.path.basename(SCREAMING_SNAKE_CASE_  )  )  )
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				str(tmp_path_factory.mktemp("data"  ) / "dataset.parquet"  )
			lowercase__					  =				pa.schema(
			    {
			        "col_1": pa.string(),
			        "col_2": pa.intaa(),
			        "col_3": pa.floataa(),
			    }  )
			with open(SCREAMING_SNAKE_CASE_						,  "wb"  ) as f:
						lowercase__					  =				pq.ParquetWriter(SCREAMING_SNAKE_CASE_						,  schema=SCREAMING_SNAKE_CASE_  )
						lowercase__					  =				pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(SCREAMING_SNAKE_CASE_  )  )] for k in DATA[0]}						,  schema=SCREAMING_SNAKE_CASE_  )
						writer.write_table(SCREAMING_SNAKE_CASE_  )
						writer.close()
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				str(tmp_path_factory.mktemp("data"  ) / "dataset.json"  )
			lowercase__					  =				{"data": DATA}
			with open(SCREAMING_SNAKE_CASE_						,  "w"  ) as f:
						json.dump(SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  )
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				str(tmp_path_factory.mktemp("data"  ) / "dataset.json"  )
			lowercase__					  =				{"data": DATA_DICT_OF_LISTS}
			with open(SCREAMING_SNAKE_CASE_						,  "w"  ) as f:
						json.dump(SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  )
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				str(tmp_path_factory.mktemp("data"  ) / "dataset.jsonl"  )
			with open(SCREAMING_SNAKE_CASE_						,  "w"  ) as f:
						for item in DATA:
									f.write(json.dumps(SCREAMING_SNAKE_CASE_  ) + "\n"  )
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				str(tmp_path_factory.mktemp("data"  ) / "dataset2.jsonl"  )
			with open(SCREAMING_SNAKE_CASE_						,  "w"  ) as f:
						for item in DATA:
									f.write(json.dumps(SCREAMING_SNAKE_CASE_  ) + "\n"  )
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				str(tmp_path_factory.mktemp("data"  ) / "dataset_312.jsonl"  )
			with open(SCREAMING_SNAKE_CASE_						,  "w"  ) as f:
						for item in DATA_312:
									f.write(json.dumps(SCREAMING_SNAKE_CASE_  ) + "\n"  )
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				str(tmp_path_factory.mktemp("data"  ) / "dataset-str.jsonl"  )
			with open(SCREAMING_SNAKE_CASE_						,  "w"  ) as f:
						for item in DATA_STR:
									f.write(json.dumps(SCREAMING_SNAKE_CASE_  ) + "\n"  )
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			import gzip

			lowercase__					  =				str(tmp_path_factory.mktemp("data"  ) / "dataset.txt.gz"  )
			with open(SCREAMING_SNAKE_CASE_						,  "rb"  ) as orig_file:
						with gzip.open(SCREAMING_SNAKE_CASE_						,  "wb"  ) as zipped_file:
									zipped_file.writelines(SCREAMING_SNAKE_CASE_  )
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			import gzip

			lowercase__					  =				str(tmp_path_factory.mktemp("data"  ) / "dataset.jsonl.gz"  )
			with open(SCREAMING_SNAKE_CASE_						,  "rb"  ) as orig_file:
						with gzip.open(SCREAMING_SNAKE_CASE_						,  "wb"  ) as zipped_file:
									zipped_file.writelines(SCREAMING_SNAKE_CASE_  )
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				tmp_path_factory.mktemp("data"  ) / "dataset.jsonl.zip"
			with zipfile.ZipFile(SCREAMING_SNAKE_CASE_						,  "w"  ) as f:
						f.write(SCREAMING_SNAKE_CASE_						,  arcname=os.path.basename(SCREAMING_SNAKE_CASE_  )  )
						f.write(SCREAMING_SNAKE_CASE_						,  arcname=os.path.basename(SCREAMING_SNAKE_CASE_  )  )
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				tmp_path_factory.mktemp("data"  ) / "dataset_nested.jsonl.zip"
			with zipfile.ZipFile(SCREAMING_SNAKE_CASE_						,  "w"  ) as f:
						f.write(SCREAMING_SNAKE_CASE_						,  arcname=os.path.join("nested"						,  os.path.basename(SCREAMING_SNAKE_CASE_  )  )  )
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				tmp_path_factory.mktemp("data"  ) / "dataset_with_dir.jsonl.zip"
			with zipfile.ZipFile(SCREAMING_SNAKE_CASE_						,  "w"  ) as f:
						f.write(SCREAMING_SNAKE_CASE_						,  arcname=os.path.join("main_dir"						,  os.path.basename(SCREAMING_SNAKE_CASE_  )  )  )
						f.write(SCREAMING_SNAKE_CASE_						,  arcname=os.path.join("main_dir"						,  os.path.basename(SCREAMING_SNAKE_CASE_  )  )  )
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				tmp_path_factory.mktemp("data"  ) / "dataset.jsonl.tar"
			with tarfile.TarFile(SCREAMING_SNAKE_CASE_						,  "w"  ) as f:
						f.add(SCREAMING_SNAKE_CASE_						,  arcname=os.path.basename(SCREAMING_SNAKE_CASE_  )  )
						f.add(SCREAMING_SNAKE_CASE_						,  arcname=os.path.basename(SCREAMING_SNAKE_CASE_  )  )
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				tmp_path_factory.mktemp("data"  ) / "dataset_nested.jsonl.tar"
			with tarfile.TarFile(SCREAMING_SNAKE_CASE_						,  "w"  ) as f:
						f.add(SCREAMING_SNAKE_CASE_						,  arcname=os.path.join("nested"						,  os.path.basename(SCREAMING_SNAKE_CASE_  )  )  )
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				["0", "1", "2", "3"]
			lowercase__					  =				str(tmp_path_factory.mktemp("data"  ) / "dataset.txt"  )
			with open(SCREAMING_SNAKE_CASE_						,  "w"  ) as f:
						for item in data:
									f.write(item + "\n"  )
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				["0", "1", "2", "3"]
			lowercase__					  =				str(tmp_path_factory.mktemp("data"  ) / "dataset2.txt"  )
			with open(SCREAMING_SNAKE_CASE_						,  "w"  ) as f:
						for item in data:
									f.write(item + "\n"  )
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				["0", "1", "2", "3"]
			lowercase__					  =				tmp_path_factory.mktemp("data"  ) / "dataset.abc"
			with open(SCREAMING_SNAKE_CASE_						,  "w"  ) as f:
						for item in data:
									f.write(item + "\n"  )
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				tmp_path_factory.mktemp("data"  ) / "dataset.text.zip"
			with zipfile.ZipFile(SCREAMING_SNAKE_CASE_						,  "w"  ) as f:
						f.write(SCREAMING_SNAKE_CASE_						,  arcname=os.path.basename(SCREAMING_SNAKE_CASE_  )  )
						f.write(SCREAMING_SNAKE_CASE_						,  arcname=os.path.basename(SCREAMING_SNAKE_CASE_  )  )
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				tmp_path_factory.mktemp("data"  ) / "dataset_with_dir.text.zip"
			with zipfile.ZipFile(SCREAMING_SNAKE_CASE_						,  "w"  ) as f:
						f.write(SCREAMING_SNAKE_CASE_						,  arcname=os.path.join("main_dir"						,  os.path.basename(SCREAMING_SNAKE_CASE_  )  )  )
						f.write(SCREAMING_SNAKE_CASE_						,  arcname=os.path.join("main_dir"						,  os.path.basename(SCREAMING_SNAKE_CASE_  )  )  )
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				tmp_path_factory.mktemp("data"  ) / "dataset.ext.zip"
			with zipfile.ZipFile(SCREAMING_SNAKE_CASE_						,  "w"  ) as f:
						f.write(SCREAMING_SNAKE_CASE_						,  arcname=os.path.basename("unsupported.ext"  )  )
						f.write(SCREAMING_SNAKE_CASE_						,  arcname=os.path.basename("unsupported_2.ext"  )  )
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				"\n".join(["First", "Second\u2029with Unicode new line", "Third"]  )
			lowercase__					  =				str(tmp_path_factory.mktemp("data"  ) / "dataset_with_unicode_new_lines.txt"  )
			with open(SCREAMING_SNAKE_CASE_						,  "w"						,  encoding="utf-8"  ) as f:
						f.write(SCREAMING_SNAKE_CASE_  )
			return path



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     ):
			return os.path.join("tests"						,  "features"						,  "data"						,  "test_image_rgb.jpg"  )



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     ):
			return os.path.join("tests"						,  "features"						,  "data"						,  "test_audio_44100.wav"  )



@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				tmp_path_factory.mktemp("data"  ) / "dataset.img.zip"
			with zipfile.ZipFile(SCREAMING_SNAKE_CASE_						,  "w"  ) as f:
						f.write(SCREAMING_SNAKE_CASE_						,  arcname=os.path.basename(SCREAMING_SNAKE_CASE_  )  )
						f.write(SCREAMING_SNAKE_CASE_						,  arcname=os.path.basename(SCREAMING_SNAKE_CASE_  ).replace(".jpg"						,  "2.jpg"  )  )
			return path




@pytest.fixture(scope="session"  )
def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				tmp_path_factory.mktemp("data_dir"  )

			(data_dir / "subdir").mkdir()
			with open(data_dir / "subdir" / "train.txt"						,  "w"  ) as f:
						f.write("foo\n" * 10  )
			with open(data_dir / "subdir" / "test.txt"						,  "w"  ) as f:
						f.write("bar\n" * 10  )
			# hidden file
			with open(data_dir / "subdir" / ".test.txt"						,  "w"  ) as f:
						f.write("bar\n" * 10  )

			# hidden directory
			(data_dir / ".subdir").mkdir()
			with open(data_dir / ".subdir" / "train.txt"						,  "w"  ) as f:
						f.write("foo\n" * 10  )
			with open(data_dir / ".subdir" / "test.txt"						,  "w"  ) as f:
						f.write("bar\n" * 10  )

			return data_dir









import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin
class 				_snake_case			(					lowercase__    ,	unittest.TestCase):
				UpperCamelCase__				:					Dict     		=TransfoXLTokenizer
				UpperCamelCase__				:					List[Any]     		=False
				UpperCamelCase__				:					List[Any]     		=False




				def 							A__      (						self					:						Union[str, Any]						):
							super().setUp()

							lowercase__					  =				[
							    "<unk>",
							    "[CLS]",
							    "[SEP]",
							    "want",
							    "unwanted",
							    "wa",
							    "un",
							    "running",
							    ",",
							    "low",
							    "l",
							]
							lowercase__					  =				os.path.join(self.tmpdirname,					VOCAB_FILES_NAMES["vocab_file"]						)
							with open(self.vocab_file,					"w",					encoding="utf-8"						) as vocab_writer:
										vocab_writer.write("".join([x + "\n" for x in vocab_tokens]						)						)




				def 							A__      (						self					:						Union[str, Any],					**__lowercase					:						Any						):
							lowercase__					  =				True
							return TransfoXLTokenizer.from_pretrained(self.tmpdirname,					**__lowercase						)




				def 							A__      (						self					:						Tuple,					__lowercase					:						Optional[int]						):
							lowercase__					  =				"<unk> UNwanted , running"
							lowercase__					  =				"<unk> unwanted, running"
							return input_text, output_text




				def 							A__      (						self					:						str						):
							lowercase__					  =				TransfoXLTokenizer(vocab_file=self.vocab_file,					lower_case=__lowercase						)

							lowercase__					  =				tokenizer.tokenize("<unk> UNwanted , running"						)
							self.assertListEqual(__lowercase,					["<unk>", "unwanted", ",", "running"]						)

							self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowercase						),					[0, 4, 8, 7]						)




				def 							A__      (						self					:						Tuple						):
							lowercase__					  =				TransfoXLTokenizer(lower_case=__lowercase						)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo ! how  \n Are yoU ?  "						),					["hello", "!", "how", "are", "you", "?"]						)




				def 							A__      (						self					:						Tuple						):
							lowercase__					  =				TransfoXLTokenizer(lower_case=__lowercase						)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo ! how  \n Are yoU ?  "						),					["HeLLo", "!", "how", "Are", "yoU", "?"]						)




				def 							A__      (						self					:						str						):
							lowercase__					  =				TransfoXLTokenizer(lower_case=__lowercase						)
							lowercase__					  =				"Hello (bracket) and side-scrolled [and] Henry's $5,000 with 3.34 m. What's up!?"
							lowercase__					  =				[
							    "Hello",
							    "(",
							    "bracket",
							    ")",
							    "and",
							    "side",
							    "@-@",
							    "scrolled",
							    "[",
							    "and",
							    "]",
							    "Henry",
							    "'s",
							    "$",
							    "5",
							    "@,@",
							    "000",
							    "with",
							    "3",
							    "@.@",
							    "34",
							    "m",
							    ".",
							    "What",
							    "'s",
							    "up",
							    "!",
							    "?",
							]

							self.assertListEqual(tokenizer.tokenize(__lowercase						),					__lowercase						)

							self.assertEqual(tokenizer.convert_tokens_to_string(__lowercase						),					__lowercase						)




				def 							A__      (						self					:						List[str]						):
							lowercase__					  =				self.get_tokenizer()
							lowercase__					  =				len(__lowercase						)

							tokenizer.add_tokens(["new1", "new2"]						)
							tokenizer.move_added_token("new1",					1						)

							# Check that moved token is not copied (duplicate)
							self.assertEqual(len(__lowercase						),					original_len + 2						)
							# Check that token is moved to specified id
							self.assertEqual(tokenizer.encode("new1"						),					[1]						)
							self.assertEqual(tokenizer.decode([1]						),					"new1"						)




