




'''simple docstring'''





import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / """utils"""))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
     from test_module.custom_tokenization_fast import CustomTokenizerFast






class  _lowerCAmelCase     (  unittest.TestCase	):

      def 	__magic_name__(   self		):
        # A mock response for an HTTP head request to emulate server down
        lowerCAmelCase__				:  Tuple		       =     mock.Mock()
        lowerCAmelCase__				:  Any		       =     500
        lowerCAmelCase__				:  Tuple		       =     {}
        lowerCAmelCase__				:  Optional[Any]		       =     HTTPError
        lowerCAmelCase__				:  List[str]		       =     {}

        # Download this model to make sure it's in the cache.
        lowerCAmelCase__				:  Optional[int]		       =     BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''		)

        # Under the mock environment we get a 500 error when trying to reach the tokenizer.
        with mock.patch('''requests.Session.request'''      ,				return_value=_a		) as mock_head:
          lowerCAmelCase__				:  int		       =     BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''		)
          # This check we did call the fake head request
          mock_head.assert_called()

      @require_tokenizers
      def 	__magic_name__(   self		):
        # A mock response for an HTTP head request to emulate server down
        lowerCAmelCase__				:  Any		       =     mock.Mock()
        lowerCAmelCase__				:  Dict		       =     500
        lowerCAmelCase__				:  Union[str, Any]		       =     {}
        lowerCAmelCase__				:  Dict		       =     HTTPError
        lowerCAmelCase__				:  List[Any]		       =     {}

        # Download this model to make sure it's in the cache.
        lowerCAmelCase__				:  str		       =     GPTaTokenizerFast.from_pretrained('''gpt2'''		)

        # Under the mock environment we get a 500 error when trying to reach the tokenizer.
        with mock.patch('''requests.Session.request'''      ,				return_value=_a		) as mock_head:
          lowerCAmelCase__				:  Tuple		       =     GPTaTokenizerFast.from_pretrained('''gpt2'''		)
          # This check we did call the fake head request
          mock_head.assert_called()

      def 	__magic_name__(   self		):
        # This test is for deprecated behavior and can be removed in v5
        try:
          lowerCAmelCase__				:  Any		       =     tempfile.mktemp()
          with open(_a      ,				'''wb'''		) as f:
            http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''      ,				_a		)

          lowerCAmelCase__				:  List[Any]		       =     AlbertTokenizer.from_pretrained(_a		)
        finally:
          os.remove(_a		)

        # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
        # the current folder and have the right name.
        if os.path.isfile('''tokenizer.json'''		):
          # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
          return
        try:
          with open('''tokenizer.json'''      ,				'''wb'''		) as f:
            http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'''      ,				_a		)
          lowerCAmelCase__				:  int		       =     AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''		)
          # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
          self.assertEqual(tokenizer.vocab_size      ,				1000		)
          # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

        finally:
          os.remove('''tokenizer.json'''		)







      def 	__magic_name__(   self		):
        # This test is for deprecated behavior and can be removed in v5
        lowerCAmelCase__				:  int		       =     AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''		)






@is_staging_test
class  _lowerCAmelCase     (  unittest.TestCase	):
      A__		     =		['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', 'bla', 'blou']

      @classmethod
      def 	__magic_name__(   cls		):
        lowerCAmelCase__				:  int		       =     TOKEN
        HfFolder.save_token(_a		)

      @classmethod
      def 	__magic_name__(   cls		):
        try:
          delete_repo(token=cls._token      ,				repo_id='''test-tokenizer'''		)
        except HTTPError:
          pass

        try:
          delete_repo(token=cls._token      ,				repo_id='''valid_org/test-tokenizer-org'''		)
        except HTTPError:
          pass

        try:
          delete_repo(token=cls._token      ,				repo_id='''test-dynamic-tokenizer'''		)
        except HTTPError:
          pass

      def 	__magic_name__(   self		):
        with tempfile.TemporaryDirectory() as tmp_dir:
          lowerCAmelCase__				:  List[str]		       =     os.path.join(_a      ,				'''vocab.txt'''		)
          with open(_a      ,				'''w'''      ,				encoding='''utf-8'''		) as vocab_writer:
            vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]		)		)
          lowerCAmelCase__				:  Union[str, Any]		       =     BertTokenizer(_a		)

        tokenizer.push_to_hub('''test-tokenizer'''      ,				use_auth_token=self._token		)
        lowerCAmelCase__				:  int		       =     BertTokenizer.from_pretrained(f"""{USER}/test-tokenizer"""		)
        self.assertDictEqual(new_tokenizer.vocab      ,				tokenizer.vocab		)

        # Reset repo
        delete_repo(token=self._token      ,				repo_id='''test-tokenizer'''		)

        # Push to hub via save_pretrained
        with tempfile.TemporaryDirectory() as tmp_dir:
          tokenizer.save_pretrained(_a      ,				repo_id='''test-tokenizer'''      ,				push_to_hub=_a      ,				use_auth_token=self._token		)

        lowerCAmelCase__				:  Optional[Any]		       =     BertTokenizer.from_pretrained(f"""{USER}/test-tokenizer"""		)
        self.assertDictEqual(new_tokenizer.vocab      ,				tokenizer.vocab		)

      def 	__magic_name__(   self		):
        with tempfile.TemporaryDirectory() as tmp_dir:
          lowerCAmelCase__				:  Union[str, Any]		       =     os.path.join(_a      ,				'''vocab.txt'''		)
          with open(_a      ,				'''w'''      ,				encoding='''utf-8'''		) as vocab_writer:
            vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]		)		)
          lowerCAmelCase__				:  int		       =     BertTokenizer(_a		)

        tokenizer.push_to_hub('''valid_org/test-tokenizer-org'''      ,				use_auth_token=self._token		)
        lowerCAmelCase__				:  Tuple		       =     BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''		)
        self.assertDictEqual(new_tokenizer.vocab      ,				tokenizer.vocab		)

        # Reset repo
        delete_repo(token=self._token      ,				repo_id='''valid_org/test-tokenizer-org'''		)

        # Push to hub via save_pretrained
        with tempfile.TemporaryDirectory() as tmp_dir:
          tokenizer.save_pretrained(
              _a      ,				repo_id='''valid_org/test-tokenizer-org'''      ,				push_to_hub=_a      ,				use_auth_token=self._token		)

        lowerCAmelCase__				:  List[Any]		       =     BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''		)
        self.assertDictEqual(new_tokenizer.vocab      ,				tokenizer.vocab		)







      @require_tokenizers
      def 	__magic_name__(   self		):
        CustomTokenizer.register_for_auto_class()
        with tempfile.TemporaryDirectory() as tmp_dir:
          lowerCAmelCase__				:  Optional[Any]		       =     os.path.join(_a      ,				'''vocab.txt'''		)
          with open(_a      ,				'''w'''      ,				encoding='''utf-8'''		) as vocab_writer:
            vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]		)		)
          lowerCAmelCase__				:  List[str]		       =     CustomTokenizer(_a		)

        # No fast custom tokenizer
        tokenizer.push_to_hub('''test-dynamic-tokenizer'''      ,				use_auth_token=self._token		)

        lowerCAmelCase__				:  List[str]		       =     AutoTokenizer.from_pretrained(f"""{USER}/test-dynamic-tokenizer"""      ,				trust_remote_code=_a		)
        # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
        self.assertEqual(tokenizer.__class__.__name__      ,				'''CustomTokenizer'''		)

        # Fast and slow custom tokenizer
        CustomTokenizerFast.register_for_auto_class()
        with tempfile.TemporaryDirectory() as tmp_dir:
          lowerCAmelCase__				:  Dict		       =     os.path.join(_a      ,				'''vocab.txt'''		)
          with open(_a      ,				'''w'''      ,				encoding='''utf-8'''		) as vocab_writer:
            vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]		)		)

          lowerCAmelCase__				:  Optional[int]		       =     BertTokenizerFast.from_pretrained(_a		)
          bert_tokenizer.save_pretrained(_a		)
          lowerCAmelCase__				:  List[str]		       =     CustomTokenizerFast.from_pretrained(_a		)

        tokenizer.push_to_hub('''test-dynamic-tokenizer'''      ,				use_auth_token=self._token		)

        lowerCAmelCase__				:  Optional[Any]		       =     AutoTokenizer.from_pretrained(f"""{USER}/test-dynamic-tokenizer"""      ,				trust_remote_code=_a		)
        # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
        self.assertEqual(tokenizer.__class__.__name__      ,				'''CustomTokenizerFast'''		)
        lowerCAmelCase__				:  Dict		       =     AutoTokenizer.from_pretrained(
            f"""{USER}/test-dynamic-tokenizer"""      ,				use_fast=_a      ,				trust_remote_code=_a		)
        # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
        self.assertEqual(tokenizer.__class__.__name__      ,				'''CustomTokenizer'''		)






class  _lowerCAmelCase     (  unittest.TestCase	):

      def 	__magic_name__(   self		):
        lowerCAmelCase__				:  Optional[Any]		       =     Trie()
        trie.add('''Hello 友達'''		)
        self.assertEqual(trie.data      ,				{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}		)
        trie.add('''Hello'''		)
        trie.data
        self.assertEqual(trie.data      ,				{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}		)

      def 	__magic_name__(   self		):
        lowerCAmelCase__				:  List[Any]		       =     Trie()
        self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''		)      ,				['''[CLS] This is a extra_id_100''']		)
        trie.add('''[CLS]'''		)
        trie.add('''extra_id_1'''		)
        trie.add('''extra_id_100'''		)
        self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''		)      ,				['''[CLS]''', ''' This is a ''', '''extra_id_100''']		)

      def 	__magic_name__(   self		):
        lowerCAmelCase__				:  int		       =     Trie()
        trie.add('''A'''		)
        self.assertEqual(trie.split('''ABC'''		)      ,				['''A''', '''BC''']		)
        self.assertEqual(trie.split('''BCA'''		)      ,				['''BC''', '''A''']		)

      def 	__magic_name__(   self		):
        lowerCAmelCase__				:  Optional[Any]		       =     Trie()
        trie.add('''TOKEN]'''		)
        trie.add('''[SPECIAL_TOKEN]'''		)
        self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''		)      ,				['''This is something ''', '''[SPECIAL_TOKEN]''']		)

      def 	__magic_name__(   self		):
        lowerCAmelCase__				:  List[Any]		       =     Trie()
        trie.add('''A'''		)
        trie.add('''P'''		)
        trie.add('''[SPECIAL_TOKEN]'''		)
        self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''		)      ,				['''This is something ''', '''[SPECIAL_TOKEN]''']		)

      def 	__magic_name__(   self		):
        lowerCAmelCase__				:  List[str]		       =     Trie()
        trie.add('''AB'''		)
        trie.add('''B'''		)
        trie.add('''C'''		)
        self.assertEqual(trie.split('''ABC'''		)      ,				['''AB''', '''C''']		)

      def 	__magic_name__(   self		):
        lowerCAmelCase__				:  Union[str, Any]		       =     Trie()
        trie.add('''ABC'''		)
        trie.add('''B'''		)
        trie.add('''CD'''		)
        self.assertEqual(trie.split('''ABCD'''		)      ,				['''ABC''', '''D''']		)







      def 	__magic_name__(   self		):
        # Even if the offsets are wrong, we necessarily output correct string
        # parts.
        lowerCAmelCase__				:  List[Any]		       =     Trie()
        lowerCAmelCase__				:  Any		       =     trie.cut_text('''ABC'''      ,				[0, 0, 2, 1, 2, 3]		)
        self.assertEqual(_a      ,				['''AB''', '''C''']		)







import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					)     ->	Optional[Any]:
  lowerCAmelCase__				:  List[str]		       =     10
  lowerCAmelCase__				:  Optional[Any]		       =     datasets.Features(
      {
          '''tokens''': datasets.Sequence(datasets.Value('''string'''      )      ),
          '''labels''': datasets.Sequence(datasets.ClassLabel(names=['''negative''', '''positive''']      )      ),
          '''answers''': datasets.Sequence(
              {
                  '''text''': datasets.Value('''string'''      ),
                  '''answer_start''': datasets.Value('''int32'''      ),
              }      ),
          '''id''': datasets.Value('''int64'''      ),
      }      )
  lowerCAmelCase__				:  List[Any]		       =     datasets.Dataset.from_dict(
      {
          '''tokens''': [['''foo'''] * 5] * n,
          '''labels''': [[1] * 5] * n,
          '''answers''': [{'''answer_start''': [97], '''text''': ['''1976''']}] * 10,
          '''id''': list(range(UpperCamelCase      )      ),
      }	,      features=UpperCamelCase	,      )
  return dataset



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase      )     ->	List[Any]:
  lowerCAmelCase__				:  str		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''file.arrow'''      )
  dataset.map(cache_file_name=UpperCamelCase      )
  return filename


# FILE_CONTENT + files


lowerCAmelCase_					    = """\
    Text data.
    Second line of data."""



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Optional[int]:
  lowerCAmelCase__				:  Union[str, Any]		       =     tmp_path_factory.mktemp('''data'''      ) / '''file.txt'''
  lowerCAmelCase__				:  int		       =     FILE_CONTENT
  with open(UpperCamelCase	,      '''w'''      ) as f:
    f.write(UpperCamelCase      )
  return filename



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Optional[Any]:
  import bza

  lowerCAmelCase__				:  List[Any]		       =     tmp_path_factory.mktemp('''data'''      ) / '''file.txt.bz2'''
  lowerCAmelCase__				:  Optional[Any]		       =     bytes(UpperCamelCase	,      '''utf-8'''      )
  with bza.open(UpperCamelCase	,      '''wb'''      ) as f:
    f.write(UpperCamelCase      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	List[Any]:
  import gzip

  lowerCAmelCase__				:  List[str]		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''file.txt.gz'''      )
  lowerCAmelCase__				:  Any		       =     bytes(UpperCamelCase	,      '''utf-8'''      )
  with gzip.open(UpperCamelCase	,      '''wb'''      ) as f:
    f.write(UpperCamelCase      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Tuple:
  if datasets.config.LZ4_AVAILABLE:
    import lza.frame

    lowerCAmelCase__				:  Union[str, Any]		       =     tmp_path_factory.mktemp('''data'''      ) / '''file.txt.lz4'''
    lowerCAmelCase__				:  Tuple		       =     bytes(UpperCamelCase	,      '''utf-8'''      )
    with lza.frame.open(UpperCamelCase	,      '''wb'''      ) as f:
      f.write(UpperCamelCase      )
    return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase      )     ->	Optional[Any]:
  if datasets.config.PY7ZR_AVAILABLE:
    import pyazr

    lowerCAmelCase__				:  int		       =     tmp_path_factory.mktemp('''data'''      ) / '''file.txt.7z'''
    with pyazr.SevenZipFile(UpperCamelCase	,      '''w'''      ) as archive:
      archive.write(UpperCamelCase	,      arcname=os.path.basename(UpperCamelCase      )      )
    return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase      )     ->	Union[str, Any]:
  import tarfile

  lowerCAmelCase__				:  List[str]		       =     tmp_path_factory.mktemp('''data'''      ) / '''file.txt.tar'''
  with tarfile.TarFile(UpperCamelCase	,      '''w'''      ) as f:
    f.add(UpperCamelCase	,      arcname=os.path.basename(UpperCamelCase      )      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Optional[int]:
  import lzma

  lowerCAmelCase__				:  Union[str, Any]		       =     tmp_path_factory.mktemp('''data'''      ) / '''file.txt.xz'''
  lowerCAmelCase__				:  str		       =     bytes(UpperCamelCase	,      '''utf-8'''      )
  with lzma.open(UpperCamelCase	,      '''wb'''      ) as f:
    f.write(UpperCamelCase      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase      )     ->	Any:
  import zipfile

  lowerCAmelCase__				:  Optional[int]		       =     tmp_path_factory.mktemp('''data'''      ) / '''file.txt.zip'''
  with zipfile.ZipFile(UpperCamelCase	,      '''w'''      ) as f:
    f.write(UpperCamelCase	,      arcname=os.path.basename(UpperCamelCase      )      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	List[str]:
  if datasets.config.ZSTANDARD_AVAILABLE:
    import zstandard as zstd

    lowerCAmelCase__				:  Tuple		       =     tmp_path_factory.mktemp('''data'''      ) / '''file.txt.zst'''
    lowerCAmelCase__				:  int		       =     bytes(UpperCamelCase	,      '''utf-8'''      )
    with zstd.open(UpperCamelCase	,      '''wb'''      ) as f:
      f.write(UpperCamelCase      )
    return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Dict:
  lowerCAmelCase__				:  Optional[int]		       =     tmp_path_factory.mktemp('''data'''      ) / '''file.xml'''
  lowerCAmelCase__				:  Tuple		       =     textwrap.dedent(
      '''\
    <?xml version="1.0" encoding="UTF-8" ?>
    <tmx version="1.4">
      <header segtype="sentence" srclang="ca" />
      <body>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 1</seg></tuv>
          <tuv xml:lang="en"><seg>Content 1</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 2</seg></tuv>
          <tuv xml:lang="en"><seg>Content 2</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 3</seg></tuv>
          <tuv xml:lang="en"><seg>Content 3</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 4</seg></tuv>
          <tuv xml:lang="en"><seg>Content 4</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 5</seg></tuv>
          <tuv xml:lang="en"><seg>Content 5</seg></tuv>
        </tu>
      </body>
    </tmx>'''      )
  with open(UpperCamelCase	,      '''w'''      ) as f:
    f.write(UpperCamelCase      )
  return filename


lowerCAmelCase_					    = [
    {"""col_1""": """0""", """col_2""": 0, """col_3""": 0.0},
    {"""col_1""": """1""", """col_2""": 1, """col_3""": 1.0},
    {"""col_1""": """2""", """col_2""": 2, """col_3""": 2.0},
    {"""col_1""": """3""", """col_2""": 3, """col_3""": 3.0},
]
lowerCAmelCase_					    = [
    {"""col_1""": """4""", """col_2""": 4, """col_3""": 4.0},
    {"""col_1""": """5""", """col_2""": 5, """col_3""": 5.0},
]
lowerCAmelCase_					    = {
    """col_1""": ["""0""", """1""", """2""", """3"""],
    """col_2""": [0, 1, 2, 3],
    """col_3""": [0.0, 1.0, 2.0, 3.0],
}

lowerCAmelCase_					    = [
    {"""col_3""": 0.0, """col_1""": """0""", """col_2""": 0},
    {"""col_3""": 1.0, """col_1""": """1""", """col_2""": 1},
]

lowerCAmelCase_					    = [
    {"""col_1""": """s0""", """col_2""": 0, """col_3""": 0.0},
    {"""col_1""": """s1""", """col_2""": 1, """col_3""": 1.0},
    {"""col_1""": """s2""", """col_2""": 2, """col_3""": 2.0},
    {"""col_1""": """s3""", """col_2""": 3, """col_3""": 3.0},
]



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					)     ->	Tuple:
  return DATA_DICT_OF_LISTS



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Any:
  lowerCAmelCase__				:  str		       =     datasets.Dataset.from_dict(UpperCamelCase      )
  lowerCAmelCase__				:  List[str]		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''dataset.arrow'''      )
  dataset.map(cache_file_name=UpperCamelCase      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Any:
  lowerCAmelCase__				:  List[str]		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''dataset.sqlite'''      )
  with contextlib.closing(sqlitea.connect(UpperCamelCase      )      ) as con:
    lowerCAmelCase__				:  int		       =     con.cursor()
    cur.execute('''CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)'''      )
    for item in DATA:
      cur.execute('''INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)'''	,      tuple(item.values()      )      )
    con.commit()
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	int:
  lowerCAmelCase__				:  List[str]		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''dataset.csv'''      )
  with open(UpperCamelCase	,      '''w'''	,      newline=''''''      ) as f:
    lowerCAmelCase__				:  List[str]		       =     csv.DictWriter(UpperCamelCase	,      fieldnames=['''col_1''', '''col_2''', '''col_3''']      )
    writer.writeheader()
    for item in DATA:
      writer.writerow(UpperCamelCase      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	List[Any]:
  lowerCAmelCase__				:  Dict		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''dataset2.csv'''      )
  with open(UpperCamelCase	,      '''w'''	,      newline=''''''      ) as f:
    lowerCAmelCase__				:  Tuple		       =     csv.DictWriter(UpperCamelCase	,      fieldnames=['''col_1''', '''col_2''', '''col_3''']      )
    writer.writeheader()
    for item in DATA:
      writer.writerow(UpperCamelCase      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase      )     ->	List[str]:
  import bza

  lowerCAmelCase__				:  Optional[int]		       =     tmp_path_factory.mktemp('''data'''      ) / '''dataset.csv.bz2'''
  with open(UpperCamelCase	,      '''rb'''      ) as f:
    lowerCAmelCase__				:  List[str]		       =     f.read()
  # data = bytes(FILE_CONTENT, "utf-8")
  with bza.open(UpperCamelCase	,      '''wb'''      ) as f:
    f.write(UpperCamelCase      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase	,      UpperCamelCase      )     ->	Tuple:
  lowerCAmelCase__				:  Optional[int]		       =     tmp_path_factory.mktemp('''data'''      ) / '''dataset.csv.zip'''
  with zipfile.ZipFile(UpperCamelCase	,      '''w'''      ) as f:
    f.write(UpperCamelCase	,      arcname=os.path.basename(UpperCamelCase      )      )
    f.write(UpperCamelCase	,      arcname=os.path.basename(UpperCamelCase      )      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase	,      UpperCamelCase      )     ->	Tuple:
  lowerCAmelCase__				:  Union[str, Any]		       =     tmp_path_factory.mktemp('''data'''      ) / '''dataset.csv.zip'''
  with zipfile.ZipFile(UpperCamelCase	,      '''w'''      ) as f:
    f.write(UpperCamelCase	,      arcname=os.path.basename(csv_path.replace('''.csv'''	,      '''.CSV'''      )      )      )
    f.write(UpperCamelCase	,      arcname=os.path.basename(csva_path.replace('''.csv'''	,      '''.CSV'''      )      )      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase	,      UpperCamelCase      )     ->	str:
  lowerCAmelCase__				:  Union[str, Any]		       =     tmp_path_factory.mktemp('''data'''      ) / '''dataset_with_dir.csv.zip'''
  with zipfile.ZipFile(UpperCamelCase	,      '''w'''      ) as f:
    f.write(UpperCamelCase	,      arcname=os.path.join('''main_dir'''	,      os.path.basename(UpperCamelCase      )      )      )
    f.write(UpperCamelCase	,      arcname=os.path.join('''main_dir'''	,      os.path.basename(UpperCamelCase      )      )      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Dict:
  lowerCAmelCase__				:  str		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''dataset.parquet'''      )
  lowerCAmelCase__				:  List[Any]		       =     pa.schema(
      {
          '''col_1''': pa.string(),
          '''col_2''': pa.intaa(),
          '''col_3''': pa.floataa(),
      }      )
  with open(UpperCamelCase	,      '''wb'''      ) as f:
    lowerCAmelCase__				:  str		       =     pq.ParquetWriter(UpperCamelCase	,      schema=UpperCamelCase      )
    lowerCAmelCase__				:  List[Any]		       =     pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(UpperCamelCase      )      )] for k in DATA[0]}	,      schema=UpperCamelCase      )
    writer.write_table(UpperCamelCase      )
    writer.close()
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Dict:
  lowerCAmelCase__				:  List[Any]		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''dataset.json'''      )
  lowerCAmelCase__				:  List[str]		       =     {'''data''': DATA}
  with open(UpperCamelCase	,      '''w'''      ) as f:
    json.dump(UpperCamelCase	,      UpperCamelCase      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Tuple:
  lowerCAmelCase__				:  Tuple		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''dataset.json'''      )
  lowerCAmelCase__				:  Optional[Any]		       =     {'''data''': DATA_DICT_OF_LISTS}
  with open(UpperCamelCase	,      '''w'''      ) as f:
    json.dump(UpperCamelCase	,      UpperCamelCase      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	int:
  lowerCAmelCase__				:  Union[str, Any]		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''dataset.jsonl'''      )
  with open(UpperCamelCase	,      '''w'''      ) as f:
    for item in DATA:
      f.write(json.dumps(UpperCamelCase      ) + '''\n'''      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	str:
  lowerCAmelCase__				:  Optional[int]		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''dataset2.jsonl'''      )
  with open(UpperCamelCase	,      '''w'''      ) as f:
    for item in DATA:
      f.write(json.dumps(UpperCamelCase      ) + '''\n'''      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Optional[Any]:
  lowerCAmelCase__				:  Dict		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''dataset_312.jsonl'''      )
  with open(UpperCamelCase	,      '''w'''      ) as f:
    for item in DATA_312:
      f.write(json.dumps(UpperCamelCase      ) + '''\n'''      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	List[Any]:
  lowerCAmelCase__				:  Optional[int]		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''dataset-str.jsonl'''      )
  with open(UpperCamelCase	,      '''w'''      ) as f:
    for item in DATA_STR:
      f.write(json.dumps(UpperCamelCase      ) + '''\n'''      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase      )     ->	int:
  import gzip

  lowerCAmelCase__				:  Dict		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''dataset.txt.gz'''      )
  with open(UpperCamelCase	,      '''rb'''      ) as orig_file:
    with gzip.open(UpperCamelCase	,      '''wb'''      ) as zipped_file:
      zipped_file.writelines(UpperCamelCase      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase      )     ->	List[Any]:
  import gzip

  lowerCAmelCase__				:  int		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''dataset.jsonl.gz'''      )
  with open(UpperCamelCase	,      '''rb'''      ) as orig_file:
    with gzip.open(UpperCamelCase	,      '''wb'''      ) as zipped_file:
      zipped_file.writelines(UpperCamelCase      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase	,      UpperCamelCase      )     ->	Union[str, Any]:
  lowerCAmelCase__				:  str		       =     tmp_path_factory.mktemp('''data'''      ) / '''dataset.jsonl.zip'''
  with zipfile.ZipFile(UpperCamelCase	,      '''w'''      ) as f:
    f.write(UpperCamelCase	,      arcname=os.path.basename(UpperCamelCase      )      )
    f.write(UpperCamelCase	,      arcname=os.path.basename(UpperCamelCase      )      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase	,      UpperCamelCase	,      UpperCamelCase      )     ->	Optional[int]:
  lowerCAmelCase__				:  List[Any]		       =     tmp_path_factory.mktemp('''data'''      ) / '''dataset_nested.jsonl.zip'''
  with zipfile.ZipFile(UpperCamelCase	,      '''w'''      ) as f:
    f.write(UpperCamelCase	,      arcname=os.path.join('''nested'''	,      os.path.basename(UpperCamelCase      )      )      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase	,      UpperCamelCase      )     ->	Optional[Any]:
  lowerCAmelCase__				:  Tuple		       =     tmp_path_factory.mktemp('''data'''      ) / '''dataset_with_dir.jsonl.zip'''
  with zipfile.ZipFile(UpperCamelCase	,      '''w'''      ) as f:
    f.write(UpperCamelCase	,      arcname=os.path.join('''main_dir'''	,      os.path.basename(UpperCamelCase      )      )      )
    f.write(UpperCamelCase	,      arcname=os.path.join('''main_dir'''	,      os.path.basename(UpperCamelCase      )      )      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase	,      UpperCamelCase      )     ->	int:
  lowerCAmelCase__				:  Optional[Any]		       =     tmp_path_factory.mktemp('''data'''      ) / '''dataset.jsonl.tar'''
  with tarfile.TarFile(UpperCamelCase	,      '''w'''      ) as f:
    f.add(UpperCamelCase	,      arcname=os.path.basename(UpperCamelCase      )      )
    f.add(UpperCamelCase	,      arcname=os.path.basename(UpperCamelCase      )      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase	,      UpperCamelCase	,      UpperCamelCase      )     ->	Optional[int]:
  lowerCAmelCase__				:  Union[str, Any]		       =     tmp_path_factory.mktemp('''data'''      ) / '''dataset_nested.jsonl.tar'''
  with tarfile.TarFile(UpperCamelCase	,      '''w'''      ) as f:
    f.add(UpperCamelCase	,      arcname=os.path.join('''nested'''	,      os.path.basename(UpperCamelCase      )      )      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Optional[Any]:
  lowerCAmelCase__				:  Dict		       =     ['''0''', '''1''', '''2''', '''3''']
  lowerCAmelCase__				:  Any		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''dataset.txt'''      )
  with open(UpperCamelCase	,      '''w'''      ) as f:
    for item in data:
      f.write(item + '''\n'''      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Any:
  lowerCAmelCase__				:  int		       =     ['''0''', '''1''', '''2''', '''3''']
  lowerCAmelCase__				:  int		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''dataset2.txt'''      )
  with open(UpperCamelCase	,      '''w'''      ) as f:
    for item in data:
      f.write(item + '''\n'''      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Optional[int]:
  lowerCAmelCase__				:  Optional[Any]		       =     ['''0''', '''1''', '''2''', '''3''']
  lowerCAmelCase__				:  str		       =     tmp_path_factory.mktemp('''data'''      ) / '''dataset.abc'''
  with open(UpperCamelCase	,      '''w'''      ) as f:
    for item in data:
      f.write(item + '''\n'''      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase	,      UpperCamelCase      )     ->	Any:
  lowerCAmelCase__				:  Optional[int]		       =     tmp_path_factory.mktemp('''data'''      ) / '''dataset.text.zip'''
  with zipfile.ZipFile(UpperCamelCase	,      '''w'''      ) as f:
    f.write(UpperCamelCase	,      arcname=os.path.basename(UpperCamelCase      )      )
    f.write(UpperCamelCase	,      arcname=os.path.basename(UpperCamelCase      )      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase	,      UpperCamelCase      )     ->	Union[str, Any]:
  lowerCAmelCase__				:  Any		       =     tmp_path_factory.mktemp('''data'''      ) / '''dataset_with_dir.text.zip'''
  with zipfile.ZipFile(UpperCamelCase	,      '''w'''      ) as f:
    f.write(UpperCamelCase	,      arcname=os.path.join('''main_dir'''	,      os.path.basename(UpperCamelCase      )      )      )
    f.write(UpperCamelCase	,      arcname=os.path.join('''main_dir'''	,      os.path.basename(UpperCamelCase      )      )      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase	,      UpperCamelCase      )     ->	int:
  lowerCAmelCase__				:  str		       =     tmp_path_factory.mktemp('''data'''      ) / '''dataset.ext.zip'''
  with zipfile.ZipFile(UpperCamelCase	,      '''w'''      ) as f:
    f.write(UpperCamelCase	,      arcname=os.path.basename('''unsupported.ext'''      )      )
    f.write(UpperCamelCase	,      arcname=os.path.basename('''unsupported_2.ext'''      )      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Dict:
  lowerCAmelCase__				:  List[Any]		       =     '''\n'''.join(['''First''', '''Second\u2029with Unicode new line''', '''Third''']      )
  lowerCAmelCase__				:  Union[str, Any]		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''dataset_with_unicode_new_lines.txt'''      )
  with open(UpperCamelCase	,      '''w'''	,      encoding='''utf-8'''      ) as f:
    f.write(UpperCamelCase      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					)     ->	Optional[int]:
  return os.path.join('''tests'''	,      '''features'''	,      '''data'''	,      '''test_image_rgb.jpg'''      )



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					)     ->	Optional[Any]:
  return os.path.join('''tests'''	,      '''features'''	,      '''data'''	,      '''test_audio_44100.wav'''      )



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase      )     ->	Union[str, Any]:
  lowerCAmelCase__				:  Tuple		       =     tmp_path_factory.mktemp('''data'''      ) / '''dataset.img.zip'''
  with zipfile.ZipFile(UpperCamelCase	,      '''w'''      ) as f:
    f.write(UpperCamelCase	,      arcname=os.path.basename(UpperCamelCase      )      )
    f.write(UpperCamelCase	,      arcname=os.path.basename(UpperCamelCase      ).replace('''.jpg'''	,      '''2.jpg'''      )      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Dict:
  lowerCAmelCase__				:  Optional[Any]		       =     tmp_path_factory.mktemp('''data_dir'''      )

  (data_dir / "subdir").mkdir()
  with open(data_dir / '''subdir''' / '''train.txt'''	,      '''w'''      ) as f:
    f.write('''foo\n''' * 10      )
  with open(data_dir / '''subdir''' / '''test.txt'''	,      '''w'''      ) as f:
    f.write('''bar\n''' * 10      )
  # hidden file
  with open(data_dir / '''subdir''' / '''.test.txt'''	,      '''w'''      ) as f:
    f.write('''bar\n''' * 10      )

  # hidden directory
  (data_dir / ".subdir").mkdir()
  with open(data_dir / '''.subdir''' / '''train.txt'''	,      '''w'''      ) as f:
    f.write('''foo\n''' * 10      )
  with open(data_dir / '''.subdir''' / '''test.txt'''	,      '''w'''      ) as f:
    f.write('''bar\n''' * 10      )

  return data_dir

