




'''simple docstring'''





import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig



def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase	,      UpperCamelCase=0      )     ->	int:






  if name is None:
    lowerCAmelCase__				:  Dict		       =     None
  else:
    lowerCAmelCase__				:  Dict		       =     """.""" * max(0	,      spaces - 2      ) + """# {:""" + str(50 - spaces      ) + """s}"""
    lowerCAmelCase__				:  Any		       =     fmt.format(__UpperCamelCase      )

  # Print and recurse (if needed).
  if isinstance(__UpperCamelCase	,      __UpperCamelCase      ):
    if msg is not None:
      print(__UpperCamelCase      )
    for k in val.keys():
      recursive_print(__UpperCamelCase	,      val[k]	,      spaces + 2      )
  elif isinstance(__UpperCamelCase	,      torch.Tensor      ):
    print(__UpperCamelCase	,      ''':'''	,      val.size()      )
  else:
    print(__UpperCamelCase	,      ''':'''	,      __UpperCamelCase      )



def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase	,      UpperCamelCase	,      UpperCamelCase	,      UpperCamelCase      )     ->	Any:
  lowerCAmelCase__				:  Any		       =     param.size()
  if checkpoint_version == 1.0:
    # version 1.0 stores [num_heads * hidden_size * num_splits, :]
    lowerCAmelCase__				:  List[str]		       =     (num_heads, hidden_size, num_splits) + input_shape[1:]
    lowerCAmelCase__				:  Tuple		       =     param.view(*__UpperCamelCase      )
    lowerCAmelCase__				:  Tuple		       =     param.transpose(0	,      2      )
    lowerCAmelCase__				:  Any		       =     param.transpose(1	,      2      ).contiguous()
  elif checkpoint_version >= 2.0:
    # other versions store [num_heads * num_splits * hidden_size, :]
    lowerCAmelCase__				:  Optional[Any]		       =     (num_heads, num_splits, hidden_size) + input_shape[1:]
    lowerCAmelCase__				:  str		       =     param.view(*__UpperCamelCase      )
    lowerCAmelCase__				:  Dict		       =     param.transpose(0	,      1      ).contiguous()
  lowerCAmelCase__				:  int		       =     param.view(*__UpperCamelCase      )
  return param



def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase	,      UpperCamelCase      )     ->	Union[str, Any]:
  lowerCAmelCase__				:  Dict		       =     {}

  # old versions did not store training args
  lowerCAmelCase__				:  List[str]		       =     input_state_dict.get('''args'''	,      __UpperCamelCase      )
  if ds_args is not None:
    # do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
    # from pprint import pprint
    # pprint(vars(ds_args))

    lowerCAmelCase__				:  Tuple		       =     ds_args.padded_vocab_size
    lowerCAmelCase__				:  Optional[int]		       =     ds_args.max_position_embeddings
    lowerCAmelCase__				:  Union[str, Any]		       =     ds_args.hidden_size
    lowerCAmelCase__				:  Union[str, Any]		       =     ds_args.num_layers
    lowerCAmelCase__				:  str		       =     ds_args.num_attention_heads
    lowerCAmelCase__				:  str		       =     ds_args.ffn_hidden_size
    # pprint(config)

  # The number of heads.
  lowerCAmelCase__				:  Union[str, Any]		       =     config.n_head
  # The hidden_size per head.
  lowerCAmelCase__				:  Optional[Any]		       =     config.n_embd // config.n_head
  # Megatron-LM checkpoint version
  if "checkpoint_version" in input_state_dict.keys():
    lowerCAmelCase__				:  Optional[Any]		       =     input_state_dict["""checkpoint_version"""]
  else:
    lowerCAmelCase__				:  int		       =     0.0

  # The model.
  lowerCAmelCase__				:  List[str]		       =     input_state_dict["""model"""]
  # The language model.
  lowerCAmelCase__				:  str		       =     model["""language_model"""]
  # The embeddings.
  lowerCAmelCase__				:  Tuple		       =     lm["""embedding"""]

  # The word embeddings.
  lowerCAmelCase__				:  List[str]		       =     embeddings["""word_embeddings"""]["""weight"""]
  # Truncate the embedding table to vocab_size rows.
  lowerCAmelCase__				:  Optional[int]		       =     word_embeddings[: config.vocab_size, :]
  lowerCAmelCase__				:  Optional[int]		       =     word_embeddings

  # The position embeddings.
  lowerCAmelCase__				:  List[Any]		       =     embeddings["""position_embeddings"""]["""weight"""]
  # Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
  lowerCAmelCase__				:  Tuple		       =     pos_embeddings.size(0      )
  if n_positions != config.n_positions:
    raise ValueError(
        F"""pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don\'t match"""      )
  # Store the position embeddings.
  lowerCAmelCase__				:  Union[str, Any]		       =     pos_embeddings

  # The transformer.
  lowerCAmelCase__				:  Optional[Any]		       =     lm["""transformer"""] if """transformer""" in lm.keys() else lm["""encoder"""]

  # The regex to extract layer names.
  lowerCAmelCase__				:  Optional[Any]		       =     re.compile(R'''layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)'''      )

  # The simple map of names for "automated" rules.
  lowerCAmelCase__				:  List[str]		       =     {
      """attention.dense""": """.attn.c_proj.""",
      """self_attention.dense""": """.attn.c_proj.""",
      """mlp.dense_h_to_4h""": """.mlp.c_fc.""",
      """mlp.dense_4h_to_h""": """.mlp.c_proj.""",
  }

  # Extract the layers.
  for key, val in transformer.items():
    # Match the name.
    lowerCAmelCase__				:  int		       =     layer_re.match(__UpperCamelCase      )

    # Stop if that's not a layer
    if m is None:
      break

    # The index of the layer.
    lowerCAmelCase__				:  Tuple		       =     int(m.group(1      )      )
    # The name of the operation.
    lowerCAmelCase__				:  Any		       =     m.group(2      )
    # Is it a weight or a bias?
    lowerCAmelCase__				:  Union[str, Any]		       =     m.group(3      )

    # The name of the layer.
    lowerCAmelCase__				:  str		       =     F"""transformer.h.{layer_idx}"""

    # For layernorm(s), simply store the layer norm.
    if op_name.endswith('''layernorm'''      ):
      lowerCAmelCase__				:  Dict		       =     """ln_1""" if op_name.startswith('''input'''      ) else """ln_2"""
      lowerCAmelCase__				:  Optional[int]		       =     val

    # Transpose the QKV matrix.
    elif (
        op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
    ) and weight_or_bias == "weight":
      # Insert a tensor of 1x1xDxD bias.
      lowerCAmelCase__				:  Optional[Any]		       =     torch.tril(torch.ones((n_positions, n_positions)	,      dtype=torch.floataa      )      ).view(
          1	,      1	,      __UpperCamelCase	,      __UpperCamelCase      )
      lowerCAmelCase__				:  List[Any]		       =     causal_mask

      # Insert a "dummy" tensor for masked_bias.
      lowerCAmelCase__				:  str		       =     torch.tensor(-1E4	,      dtype=torch.floataa      )
      lowerCAmelCase__				:  List[Any]		       =     masked_bias

      lowerCAmelCase__				:  Optional[int]		       =     fix_query_key_value_ordering(__UpperCamelCase	,      __UpperCamelCase	,      3	,      __UpperCamelCase	,      __UpperCamelCase      )
      # Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
      lowerCAmelCase__				:  str		       =     out_val.transpose(0	,      1      ).contiguous()
      # Store.
      lowerCAmelCase__				:  Tuple		       =     out_val

    # Transpose the bias.
    elif (
        op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
    ) and weight_or_bias == "bias":
      lowerCAmelCase__				:  Optional[Any]		       =     fix_query_key_value_ordering(__UpperCamelCase	,      __UpperCamelCase	,      3	,      __UpperCamelCase	,      __UpperCamelCase      )
      # Store. No change of shape.
      lowerCAmelCase__				:  List[Any]		       =     out_val

    # Transpose the weights.
    elif weight_or_bias == "weight":
      lowerCAmelCase__				:  Any		       =     megatron_to_transformers[op_name]
      lowerCAmelCase__				:  str		       =     val.transpose(0	,      1      )

    # Copy the bias.
    elif weight_or_bias == "bias":
      lowerCAmelCase__				:  List[str]		       =     megatron_to_transformers[op_name]
      lowerCAmelCase__				:  Tuple		       =     val

    # DEBUG.
  assert config.n_layer == layer_idx + 1

  # The final layernorm.
  lowerCAmelCase__				:  Dict		       =     transformer["""final_layernorm.weight"""]
  lowerCAmelCase__				:  Dict		       =     transformer["""final_layernorm.bias"""]

  # For LM head, transformers' wants the matrix to weight embeddings.
  lowerCAmelCase__				:  Optional[int]		       =     word_embeddings

  # It should be done!
  return output_state_dict



def      __lowerCAmelCase			(					)     ->	Optional[Any]:
  lowerCAmelCase__				:  List[str]		       =     argparse.ArgumentParser()
  parser.add_argument('''--print-checkpoint-structure'''	,      action='''store_true'''      )
  parser.add_argument(
      '''path_to_checkpoint'''	,      type=__UpperCamelCase	,      help='''Path to the checkpoint file (.zip archive or direct .pt file)'''	,      )
  parser.add_argument(
      '''--config_file'''	,      default=''''''	,      type=__UpperCamelCase	,      help='''An optional config json file describing the pre-trained model.'''	,      )
  lowerCAmelCase__				:  str		       =     parser.parse_args()

  # Extract the basename.
  lowerCAmelCase__				:  Optional[Any]		       =     os.path.dirname(args.path_to_checkpoint      )

  # Load the model.
  # the .zip is very optional, let's keep it for backward compatibility
  print(F"""Extracting PyTorch state dictionary from {args.path_to_checkpoint}"""      )
  if args.path_to_checkpoint.endswith('''.zip'''      ):
    with zipfile.ZipFile(args.path_to_checkpoint	,      '''r'''      ) as checkpoint:
      with checkpoint.open('''release/mp_rank_00/model_optim_rng.pt'''      ) as pytorch_dict:
        lowerCAmelCase__				:  Optional[int]		       =     torch.load(__UpperCamelCase	,      map_location='''cpu'''      )
  else:
    lowerCAmelCase__				:  List[Any]		       =     torch.load(args.path_to_checkpoint	,      map_location='''cpu'''      )

  lowerCAmelCase__				:  Any		       =     input_state_dict.get('''args'''	,      __UpperCamelCase      )

  # Read the config, or default to the model released by NVIDIA.
  if args.config_file == "":
    if ds_args is not None:
      if ds_args.bias_gelu_fusion:
        lowerCAmelCase__				:  Any		       =     """gelu_fast"""
      elif ds_args.openai_gelu:
        lowerCAmelCase__				:  Tuple		       =     """gelu_new"""
      else:
        lowerCAmelCase__				:  List[str]		       =     """gelu"""
    else:
      # in the very early days this used to be "gelu_new"
      lowerCAmelCase__				:  Dict		       =     """gelu_new"""

    # Spell out all parameters in case the defaults change.
    lowerCAmelCase__				:  List[str]		       =     GPTaConfig(
        vocab_size=50257	,      n_positions=1024	,      n_embd=1024	,      n_layer=24	,      n_head=16	,      n_inner=4096	,      activation_function=__UpperCamelCase	,      resid_pdrop=0.1	,      embd_pdrop=0.1	,      attn_pdrop=0.1	,      layer_norm_epsilon=1E-5	,      initializer_range=0.02	,      summary_type='''cls_index'''	,      summary_use_proj=__UpperCamelCase	,      summary_activation=__UpperCamelCase	,      summary_proj_to_labels=__UpperCamelCase	,      summary_first_dropout=0.1	,      scale_attn_weights=__UpperCamelCase	,      use_cache=__UpperCamelCase	,      bos_token_id=50256	,      eos_token_id=50256	,      )
  else:
    lowerCAmelCase__				:  List[Any]		       =     GPTaConfig.from_json_file(args.config_file      )

  lowerCAmelCase__				:  int		       =     ["""GPT2LMHeadModel"""]

  # Convert.
  print('''Converting'''      )
  lowerCAmelCase__				:  Tuple		       =     convert_megatron_checkpoint(__UpperCamelCase	,      __UpperCamelCase	,      __UpperCamelCase      )

  # Print the structure of converted state dict.
  if args.print_checkpoint_structure:
    recursive_print(__UpperCamelCase	,      __UpperCamelCase      )

  # Add tokenizer class info to config
  # see https://github.com/huggingface/transformers/issues/13906)
  if ds_args is not None:
    lowerCAmelCase__				:  str		       =     ds_args.tokenizer_type
    if tokenizer_type == "GPT2BPETokenizer":
      lowerCAmelCase__				:  Optional[Any]		       =     """gpt2"""
    elif tokenizer_type == "PretrainedFromHF":
      lowerCAmelCase__				:  str		       =     ds_args.tokenizer_name_or_path
    else:
      raise ValueError(F"""Unrecognized tokenizer_type {tokenizer_type}"""      )
  else:
    lowerCAmelCase__				:  List[str]		       =     """gpt2"""

  lowerCAmelCase__				:  List[Any]		       =     AutoTokenizer.from_pretrained(__UpperCamelCase      )
  lowerCAmelCase__				:  List[str]		       =     type(__UpperCamelCase      ).__name__
  lowerCAmelCase__				:  Optional[int]		       =     tokenizer_class

  # Store the config to file.
  print('''Saving config'''      )
  config.save_pretrained(__UpperCamelCase      )

  # Save tokenizer based on args
  print(F"""Adding {tokenizer_class} tokenizer files"""      )
  tokenizer.save_pretrained(__UpperCamelCase      )

  # Store the state_dict to file.
  lowerCAmelCase__				:  List[Any]		       =     os.path.join(__UpperCamelCase	,      '''pytorch_model.bin'''      )
  print(F"""Saving checkpoint to \"{output_checkpoint_file}\""""      )
  torch.save(__UpperCamelCase	,      __UpperCamelCase      )


####################################################################################################

if __name__ == "__main__":
     main()

####################################################################################################







import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					)     ->	Optional[Any]:
  lowerCAmelCase__				:  List[str]		       =     10
  lowerCAmelCase__				:  Optional[Any]		       =     datasets.Features(
      {
          '''tokens''': datasets.Sequence(datasets.Value('''string'''      )      ),
          '''labels''': datasets.Sequence(datasets.ClassLabel(names=['''negative''', '''positive''']      )      ),
          '''answers''': datasets.Sequence(
              {
                  '''text''': datasets.Value('''string'''      ),
                  '''answer_start''': datasets.Value('''int32'''      ),
              }      ),
          '''id''': datasets.Value('''int64'''      ),
      }      )
  lowerCAmelCase__				:  List[Any]		       =     datasets.Dataset.from_dict(
      {
          '''tokens''': [['''foo'''] * 5] * n,
          '''labels''': [[1] * 5] * n,
          '''answers''': [{'''answer_start''': [97], '''text''': ['''1976''']}] * 10,
          '''id''': list(range(UpperCamelCase      )      ),
      }	,      features=UpperCamelCase	,      )
  return dataset



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase      )     ->	List[Any]:
  lowerCAmelCase__				:  str		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''file.arrow'''      )
  dataset.map(cache_file_name=UpperCamelCase      )
  return filename


# FILE_CONTENT + files


lowerCAmelCase_					    = """\
    Text data.
    Second line of data."""



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Optional[int]:
  lowerCAmelCase__				:  Union[str, Any]		       =     tmp_path_factory.mktemp('''data'''      ) / '''file.txt'''
  lowerCAmelCase__				:  int		       =     FILE_CONTENT
  with open(UpperCamelCase	,      '''w'''      ) as f:
    f.write(UpperCamelCase      )
  return filename



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Optional[Any]:
  import bza

  lowerCAmelCase__				:  List[Any]		       =     tmp_path_factory.mktemp('''data'''      ) / '''file.txt.bz2'''
  lowerCAmelCase__				:  Optional[Any]		       =     bytes(UpperCamelCase	,      '''utf-8'''      )
  with bza.open(UpperCamelCase	,      '''wb'''      ) as f:
    f.write(UpperCamelCase      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	List[Any]:
  import gzip

  lowerCAmelCase__				:  List[str]		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''file.txt.gz'''      )
  lowerCAmelCase__				:  Any		       =     bytes(UpperCamelCase	,      '''utf-8'''      )
  with gzip.open(UpperCamelCase	,      '''wb'''      ) as f:
    f.write(UpperCamelCase      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Tuple:
  if datasets.config.LZ4_AVAILABLE:
    import lza.frame

    lowerCAmelCase__				:  Union[str, Any]		       =     tmp_path_factory.mktemp('''data'''      ) / '''file.txt.lz4'''
    lowerCAmelCase__				:  Tuple		       =     bytes(UpperCamelCase	,      '''utf-8'''      )
    with lza.frame.open(UpperCamelCase	,      '''wb'''      ) as f:
      f.write(UpperCamelCase      )
    return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase      )     ->	Optional[Any]:
  if datasets.config.PY7ZR_AVAILABLE:
    import pyazr

    lowerCAmelCase__				:  int		       =     tmp_path_factory.mktemp('''data'''      ) / '''file.txt.7z'''
    with pyazr.SevenZipFile(UpperCamelCase	,      '''w'''      ) as archive:
      archive.write(UpperCamelCase	,      arcname=os.path.basename(UpperCamelCase      )      )
    return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase      )     ->	Union[str, Any]:
  import tarfile

  lowerCAmelCase__				:  List[str]		       =     tmp_path_factory.mktemp('''data'''      ) / '''file.txt.tar'''
  with tarfile.TarFile(UpperCamelCase	,      '''w'''      ) as f:
    f.add(UpperCamelCase	,      arcname=os.path.basename(UpperCamelCase      )      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Optional[int]:
  import lzma

  lowerCAmelCase__				:  Union[str, Any]		       =     tmp_path_factory.mktemp('''data'''      ) / '''file.txt.xz'''
  lowerCAmelCase__				:  str		       =     bytes(UpperCamelCase	,      '''utf-8'''      )
  with lzma.open(UpperCamelCase	,      '''wb'''      ) as f:
    f.write(UpperCamelCase      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase      )     ->	Any:
  import zipfile

  lowerCAmelCase__				:  Optional[int]		       =     tmp_path_factory.mktemp('''data'''      ) / '''file.txt.zip'''
  with zipfile.ZipFile(UpperCamelCase	,      '''w'''      ) as f:
    f.write(UpperCamelCase	,      arcname=os.path.basename(UpperCamelCase      )      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	List[str]:
  if datasets.config.ZSTANDARD_AVAILABLE:
    import zstandard as zstd

    lowerCAmelCase__				:  Tuple		       =     tmp_path_factory.mktemp('''data'''      ) / '''file.txt.zst'''
    lowerCAmelCase__				:  int		       =     bytes(UpperCamelCase	,      '''utf-8'''      )
    with zstd.open(UpperCamelCase	,      '''wb'''      ) as f:
      f.write(UpperCamelCase      )
    return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Dict:
  lowerCAmelCase__				:  Optional[int]		       =     tmp_path_factory.mktemp('''data'''      ) / '''file.xml'''
  lowerCAmelCase__				:  Tuple		       =     textwrap.dedent(
      '''\
    <?xml version="1.0" encoding="UTF-8" ?>
    <tmx version="1.4">
      <header segtype="sentence" srclang="ca" />
      <body>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 1</seg></tuv>
          <tuv xml:lang="en"><seg>Content 1</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 2</seg></tuv>
          <tuv xml:lang="en"><seg>Content 2</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 3</seg></tuv>
          <tuv xml:lang="en"><seg>Content 3</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 4</seg></tuv>
          <tuv xml:lang="en"><seg>Content 4</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 5</seg></tuv>
          <tuv xml:lang="en"><seg>Content 5</seg></tuv>
        </tu>
      </body>
    </tmx>'''      )
  with open(UpperCamelCase	,      '''w'''      ) as f:
    f.write(UpperCamelCase      )
  return filename


lowerCAmelCase_					    = [
    {"""col_1""": """0""", """col_2""": 0, """col_3""": 0.0},
    {"""col_1""": """1""", """col_2""": 1, """col_3""": 1.0},
    {"""col_1""": """2""", """col_2""": 2, """col_3""": 2.0},
    {"""col_1""": """3""", """col_2""": 3, """col_3""": 3.0},
]
lowerCAmelCase_					    = [
    {"""col_1""": """4""", """col_2""": 4, """col_3""": 4.0},
    {"""col_1""": """5""", """col_2""": 5, """col_3""": 5.0},
]
lowerCAmelCase_					    = {
    """col_1""": ["""0""", """1""", """2""", """3"""],
    """col_2""": [0, 1, 2, 3],
    """col_3""": [0.0, 1.0, 2.0, 3.0],
}

lowerCAmelCase_					    = [
    {"""col_3""": 0.0, """col_1""": """0""", """col_2""": 0},
    {"""col_3""": 1.0, """col_1""": """1""", """col_2""": 1},
]

lowerCAmelCase_					    = [
    {"""col_1""": """s0""", """col_2""": 0, """col_3""": 0.0},
    {"""col_1""": """s1""", """col_2""": 1, """col_3""": 1.0},
    {"""col_1""": """s2""", """col_2""": 2, """col_3""": 2.0},
    {"""col_1""": """s3""", """col_2""": 3, """col_3""": 3.0},
]



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					)     ->	Tuple:
  return DATA_DICT_OF_LISTS



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Any:
  lowerCAmelCase__				:  str		       =     datasets.Dataset.from_dict(UpperCamelCase      )
  lowerCAmelCase__				:  List[str]		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''dataset.arrow'''      )
  dataset.map(cache_file_name=UpperCamelCase      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Any:
  lowerCAmelCase__				:  List[str]		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''dataset.sqlite'''      )
  with contextlib.closing(sqlitea.connect(UpperCamelCase      )      ) as con:
    lowerCAmelCase__				:  int		       =     con.cursor()
    cur.execute('''CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)'''      )
    for item in DATA:
      cur.execute('''INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)'''	,      tuple(item.values()      )      )
    con.commit()
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	int:
  lowerCAmelCase__				:  List[str]		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''dataset.csv'''      )
  with open(UpperCamelCase	,      '''w'''	,      newline=''''''      ) as f:
    lowerCAmelCase__				:  List[str]		       =     csv.DictWriter(UpperCamelCase	,      fieldnames=['''col_1''', '''col_2''', '''col_3''']      )
    writer.writeheader()
    for item in DATA:
      writer.writerow(UpperCamelCase      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	List[Any]:
  lowerCAmelCase__				:  Dict		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''dataset2.csv'''      )
  with open(UpperCamelCase	,      '''w'''	,      newline=''''''      ) as f:
    lowerCAmelCase__				:  Tuple		       =     csv.DictWriter(UpperCamelCase	,      fieldnames=['''col_1''', '''col_2''', '''col_3''']      )
    writer.writeheader()
    for item in DATA:
      writer.writerow(UpperCamelCase      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase      )     ->	List[str]:
  import bza

  lowerCAmelCase__				:  Optional[int]		       =     tmp_path_factory.mktemp('''data'''      ) / '''dataset.csv.bz2'''
  with open(UpperCamelCase	,      '''rb'''      ) as f:
    lowerCAmelCase__				:  List[str]		       =     f.read()
  # data = bytes(FILE_CONTENT, "utf-8")
  with bza.open(UpperCamelCase	,      '''wb'''      ) as f:
    f.write(UpperCamelCase      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase	,      UpperCamelCase      )     ->	Tuple:
  lowerCAmelCase__				:  Optional[int]		       =     tmp_path_factory.mktemp('''data'''      ) / '''dataset.csv.zip'''
  with zipfile.ZipFile(UpperCamelCase	,      '''w'''      ) as f:
    f.write(UpperCamelCase	,      arcname=os.path.basename(UpperCamelCase      )      )
    f.write(UpperCamelCase	,      arcname=os.path.basename(UpperCamelCase      )      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase	,      UpperCamelCase      )     ->	Tuple:
  lowerCAmelCase__				:  Union[str, Any]		       =     tmp_path_factory.mktemp('''data'''      ) / '''dataset.csv.zip'''
  with zipfile.ZipFile(UpperCamelCase	,      '''w'''      ) as f:
    f.write(UpperCamelCase	,      arcname=os.path.basename(csv_path.replace('''.csv'''	,      '''.CSV'''      )      )      )
    f.write(UpperCamelCase	,      arcname=os.path.basename(csva_path.replace('''.csv'''	,      '''.CSV'''      )      )      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase	,      UpperCamelCase      )     ->	str:
  lowerCAmelCase__				:  Union[str, Any]		       =     tmp_path_factory.mktemp('''data'''      ) / '''dataset_with_dir.csv.zip'''
  with zipfile.ZipFile(UpperCamelCase	,      '''w'''      ) as f:
    f.write(UpperCamelCase	,      arcname=os.path.join('''main_dir'''	,      os.path.basename(UpperCamelCase      )      )      )
    f.write(UpperCamelCase	,      arcname=os.path.join('''main_dir'''	,      os.path.basename(UpperCamelCase      )      )      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Dict:
  lowerCAmelCase__				:  str		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''dataset.parquet'''      )
  lowerCAmelCase__				:  List[Any]		       =     pa.schema(
      {
          '''col_1''': pa.string(),
          '''col_2''': pa.intaa(),
          '''col_3''': pa.floataa(),
      }      )
  with open(UpperCamelCase	,      '''wb'''      ) as f:
    lowerCAmelCase__				:  str		       =     pq.ParquetWriter(UpperCamelCase	,      schema=UpperCamelCase      )
    lowerCAmelCase__				:  List[Any]		       =     pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(UpperCamelCase      )      )] for k in DATA[0]}	,      schema=UpperCamelCase      )
    writer.write_table(UpperCamelCase      )
    writer.close()
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Dict:
  lowerCAmelCase__				:  List[Any]		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''dataset.json'''      )
  lowerCAmelCase__				:  List[str]		       =     {'''data''': DATA}
  with open(UpperCamelCase	,      '''w'''      ) as f:
    json.dump(UpperCamelCase	,      UpperCamelCase      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Tuple:
  lowerCAmelCase__				:  Tuple		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''dataset.json'''      )
  lowerCAmelCase__				:  Optional[Any]		       =     {'''data''': DATA_DICT_OF_LISTS}
  with open(UpperCamelCase	,      '''w'''      ) as f:
    json.dump(UpperCamelCase	,      UpperCamelCase      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	int:
  lowerCAmelCase__				:  Union[str, Any]		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''dataset.jsonl'''      )
  with open(UpperCamelCase	,      '''w'''      ) as f:
    for item in DATA:
      f.write(json.dumps(UpperCamelCase      ) + '''\n'''      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	str:
  lowerCAmelCase__				:  Optional[int]		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''dataset2.jsonl'''      )
  with open(UpperCamelCase	,      '''w'''      ) as f:
    for item in DATA:
      f.write(json.dumps(UpperCamelCase      ) + '''\n'''      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Optional[Any]:
  lowerCAmelCase__				:  Dict		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''dataset_312.jsonl'''      )
  with open(UpperCamelCase	,      '''w'''      ) as f:
    for item in DATA_312:
      f.write(json.dumps(UpperCamelCase      ) + '''\n'''      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	List[Any]:
  lowerCAmelCase__				:  Optional[int]		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''dataset-str.jsonl'''      )
  with open(UpperCamelCase	,      '''w'''      ) as f:
    for item in DATA_STR:
      f.write(json.dumps(UpperCamelCase      ) + '''\n'''      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase      )     ->	int:
  import gzip

  lowerCAmelCase__				:  Dict		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''dataset.txt.gz'''      )
  with open(UpperCamelCase	,      '''rb'''      ) as orig_file:
    with gzip.open(UpperCamelCase	,      '''wb'''      ) as zipped_file:
      zipped_file.writelines(UpperCamelCase      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase      )     ->	List[Any]:
  import gzip

  lowerCAmelCase__				:  int		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''dataset.jsonl.gz'''      )
  with open(UpperCamelCase	,      '''rb'''      ) as orig_file:
    with gzip.open(UpperCamelCase	,      '''wb'''      ) as zipped_file:
      zipped_file.writelines(UpperCamelCase      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase	,      UpperCamelCase      )     ->	Union[str, Any]:
  lowerCAmelCase__				:  str		       =     tmp_path_factory.mktemp('''data'''      ) / '''dataset.jsonl.zip'''
  with zipfile.ZipFile(UpperCamelCase	,      '''w'''      ) as f:
    f.write(UpperCamelCase	,      arcname=os.path.basename(UpperCamelCase      )      )
    f.write(UpperCamelCase	,      arcname=os.path.basename(UpperCamelCase      )      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase	,      UpperCamelCase	,      UpperCamelCase      )     ->	Optional[int]:
  lowerCAmelCase__				:  List[Any]		       =     tmp_path_factory.mktemp('''data'''      ) / '''dataset_nested.jsonl.zip'''
  with zipfile.ZipFile(UpperCamelCase	,      '''w'''      ) as f:
    f.write(UpperCamelCase	,      arcname=os.path.join('''nested'''	,      os.path.basename(UpperCamelCase      )      )      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase	,      UpperCamelCase      )     ->	Optional[Any]:
  lowerCAmelCase__				:  Tuple		       =     tmp_path_factory.mktemp('''data'''      ) / '''dataset_with_dir.jsonl.zip'''
  with zipfile.ZipFile(UpperCamelCase	,      '''w'''      ) as f:
    f.write(UpperCamelCase	,      arcname=os.path.join('''main_dir'''	,      os.path.basename(UpperCamelCase      )      )      )
    f.write(UpperCamelCase	,      arcname=os.path.join('''main_dir'''	,      os.path.basename(UpperCamelCase      )      )      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase	,      UpperCamelCase      )     ->	int:
  lowerCAmelCase__				:  Optional[Any]		       =     tmp_path_factory.mktemp('''data'''      ) / '''dataset.jsonl.tar'''
  with tarfile.TarFile(UpperCamelCase	,      '''w'''      ) as f:
    f.add(UpperCamelCase	,      arcname=os.path.basename(UpperCamelCase      )      )
    f.add(UpperCamelCase	,      arcname=os.path.basename(UpperCamelCase      )      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase	,      UpperCamelCase	,      UpperCamelCase      )     ->	Optional[int]:
  lowerCAmelCase__				:  Union[str, Any]		       =     tmp_path_factory.mktemp('''data'''      ) / '''dataset_nested.jsonl.tar'''
  with tarfile.TarFile(UpperCamelCase	,      '''w'''      ) as f:
    f.add(UpperCamelCase	,      arcname=os.path.join('''nested'''	,      os.path.basename(UpperCamelCase      )      )      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Optional[Any]:
  lowerCAmelCase__				:  Dict		       =     ['''0''', '''1''', '''2''', '''3''']
  lowerCAmelCase__				:  Any		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''dataset.txt'''      )
  with open(UpperCamelCase	,      '''w'''      ) as f:
    for item in data:
      f.write(item + '''\n'''      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Any:
  lowerCAmelCase__				:  int		       =     ['''0''', '''1''', '''2''', '''3''']
  lowerCAmelCase__				:  int		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''dataset2.txt'''      )
  with open(UpperCamelCase	,      '''w'''      ) as f:
    for item in data:
      f.write(item + '''\n'''      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Optional[int]:
  lowerCAmelCase__				:  Optional[Any]		       =     ['''0''', '''1''', '''2''', '''3''']
  lowerCAmelCase__				:  str		       =     tmp_path_factory.mktemp('''data'''      ) / '''dataset.abc'''
  with open(UpperCamelCase	,      '''w'''      ) as f:
    for item in data:
      f.write(item + '''\n'''      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase	,      UpperCamelCase      )     ->	Any:
  lowerCAmelCase__				:  Optional[int]		       =     tmp_path_factory.mktemp('''data'''      ) / '''dataset.text.zip'''
  with zipfile.ZipFile(UpperCamelCase	,      '''w'''      ) as f:
    f.write(UpperCamelCase	,      arcname=os.path.basename(UpperCamelCase      )      )
    f.write(UpperCamelCase	,      arcname=os.path.basename(UpperCamelCase      )      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase	,      UpperCamelCase      )     ->	Union[str, Any]:
  lowerCAmelCase__				:  Any		       =     tmp_path_factory.mktemp('''data'''      ) / '''dataset_with_dir.text.zip'''
  with zipfile.ZipFile(UpperCamelCase	,      '''w'''      ) as f:
    f.write(UpperCamelCase	,      arcname=os.path.join('''main_dir'''	,      os.path.basename(UpperCamelCase      )      )      )
    f.write(UpperCamelCase	,      arcname=os.path.join('''main_dir'''	,      os.path.basename(UpperCamelCase      )      )      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase	,      UpperCamelCase      )     ->	int:
  lowerCAmelCase__				:  str		       =     tmp_path_factory.mktemp('''data'''      ) / '''dataset.ext.zip'''
  with zipfile.ZipFile(UpperCamelCase	,      '''w'''      ) as f:
    f.write(UpperCamelCase	,      arcname=os.path.basename('''unsupported.ext'''      )      )
    f.write(UpperCamelCase	,      arcname=os.path.basename('''unsupported_2.ext'''      )      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Dict:
  lowerCAmelCase__				:  List[Any]		       =     '''\n'''.join(['''First''', '''Second\u2029with Unicode new line''', '''Third''']      )
  lowerCAmelCase__				:  Union[str, Any]		       =     str(tmp_path_factory.mktemp('''data'''      ) / '''dataset_with_unicode_new_lines.txt'''      )
  with open(UpperCamelCase	,      '''w'''	,      encoding='''utf-8'''      ) as f:
    f.write(UpperCamelCase      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					)     ->	Optional[int]:
  return os.path.join('''tests'''	,      '''features'''	,      '''data'''	,      '''test_image_rgb.jpg'''      )



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					)     ->	Optional[Any]:
  return os.path.join('''tests'''	,      '''features'''	,      '''data'''	,      '''test_audio_44100.wav'''      )



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase	,      UpperCamelCase      )     ->	Union[str, Any]:
  lowerCAmelCase__				:  Tuple		       =     tmp_path_factory.mktemp('''data'''      ) / '''dataset.img.zip'''
  with zipfile.ZipFile(UpperCamelCase	,      '''w'''      ) as f:
    f.write(UpperCamelCase	,      arcname=os.path.basename(UpperCamelCase      )      )
    f.write(UpperCamelCase	,      arcname=os.path.basename(UpperCamelCase      ).replace('''.jpg'''	,      '''2.jpg'''      )      )
  return path



@pytest.fixture(scope='''session'''      )
def      __lowerCAmelCase			(					UpperCamelCase      )     ->	Dict:
  lowerCAmelCase__				:  Optional[Any]		       =     tmp_path_factory.mktemp('''data_dir'''      )

  (data_dir / "subdir").mkdir()
  with open(data_dir / '''subdir''' / '''train.txt'''	,      '''w'''      ) as f:
    f.write('''foo\n''' * 10      )
  with open(data_dir / '''subdir''' / '''test.txt'''	,      '''w'''      ) as f:
    f.write('''bar\n''' * 10      )
  # hidden file
  with open(data_dir / '''subdir''' / '''.test.txt'''	,      '''w'''      ) as f:
    f.write('''bar\n''' * 10      )

  # hidden directory
  (data_dir / ".subdir").mkdir()
  with open(data_dir / '''.subdir''' / '''train.txt'''	,      '''w'''      ) as f:
    f.write('''foo\n''' * 10      )
  with open(data_dir / '''.subdir''' / '''test.txt'''	,      '''w'''      ) as f:
    f.write('''bar\n''' * 10      )

  return data_dir

