
import itertools
from dataclasses import dataclass
from typing import List, Optional

import pyarrow as pa
import pyarrow.parquet as pq

import datasets
from datasets.table import table_cast


lowerCamelCase__    :		Optional[int]          =   datasets.utils.logging.get_logger(__name__)



@dataclass
class       _snake_case   (	datasets.BuilderConfig							):
 __lowerCAmelCase  :     int        =			10_000
 __lowerCAmelCase  :     Optional[List[str]]        =			None
 __lowerCAmelCase  :     Optional[datasets.Features]        =			None



class       _snake_case   (	datasets.ArrowBasedBuilder							):
 __lowerCAmelCase  :     Any        =			ParquetConfig


 def      lowercase__     (					self):


   '''simple docstring'''




   return datasets.DatasetInfo(features=self.config.features)


 def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   if not self.config.data_files:
     raise ValueError(f'At least one data file must be specified, but got data_files={self.config.data_files}')
   lowercase__  :   List[Any]    		=      dl_manager.download_and_extract(self.config.data_files)
   if isinstance(SCREAMING_SNAKE_CASE_					,  (str, list, tuple)):
     lowercase__  :   str    		=      data_files
     if isinstance(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_):
       lowercase__  :   str    		=      [files]
     # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
     lowercase__  :   Optional[Any]    		=      [dl_manager.iter_files(SCREAMING_SNAKE_CASE_) for file in files]
     return [datasets.SplitGenerator(name=datasets.Split.TRAIN					,  gen_kwargs={"""files""": files})]
   lowercase__  :   int    		=      []
   for split_name, files in data_files.items():
     if isinstance(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_):
       lowercase__  :   Optional[int]    		=      [files]
     # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
     lowercase__  :   List[str]    		=      [dl_manager.iter_files(SCREAMING_SNAKE_CASE_) for file in files]
     # Infer features is they are stoed in the arrow schema
     if self.info.features is None:
       for file in itertools.chain.from_iterable(SCREAMING_SNAKE_CASE_):
         with open(SCREAMING_SNAKE_CASE_					,  """rb""") as f:
           lowercase__  :   Optional[Any]    		=      datasets.Features.from_arrow_schema(pq.read_schema(SCREAMING_SNAKE_CASE_))
         break
     splits.append(datasets.SplitGenerator(name=SCREAMING_SNAKE_CASE_					,  gen_kwargs={"""files""": files}))
   return splits


 def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   if self.info.features is not None:
     # more expensive cast to support nested features with keys in a different order
     # allows str <-> int/float or str to Audio for example
     lowercase__  :   List[Any]    		=      table_cast(SCREAMING_SNAKE_CASE_					,  self.info.features.arrow_schema)
   return pa_table



 def      lowercase__     (					self					,  SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   lowercase__  :   int    		=      self.info.features.arrow_schema if self.info.features is not None else None
   if self.info.features is not None and self.config.columns is not None:
     if sorted(field.name for field in schema) != sorted(self.config.columns):
       raise ValueError(
           f'Tried to load parquet data with columns \'{self.config.columns}\' with mismatching features \'{self.info.features}\'')
   for file_idx, file in enumerate(itertools.chain.from_iterable(SCREAMING_SNAKE_CASE_)):
     with open(SCREAMING_SNAKE_CASE_					,  """rb""") as f:
       lowercase__  :   int    		=      pq.ParquetFile(SCREAMING_SNAKE_CASE_)
       try:
         for batch_idx, record_batch in enumerate(
             parquet_file.iter_batches(batch_size=self.config.batch_size					,  columns=self.config.columns)):
           lowercase__  :   Tuple    		=      pa.Table.from_batches([record_batch])
           # Uncomment for debugging (will print the Arrow table size and elements)
           # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
           # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
           yield f'{file_idx}_{batch_idx}', self._cast_table(SCREAMING_SNAKE_CASE_)
       except ValueError as e:
         logger.error(f'Failed to read file \'{file}\' with error {type(SCREAMING_SNAKE_CASE_)}: {e}')
         raise








import argparse
import os

import numpy as np
import tensorflow as tf
import torch

from transformers import BertModel

def 		UpperCamelCase	(      lowercase_ , lowercase_ , lowercase_   )		->	Dict:


  '''simple docstring'''






  lowercase__  :   Dict    		=      ("""dense.weight""", """attention.self.query""", """attention.self.key""", """attention.self.value""")

  lowercase__  :   Union[str, Any]    		=      (
      ("""layer.""", """layer_"""),
      ("""word_embeddings.weight""", """word_embeddings"""),
      ("""position_embeddings.weight""", """position_embeddings"""),
      ("""token_type_embeddings.weight""", """token_type_embeddings"""),
      (""".""", """/"""),
      ("""LayerNorm/weight""", """LayerNorm/gamma"""),
      ("""LayerNorm/bias""", """LayerNorm/beta"""),
      ("""weight""", """kernel"""),
  )

  if not os.path.isdir(lowercase_   ):
    os.makedirs(lowercase_   )

  lowercase__  :   Optional[Any]    		=      model.state_dict()

  def to_tf_var_name(lowercase_   ):
    for patt, repl in iter(lowercase_   ):
      lowercase__  :   str    		=      name.replace(lowercase_ , lowercase_   )
    return F'bert/{name}'

  def create_tf_var(lowercase_ , lowercase_ , lowercase_   ):
    lowercase__  :   List[str]    		=      tf.dtypes.as_dtype(tensor.dtype   )
    lowercase__  :   Tuple    		=      tf.get_variable(dtype=lowercase_ , shape=tensor.shape , name=lowercase_ , initializer=tf.zeros_initializer()   )
    session.run(tf.variables_initializer([tf_var]   )   )
    session.run(lowercase_   )
    return tf_var

  tf.reset_default_graph()
  with tf.Session() as session:
    for var_name in state_dict:
      lowercase__  :   Any    		=      to_tf_var_name(lowercase_   )
      lowercase__  :   Dict    		=      state_dict[var_name].numpy()
      if any(x in var_name for x in tensors_to_transpose   ):
        lowercase__  :   Optional[int]    		=      torch_tensor.T
      lowercase__  :   Any    		=      create_tf_var(tensor=lowercase_ , name=lowercase_ , session=lowercase_   )
      tf.keras.backend.set_value(lowercase_ , lowercase_   )
      lowercase__  :   Optional[int]    		=      session.run(lowercase_   )
      print(F'Successfully created {tf_name}: {np.allclose(lowercase_ , lowercase_   )}'   )

    lowercase__  :   List[Any]    		=      tf.train.Saver(tf.trainable_variables()   )
    saver.save(lowercase_ , os.path.join(lowercase_ , model_name.replace("""-""" , """_"""   ) + """.ckpt"""   )   )

def 		UpperCamelCase	(      lowercase_=None   )		->	Union[str, Any]:


  '''simple docstring'''






  lowercase__  :   List[str]    		=      argparse.ArgumentParser()
  parser.add_argument("""--model_name""" , type=lowercase_ , required=lowercase_ , help="""model name e.g. bert-base-uncased"""   )
  parser.add_argument(
      """--cache_dir""" , type=lowercase_ , default=lowercase_ , required=lowercase_ , help="""Directory containing pytorch model"""   )
  parser.add_argument("""--pytorch_model_path""" , type=lowercase_ , required=lowercase_ , help="""/path/to/<pytorch-model-name>.bin"""   )
  parser.add_argument("""--tf_cache_dir""" , type=lowercase_ , required=lowercase_ , help="""Directory in which to save tensorflow model"""   )
  lowercase__  :   List[str]    		=      parser.parse_args(lowercase_   )

  lowercase__  :   Any    		=      BertModel.from_pretrained(
      pretrained_model_name_or_path=args.model_name , state_dict=torch.load(args.pytorch_model_path   ) , cache_dir=args.cache_dir , )

  convert_pytorch_checkpoint_to_tf(model=lowercase_ , ckpt_dir=args.tf_cache_dir , model_name=args.model_name   )


if __name__ == "__main__":
     main()






