
'''simple docstring'''






import gzip
import hashlib
import json
import multiprocessing
import os
import re
import shutil
import time
from pathlib import Path

import numpy as np
from arguments import PreprocessingArguments
from datasets import load_dataset
from minhash_deduplication import deduplicate_dataset

from transformers import AutoTokenizer, HfArgumentParser


UpperCAmelCase_       :   Any   =       re.compile(R'\s+')







def 		A_       (	_lowerCAmelCase   :    List[str]							)     ->   Optional[int]:



      """simple docstring"""




      return {"hash": hashlib.mda(re.sub(_lowerCAmelCase				,	""				,	example["content"]							).encode("utf-8"							)							).hexdigest()}







def 		A_       (	_lowerCAmelCase   :    Union[str, Any]							)     ->   List[Any]:



      """simple docstring"""




      _lowerCamelCase				:   Union[str, Any]								=   [len(_lowerCAmelCase							) for line in example["content"].splitlines()]
      return {"line_mean": np.mean(_lowerCAmelCase							), "line_max": max(_lowerCAmelCase							)}







def 		A_       (	_lowerCAmelCase   :    int							)     ->   Optional[Any]:



      """simple docstring"""




      _lowerCamelCase				:   Union[str, Any]								=   np.mean([c.isalnum() for c in example["content"]]							)
      return {"alpha_frac": alpha_frac}







def 		A_       (	_lowerCAmelCase   :    str				,	_lowerCAmelCase   :    Tuple							)     ->   Any:



      """simple docstring"""




      if example["hash"] in uniques:
            uniques.remove(example["hash"]							)
            return True
      else:
            return False







def 		A_       (	_lowerCAmelCase   :    Optional[int]				,	_lowerCAmelCase   :    Dict=5							)     ->   Tuple:



      """simple docstring"""




      _lowerCamelCase				:   Optional[int]								=   ["auto-generated", "autogenerated", "automatically generated"]
      _lowerCamelCase				:   Dict								=   example["content"].splitlines()
      for _, line in zip(range(_lowerCAmelCase							)				,	_lowerCAmelCase							):
            for keyword in keywords:
                  if keyword in line.lower():
                        return {"autogenerated": True}
      else:
            return {"autogenerated": False}







def 		A_       (	_lowerCAmelCase   :    List[str]				,	_lowerCAmelCase   :    Optional[Any]=5				,	_lowerCAmelCase   :    List[Any]=0.0_5							)     ->   List[Any]:



      """simple docstring"""




      _lowerCamelCase				:   Union[str, Any]								=   ["unit tests", "test file", "configuration file"]
      _lowerCamelCase				:   int								=   example["content"].splitlines()
      _lowerCamelCase				:   int								=   0
      _lowerCamelCase				:   Optional[Any]								=   0
      # first test
      for _, line in zip(range(_lowerCAmelCase							)				,	_lowerCAmelCase							):
            for keyword in keywords:
                  if keyword in line.lower():
                        return {"config_or_test": True}
    # second test
      _lowerCamelCase				:   Union[str, Any]								=   example["content"].count("\n"							)
      _lowerCamelCase				:   Any								=   int(coeff * nlines							)
      for line in lines:
            count_config += line.lower().count("config"							)
            count_test += line.lower().count("test"							)
            if count_config > threshold or count_test > threshold:
                  return {"config_or_test": True}
      return {"config_or_test": False}







def 		A_       (	_lowerCAmelCase   :    int							)     ->   Dict:



      """simple docstring"""




      _lowerCamelCase				:   str								=   ["def ", "class ", "for ", "while "]
      _lowerCamelCase				:   List[str]								=   example["content"].splitlines()
      for line in lines:
            for keyword in keywords:
                  if keyword in line.lower():
                        return {"has_no_keywords": False}
      return {"has_no_keywords": True}







def 		A_       (	_lowerCAmelCase   :    Dict				,	_lowerCAmelCase   :    int=4							)     ->   Union[str, Any]:



      """simple docstring"""




      _lowerCamelCase				:   List[Any]								=   example["content"].splitlines()
      _lowerCamelCase				:   Union[str, Any]								=   0
      for line in lines:
            counter += line.lower().count("="							)
            if counter > minimum:
                  return {"has_few_assignments": False}
      return {"has_few_assignments": True}







def 		A_       (	_lowerCAmelCase   :    List[str]							)     ->   str:



      """simple docstring"""




      _lowerCamelCase				:   List[Any]								=   tokenizer(example["content"]				,	truncation=_lowerCAmelCase							)["input_ids"]
      _lowerCamelCase				:   str								=   len(example["content"]							) / len(_lowerCAmelCase							)
      return {"ratio": ratio}







def 		A_       (	_lowerCAmelCase   :    Optional[Any]							)     ->   List[str]:



      """simple docstring"""




      _lowerCamelCase				:   Optional[Any]								=   {}
      results.update(get_hash(_lowerCAmelCase							)							)
      results.update(line_stats(_lowerCAmelCase							)							)
      results.update(alpha_stats(_lowerCAmelCase							)							)
      results.update(char_token_ratio(_lowerCAmelCase							)							)
      results.update(is_autogenerated(_lowerCAmelCase							)							)
      results.update(is_config_or_test(_lowerCAmelCase							)							)
      results.update(has_no_keywords(_lowerCAmelCase							)							)
      results.update(has_few_assignments(_lowerCAmelCase							)							)
      return results







def 		A_       (	_lowerCAmelCase   :    Union[str, Any]				,	_lowerCAmelCase   :    Optional[Any]				,	_lowerCAmelCase   :    str							)     ->   int:



      """simple docstring"""




      if not check_uniques(_lowerCAmelCase				,	_lowerCAmelCase							):
            return False
      elif example["autogenerated"]:
            return False
      elif example["line_max"] > args.line_max:
            return False
      elif example["line_mean"] > args.line_mean:
            return False
      elif example["alpha_frac"] < args.alpha_frac:
            return False
      elif example["ratio"] < args.min_token_ratio:
            return False
      elif example["config_or_test"] and np.random.rand() <= args.filter_proba:
            return False
      elif example["has_no_keywords"] and np.random.rand() <= args.filter_proba:
            return False
      elif example["has_few_assignments"]:
            return False
      else:
            return True







def 		A_       (	_lowerCAmelCase   :    int							)     ->   Union[str, Any]:



      """simple docstring"""




      with open(_lowerCAmelCase				,	"rb"							) as f_in:
            with gzip.open(str(_lowerCAmelCase							) + ".gz"				,	"wb"				,	compresslevel=6							) as f_out:
                  shutil.copyfileobj(_lowerCAmelCase				,	_lowerCAmelCase							)
      os.unlink(_lowerCAmelCase							)


# Settings
UpperCAmelCase_       :   Any   =       HfArgumentParser(PreprocessingArguments)
UpperCAmelCase_       :   Optional[int]   =       parser.parse_args()
if args.num_workers is None:
 UpperCAmelCase_       :   str   =       multiprocessing.cpu_count()
UpperCAmelCase_       :   Tuple   =       AutoTokenizer.from_pretrained(args.tokenizer_dir)

# Load dataset
UpperCAmelCase_       :   Dict   =       time.time()
UpperCAmelCase_       :   Any   =       load_dataset(args.dataset_name, split='train')
print(f'''Time to load dataset: {time.time()-t_start:.2f}''')

# Run preprocessing
UpperCAmelCase_       :   Tuple   =       time.time()
UpperCAmelCase_       :   Optional[int]   =       ds.map(preprocess, num_proc=args.num_workers)
print(f'''Time to preprocess dataset: {time.time()-t_start:.2f}''')

# Deduplicate hashes
UpperCAmelCase_       :   Any   =       set(ds.unique('hash'))
UpperCAmelCase_       :   Dict   =       len(uniques) / len(ds)
print(f'''Fraction of duplicates: {1-frac:.2%}''')

# Deduplicate data and apply heuristics
UpperCAmelCase_       :   Optional[int]   =       time.time()
UpperCAmelCase_       :   int   =       ds.filter(filter, fn_kwargs={'uniques': uniques, 'args': args})
print(f'''Time to filter dataset: {time.time()-t_start:.2f}''')
print(f'''Size of filtered dataset: {len(ds_filter)}''')

# Deduplicate with minhash and jaccard similarity
if args.near_deduplication:
 UpperCAmelCase_       :   Tuple   =       time.time()
 UpperCAmelCase_       :   List[str]   =       deduplicate_dataset(ds_filter, args.jaccard_threshold)
 print(f'''Time to deduplicate dataset: {time.time()-t_start:.2f}''')
 print(f'''Size of deduplicate dataset: {len(ds_filter)}''')

# Save data in batches of samples_per_file
UpperCAmelCase_       :   List[str]   =       Path(args.output_dir)
output_dir.mkdir(exist_ok=True)

# save duplicate_clusters in the output_dir as artifacts
# not sure it is the right place the save it
if args.near_deduplication:
 with open(output_dir / 'duplicate_clusters.json', 'w') as f:
  json.dump(duplicate_clusters, f)

UpperCAmelCase_       :   List[Any]   =       output_dir / 'data'
data_dir.mkdir(exist_ok=True)

UpperCAmelCase_       :   str   =       time.time()
for file_number, index in enumerate(range(0, len(ds_filter), args.samples_per_file)):
 UpperCAmelCase_       :   Tuple   =       str(data_dir / f'''file-{file_number+1:012}.json''')
 UpperCAmelCase_       :   int   =       min(len(ds_filter), index + args.samples_per_file)
 ds_filter.select(list(range(index, end_index))).to_json(file_path)
 compress_file(file_path)



print(f'''Time to save dataset: {time.time()-t_start:.2f}''')

'''simple docstring'''






import unittest

import numpy as np

from transformers import AlbertConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
 import jax.numpy as jnp

 from transformers.models.albert.modeling_flax_albert import (
     FlaxAlbertForMaskedLM,
     FlaxAlbertForMultipleChoice,
     FlaxAlbertForPreTraining,
     FlaxAlbertForQuestionAnswering,
     FlaxAlbertForSequenceClassification,
     FlaxAlbertForTokenClassification,
     FlaxAlbertModel,
 )

class  UpperCAmelCase__						(       unittest.TestCase ):



    def __init__(	self      :       Union[str, Any],__A      :       Union[str, Any],__A      :       Optional[int]=1_3,__A      :       Optional[Any]=7,__A      :       int=True,__A      :       Any=True,__A      :       Dict=True,__A      :       List[Any]=True,__A      :       Optional[Any]=9_9,__A      :       List[str]=3_2,__A      :       int=5,__A      :       str=4,__A      :       int=3_7,__A      :       Optional[Any]="gelu",__A      :       Union[str, Any]=0.1,__A      :       str=0.1,__A      :       Any=5_1_2,__A      :       Any=1_6,__A      :       Optional[int]=2,__A      :       int=0.02,__A      :       Optional[int]=4,):
          _lowerCamelCase				:   Union[str, Any]								=   parent
          _lowerCamelCase				:   Tuple								=   batch_size
          _lowerCamelCase				:   List[str]								=   seq_length
          _lowerCamelCase				:   Optional[int]								=   is_training
          _lowerCamelCase				:   str								=   use_attention_mask
          _lowerCamelCase				:   str								=   use_token_type_ids
          _lowerCamelCase				:   List[str]								=   use_labels
          _lowerCamelCase				:   Tuple								=   vocab_size
          _lowerCamelCase				:   Dict								=   hidden_size
          _lowerCamelCase				:   str								=   num_hidden_layers
          _lowerCamelCase				:   List[str]								=   num_attention_heads
          _lowerCamelCase				:   Optional[Any]								=   intermediate_size
          _lowerCamelCase				:   Optional[int]								=   hidden_act
          _lowerCamelCase				:   Any								=   hidden_dropout_prob
          _lowerCamelCase				:   List[Any]								=   attention_probs_dropout_prob
          _lowerCamelCase				:   Tuple								=   max_position_embeddings
          _lowerCamelCase				:   str								=   type_vocab_size
          _lowerCamelCase				:   str								=   type_sequence_label_size
          _lowerCamelCase				:   int								=   initializer_range
          _lowerCamelCase				:   List[Any]								=   num_choices



    def   lowerCamelCase_  (	self      :       Optional[int]     ):
          _lowerCamelCase				:   List[Any]								=   ids_tensor([self.batch_size, self.seq_length],self.vocab_size     )

          _lowerCamelCase				:   Union[str, Any]								=   None
          if self.use_attention_mask:
                _lowerCamelCase				:   Tuple								=   random_attention_mask([self.batch_size, self.seq_length]     )

          _lowerCamelCase				:   Any								=   None
          if self.use_token_type_ids:
                _lowerCamelCase				:   int								=   ids_tensor([self.batch_size, self.seq_length],self.type_vocab_size     )

          _lowerCamelCase				:   int								=   AlbertConfig(
              vocab_size=self.vocab_size,hidden_size=self.hidden_size,num_hidden_layers=self.num_hidden_layers,num_attention_heads=self.num_attention_heads,intermediate_size=self.intermediate_size,hidden_act=self.hidden_act,hidden_dropout_prob=self.hidden_dropout_prob,attention_probs_dropout_prob=self.attention_probs_dropout_prob,max_position_embeddings=self.max_position_embeddings,type_vocab_size=self.type_vocab_size,is_decoder=__A,initializer_range=self.initializer_range,)

          return config, input_ids, token_type_ids, attention_mask



    def   lowerCamelCase_  (	self      :       Dict     ):
          _lowerCamelCase				:   str								=   self.prepare_config_and_inputs()
          _lowerCamelCase						,		_lowerCamelCase						,		_lowerCamelCase						,		_lowerCamelCase				:   Union[str, Any]								=   config_and_inputs
          _lowerCamelCase				:   Any								=   {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": attention_mask}
          return config, inputs_dict


@require_flax
class  UpperCAmelCase__						(       A						,	unittest.TestCase ):
    lowerCAmelCase_				    =		(
        (
            FlaxAlbertModel,
            FlaxAlbertForPreTraining,
            FlaxAlbertForMaskedLM,
            FlaxAlbertForMultipleChoice,
            FlaxAlbertForQuestionAnswering,
            FlaxAlbertForSequenceClassification,
            FlaxAlbertForTokenClassification,
            FlaxAlbertForQuestionAnswering,
        )
        if is_flax_available()
        else ()
    )



    def   lowerCamelCase_  (	self      :       Any     ):
          _lowerCamelCase				:   List[str]								=   FlaxAlbertModelTester(self     )



    @slow
    def   lowerCamelCase_  (	self      :       Union[str, Any]     ):
          for model_class_name in self.all_model_classes:
                _lowerCamelCase				:   str								=   model_class_name.from_pretrained("albert-base-v2"     )
                _lowerCamelCase				:   Any								=   model(np.ones((1, 1)     )     )
                self.assertIsNotNone(__A     )


@require_flax
class  UpperCAmelCase__						(       unittest.TestCase ):



    @slow
    def   lowerCamelCase_  (	self      :       Tuple     ):
          _lowerCamelCase				:   Optional[int]								=   FlaxAlbertModel.from_pretrained("albert-base-v2"     )
          _lowerCamelCase				:   int								=   np.array([[0, 3_4_5, 2_3_2, 3_2_8, 7_4_0, 1_4_0, 1_6_9_5, 6_9, 6_0_7_8, 1_5_8_8, 2]]     )
          _lowerCamelCase				:   Union[str, Any]								=   np.array([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]     )
          _lowerCamelCase				:   Optional[Any]								=   model(__A,attention_mask=__A     )[0]
          _lowerCamelCase				:   Dict								=   (1, 1_1, 7_6_8)
          self.assertEqual(output.shape,__A     )
          _lowerCamelCase				:   Tuple								=   np.array(
              [[[-0.6513, 1.5035, -0.2766], [-0.6515, 1.5046, -0.2780], [-0.6512, 1.5049, -0.2784]]]     )

          self.assertTrue(jnp.allclose(output[:, 1:4, 1:4],__A,atol=1e-4     )     )