
'''simple docstring'''






import gc
import unittest

import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTextModelWithProjection, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMScheduler,
    DDPMScheduler,
    PriorTransformer,
    StableUnCLIPPipeline,
    UNetaDConditionModel,
)
from diffusers.pipelines.stable_diffusion.stable_unclip_image_normalizer import StableUnCLIPImageNormalizer
from diffusers.utils.testing_utils import enable_full_determinism, load_numpy, require_torch_gpu, slow, torch_device

from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_IMAGE_PARAMS, TEXT_TO_IMAGE_PARAMS
from ..test_pipelines_common import (
    PipelineKarrasSchedulerTesterMixin,
    PipelineLatentTesterMixin,
    PipelineTesterMixin,
    assert_mean_pixel_difference,
)


enable_full_determinism()

class  UpperCAmelCase__						(       A						,	A						,	A						,	unittest.TestCase ):
    lowerCAmelCase_				    =		StableUnCLIPPipeline
    lowerCAmelCase_				    =		TEXT_TO_IMAGE_PARAMS
    lowerCAmelCase_				    =		TEXT_TO_IMAGE_BATCH_PARAMS
    lowerCAmelCase_				    =		TEXT_TO_IMAGE_IMAGE_PARAMS
    lowerCAmelCase_				    =		TEXT_TO_IMAGE_IMAGE_PARAMS

    # TODO(will) Expected attn_bias.stride(1) == 0 to be true, but got false
    lowerCAmelCase_				    =		False



    def   lowerCamelCase_  (	self      :       str     ):
          _lowerCamelCase				:   List[str]								=   3_2
          _lowerCamelCase				:   Dict								=   embedder_hidden_size

          # prior components

          torch.manual_seed(0     )
          _lowerCamelCase				:   Optional[Any]								=   CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip"     )

          torch.manual_seed(0     )
          _lowerCamelCase				:   List[str]								=   CLIPTextModelWithProjection(
              CLIPTextConfig(
                  bos_token_id=0,eos_token_id=2,hidden_size=__A,projection_dim=__A,intermediate_size=3_7,layer_norm_eps=1e-05,num_attention_heads=4,num_hidden_layers=5,pad_token_id=1,vocab_size=1_0_0_0,)     )

          torch.manual_seed(0     )
          _lowerCamelCase				:   int								=   PriorTransformer(
              num_attention_heads=2,attention_head_dim=1_2,embedding_dim=__A,num_layers=1,)

          torch.manual_seed(0     )
          _lowerCamelCase				:   List[Any]								=   DDPMScheduler(
              variance_type="fixed_small_log",prediction_type="sample",num_train_timesteps=1_0_0_0,clip_sample=__A,clip_sample_range=5.0,beta_schedule="squaredcos_cap_v2",)

          # regular denoising components

          torch.manual_seed(0     )
          _lowerCamelCase				:   Union[str, Any]								=   StableUnCLIPImageNormalizer(embedding_dim=__A     )
          _lowerCamelCase				:   str								=   DDPMScheduler(beta_schedule="squaredcos_cap_v2"     )

          torch.manual_seed(0     )
          _lowerCamelCase				:   Any								=   CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip"     )

          torch.manual_seed(0     )
          _lowerCamelCase				:   Any								=   CLIPTextModel(
              CLIPTextConfig(
                  bos_token_id=0,eos_token_id=2,hidden_size=__A,projection_dim=3_2,intermediate_size=3_7,layer_norm_eps=1e-05,num_attention_heads=4,num_hidden_layers=5,pad_token_id=1,vocab_size=1_0_0_0,)     )

          torch.manual_seed(0     )
          _lowerCamelCase				:   List[Any]								=   UNetaDConditionModel(
              sample_size=3_2,in_channels=4,out_channels=4,down_block_types=("CrossAttnDownBlock2D", "DownBlock2D"),up_block_types=("UpBlock2D", "CrossAttnUpBlock2D"),block_out_channels=(3_2, 6_4),attention_head_dim=(2, 4),class_embed_type="projection",projection_class_embeddings_input_dim=embedder_projection_dim * 2,cross_attention_dim=__A,layers_per_block=1,upcast_attention=__A,use_linear_projection=__A,)

          torch.manual_seed(0     )
          _lowerCamelCase				:   List[Any]								=   DDIMScheduler(
              beta_schedule="scaled_linear",beta_start=0.00085,beta_end=0.012,prediction_type="v_prediction",set_alpha_to_one=__A,steps_offset=1,)

          torch.manual_seed(0     )
          _lowerCamelCase				:   Union[str, Any]								=   AutoencoderKL()

          _lowerCamelCase				:   str								=   {
              # prior components
              "prior_tokenizer": prior_tokenizer,
              "prior_text_encoder": prior_text_encoder,
              "prior": prior,
              "prior_scheduler": prior_scheduler,
              # image noising components
              "image_normalizer": image_normalizer,
              "image_noising_scheduler": image_noising_scheduler,
              # regular denoising components
              "tokenizer": tokenizer,
              "text_encoder": text_encoder,
              "unet": unet,
              "scheduler": scheduler,
              "vae": vae,
          }

          return components



    def   lowerCamelCase_  (	self      :       List[str],__A      :       Any,__A      :       List[str]=0     ):
          if str(__A     ).startswith("mps"     ):
                _lowerCamelCase				:   List[str]								=   torch.manual_seed(__A     )
          else:
                _lowerCamelCase				:   List[Any]								=   torch.Generator(device=__A     ).manual_seed(__A     )
          _lowerCamelCase				:   List[str]								=   {
              "prompt": "A painting of a squirrel eating a burger",
              "generator": generator,
              "num_inference_steps": 2,
              "prior_num_inference_steps": 2,
              "output_type": "numpy",
          }
          return inputs



    def   lowerCamelCase_  (	self      :       Any     ):
          _lowerCamelCase				:   int								=   torch_device == "cpu"

          self._test_attention_slicing_forward_pass(test_max_difference=__A     )



    def   lowerCamelCase_  (	self      :       int     ):
          _lowerCamelCase				:   int								=   torch_device in ["cpu", "mps"]

          self._test_inference_batch_single_identical(test_max_difference=__A     )


@slow
@require_torch_gpu
class  UpperCAmelCase__						(       unittest.TestCase ):



    def   lowerCamelCase_  (	self      :       Any     ):
          # clean up the VRAM after each test
          super().tearDown()
          gc.collect()
          torch.cuda.empty_cache()



    def   lowerCamelCase_  (	self      :       str     ):
          _lowerCamelCase				:   Union[str, Any]								=   load_numpy(
              "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/stable_unclip/stable_unclip_2_1_l_anime_turtle_fp16.npy"     )

          _lowerCamelCase				:   List[Any]								=   StableUnCLIPPipeline.from_pretrained("fusing/stable-unclip-2-1-l",torch_dtype=torch.floataa     )
          pipe.to(__A     )
          pipe.set_progress_bar_config(disable=__A     )
          # stable unclip will oom when integration tests are run on a V100,
          # so turn on memory savings
          pipe.enable_attention_slicing()
          pipe.enable_sequential_cpu_offload()

          _lowerCamelCase				:   int								=   torch.Generator(device="cpu"     ).manual_seed(0     )
          _lowerCamelCase				:   Tuple								=   pipe("anime turle",generator=__A,output_type="np"     )

          _lowerCamelCase				:   Optional[Any]								=   output.images[0]

          assert image.shape == (7_6_8, 7_6_8, 3)

          assert_mean_pixel_difference(__A,__A     )



    def   lowerCamelCase_  (	self      :       Optional[int]     ):
          torch.cuda.empty_cache()
          torch.cuda.reset_max_memory_allocated()
          torch.cuda.reset_peak_memory_stats()

          _lowerCamelCase				:   Dict								=   StableUnCLIPPipeline.from_pretrained("fusing/stable-unclip-2-1-l",torch_dtype=torch.floataa     )
          _lowerCamelCase				:   Optional[Any]								=   pipe.to(__A     )
          pipe.set_progress_bar_config(disable=__A     )
          pipe.enable_attention_slicing()
          pipe.enable_sequential_cpu_offload()

          _lowerCamelCase				:   List[str]								=   pipe(
              "anime turtle",prior_num_inference_steps=2,num_inference_steps=2,output_type="np",)

          _lowerCamelCase				:   Optional[int]								=   torch.cuda.max_memory_allocated()
          # make sure that less than 7 GB is allocated
          assert mem_bytes < 7 * 1_0**9

'''simple docstring'''






import gzip
import hashlib
import json
import multiprocessing
import os
import re
import shutil
import time
from pathlib import Path

import numpy as np
from arguments import PreprocessingArguments
from datasets import load_dataset
from minhash_deduplication import deduplicate_dataset

from transformers import AutoTokenizer, HfArgumentParser


UpperCAmelCase_       :   Any   =       re.compile(R'\s+')







def 		A_       (	_lowerCAmelCase   :    List[str]							):



      """simple docstring"""




      return {"hash": hashlib.mda(re.sub(_lowerCAmelCase				,	""				,	example["content"]							).encode("utf-8"							)							).hexdigest()}







def 		A_       (	_lowerCAmelCase   :    Union[str, Any]							):



      """simple docstring"""




      _lowerCamelCase				:   Union[str, Any]								=   [len(_lowerCAmelCase							) for line in example["content"].splitlines()]
      return {"line_mean": np.mean(_lowerCAmelCase							), "line_max": max(_lowerCAmelCase							)}







def 		A_       (	_lowerCAmelCase   :    int							):



      """simple docstring"""




      _lowerCamelCase				:   Union[str, Any]								=   np.mean([c.isalnum() for c in example["content"]]							)
      return {"alpha_frac": alpha_frac}







def 		A_       (	_lowerCAmelCase   :    str				,	_lowerCAmelCase   :    Tuple							):



      """simple docstring"""




      if example["hash"] in uniques:
            uniques.remove(example["hash"]							)
            return True
      else:
            return False







def 		A_       (	_lowerCAmelCase   :    Optional[int]				,	_lowerCAmelCase   :    Dict=5							):



      """simple docstring"""




      _lowerCamelCase				:   Optional[int]								=   ["auto-generated", "autogenerated", "automatically generated"]
      _lowerCamelCase				:   Dict								=   example["content"].splitlines()
      for _, line in zip(range(_lowerCAmelCase							)				,	_lowerCAmelCase							):
            for keyword in keywords:
                  if keyword in line.lower():
                        return {"autogenerated": True}
      else:
            return {"autogenerated": False}







def 		A_       (	_lowerCAmelCase   :    List[str]				,	_lowerCAmelCase   :    Optional[Any]=5				,	_lowerCAmelCase   :    List[Any]=0.0_5							):



      """simple docstring"""




      _lowerCamelCase				:   Union[str, Any]								=   ["unit tests", "test file", "configuration file"]
      _lowerCamelCase				:   int								=   example["content"].splitlines()
      _lowerCamelCase				:   int								=   0
      _lowerCamelCase				:   Optional[Any]								=   0
      # first test
      for _, line in zip(range(_lowerCAmelCase							)				,	_lowerCAmelCase							):
            for keyword in keywords:
                  if keyword in line.lower():
                        return {"config_or_test": True}
    # second test
      _lowerCamelCase				:   Union[str, Any]								=   example["content"].count("\n"							)
      _lowerCamelCase				:   Any								=   int(coeff * nlines							)
      for line in lines:
            count_config += line.lower().count("config"							)
            count_test += line.lower().count("test"							)
            if count_config > threshold or count_test > threshold:
                  return {"config_or_test": True}
      return {"config_or_test": False}







def 		A_       (	_lowerCAmelCase   :    int							):



      """simple docstring"""




      _lowerCamelCase				:   str								=   ["def ", "class ", "for ", "while "]
      _lowerCamelCase				:   List[str]								=   example["content"].splitlines()
      for line in lines:
            for keyword in keywords:
                  if keyword in line.lower():
                        return {"has_no_keywords": False}
      return {"has_no_keywords": True}







def 		A_       (	_lowerCAmelCase   :    Dict				,	_lowerCAmelCase   :    int=4							):



      """simple docstring"""




      _lowerCamelCase				:   List[Any]								=   example["content"].splitlines()
      _lowerCamelCase				:   Union[str, Any]								=   0
      for line in lines:
            counter += line.lower().count("="							)
            if counter > minimum:
                  return {"has_few_assignments": False}
      return {"has_few_assignments": True}







def 		A_       (	_lowerCAmelCase   :    List[str]							):



      """simple docstring"""




      _lowerCamelCase				:   List[Any]								=   tokenizer(example["content"]				,	truncation=_lowerCAmelCase							)["input_ids"]
      _lowerCamelCase				:   str								=   len(example["content"]							) / len(_lowerCAmelCase							)
      return {"ratio": ratio}







def 		A_       (	_lowerCAmelCase   :    Optional[Any]							):



      """simple docstring"""




      _lowerCamelCase				:   Optional[Any]								=   {}
      results.update(get_hash(_lowerCAmelCase							)							)
      results.update(line_stats(_lowerCAmelCase							)							)
      results.update(alpha_stats(_lowerCAmelCase							)							)
      results.update(char_token_ratio(_lowerCAmelCase							)							)
      results.update(is_autogenerated(_lowerCAmelCase							)							)
      results.update(is_config_or_test(_lowerCAmelCase							)							)
      results.update(has_no_keywords(_lowerCAmelCase							)							)
      results.update(has_few_assignments(_lowerCAmelCase							)							)
      return results







def 		A_       (	_lowerCAmelCase   :    Union[str, Any]				,	_lowerCAmelCase   :    Optional[Any]				,	_lowerCAmelCase   :    str							):



      """simple docstring"""




      if not check_uniques(_lowerCAmelCase				,	_lowerCAmelCase							):
            return False
      elif example["autogenerated"]:
            return False
      elif example["line_max"] > args.line_max:
            return False
      elif example["line_mean"] > args.line_mean:
            return False
      elif example["alpha_frac"] < args.alpha_frac:
            return False
      elif example["ratio"] < args.min_token_ratio:
            return False
      elif example["config_or_test"] and np.random.rand() <= args.filter_proba:
            return False
      elif example["has_no_keywords"] and np.random.rand() <= args.filter_proba:
            return False
      elif example["has_few_assignments"]:
            return False
      else:
            return True







def 		A_       (	_lowerCAmelCase   :    int							):



      """simple docstring"""




      with open(_lowerCAmelCase				,	"rb"							) as f_in:
            with gzip.open(str(_lowerCAmelCase							) + ".gz"				,	"wb"				,	compresslevel=6							) as f_out:
                  shutil.copyfileobj(_lowerCAmelCase				,	_lowerCAmelCase							)
      os.unlink(_lowerCAmelCase							)


# Settings
UpperCAmelCase_       :   Any   =       HfArgumentParser(PreprocessingArguments)
UpperCAmelCase_       :   Optional[int]   =       parser.parse_args()
if args.num_workers is None:
 UpperCAmelCase_       :   str   =       multiprocessing.cpu_count()
UpperCAmelCase_       :   Tuple   =       AutoTokenizer.from_pretrained(args.tokenizer_dir)

# Load dataset
UpperCAmelCase_       :   Dict   =       time.time()
UpperCAmelCase_       :   Any   =       load_dataset(args.dataset_name, split='train')
print(f'''Time to load dataset: {time.time()-t_start:.2f}''')

# Run preprocessing
UpperCAmelCase_       :   Tuple   =       time.time()
UpperCAmelCase_       :   Optional[int]   =       ds.map(preprocess, num_proc=args.num_workers)
print(f'''Time to preprocess dataset: {time.time()-t_start:.2f}''')

# Deduplicate hashes
UpperCAmelCase_       :   Any   =       set(ds.unique('hash'))
UpperCAmelCase_       :   Dict   =       len(uniques) / len(ds)
print(f'''Fraction of duplicates: {1-frac:.2%}''')

# Deduplicate data and apply heuristics
UpperCAmelCase_       :   Optional[int]   =       time.time()
UpperCAmelCase_       :   int   =       ds.filter(filter, fn_kwargs={'uniques': uniques, 'args': args})
print(f'''Time to filter dataset: {time.time()-t_start:.2f}''')
print(f'''Size of filtered dataset: {len(ds_filter)}''')

# Deduplicate with minhash and jaccard similarity
if args.near_deduplication:
 UpperCAmelCase_       :   Tuple   =       time.time()
 UpperCAmelCase_,   UpperCAmelCase_       :   List[str]   =       deduplicate_dataset(ds_filter, args.jaccard_threshold)
 print(f'''Time to deduplicate dataset: {time.time()-t_start:.2f}''')
 print(f'''Size of deduplicate dataset: {len(ds_filter)}''')

# Save data in batches of samples_per_file
UpperCAmelCase_       :   List[str]   =       Path(args.output_dir)
output_dir.mkdir(exist_ok=True)

# save duplicate_clusters in the output_dir as artifacts
# not sure it is the right place the save it
if args.near_deduplication:
 with open(output_dir / 'duplicate_clusters.json', 'w') as f:
  json.dump(duplicate_clusters, f)

UpperCAmelCase_       :   List[Any]   =       output_dir / 'data'
data_dir.mkdir(exist_ok=True)

UpperCAmelCase_       :   str   =       time.time()
for file_number, index in enumerate(range(0, len(ds_filter), args.samples_per_file)):
 UpperCAmelCase_       :   Tuple   =       str(data_dir / f'''file-{file_number+1:012}.json''')
 UpperCAmelCase_       :   int   =       min(len(ds_filter), index + args.samples_per_file)
 ds_filter.select(list(range(index, end_index))).to_json(file_path)
 compress_file(file_path)



print(f'''Time to save dataset: {time.time()-t_start:.2f}''')