



import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


__lowerCamelCase     :							Optional[Any]  							=				logging.getLogger(__name__)

def    lowerCamelCase_() ->       Optional[Any]:
						UpperCAmelCase         =     argparse.ArgumentParser(
						    description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."    )
						parser.add_argument(
						    "--dataset_name"    ,   type=lowerCamelCase_    ,   default="wikitext"    ,   help="Name of the training. Explore datasets at: hf.co/datasets."    ,   )
						parser.add_argument(
						    "--dataset_config"    ,   type=lowerCamelCase_    ,   default="wikitext-103-raw-v1"    ,   help="Configuration name of the dataset."    )
						parser.add_argument(
						    "--tokenizer_name_or_path"    ,   type=lowerCamelCase_    ,   default="sayakpaul/unigram-tokenizer-wikitext"    ,   help="Tokenizer identifier. Can be a local filepath or a Hub identifier."    ,   )
						parser.add_argument(
						    "--shard_size"    ,   type=lowerCamelCase_    ,   default=1_000    ,   help="Number of entries to go in a single shard."    ,   )
						parser.add_argument("--split"    ,   type=lowerCamelCase_    ,   default="train"    ,   choices=["train", "test", "validation"]    )
						parser.add_argument(
						    "--limit"    ,   default=lowerCamelCase_    ,   type=lowerCamelCase_    ,   help="Limit the number of shards (used for debugging)."    ,   )
						parser.add_argument(
						    "--max_length"    ,   type=lowerCamelCase_    ,   default=512    ,   help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
						    " sequence length that is a multiple of 8."    ,   )
						parser.add_argument(
						    "--output_dir"    ,   default="tf-tpu"    ,   type=lowerCamelCase_    ,   help="Output directory where the TFRecord shards will be saved. If the"
						    " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
						    " shards will be directly saved to a Google Cloud Storage bucket."    ,   )

						UpperCAmelCase         =     parser.parse_args()
						return args

def    lowerCamelCase_(lowerCamelCase_    ) ->       int:
						def fn(lowerCamelCase_    ):
												return tokenizer(examples["text"]    )

						return fn

def    lowerCamelCase_(lowerCamelCase_    ) ->       List[Any]:
						UpperCAmelCase         =     []
						for i in range(len(tokenized_data["input_ids"]    )    ):
												UpperCAmelCase         =     {
												    "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]    )    ),
												    "attention_mask": tf.train.Feature(
												        intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]    )    ),
												}
												UpperCAmelCase         =     tf.train.Features(feature=lowerCamelCase_    )
												UpperCAmelCase         =     tf.train.Example(features=lowerCamelCase_    )
												UpperCAmelCase         =     example.SerializeToString()
												records.append(lowerCamelCase_    )
						return records

def    lowerCamelCase_(lowerCamelCase_    ) ->       Any:
						UpperCAmelCase         =     datasets.load_dataset(args.dataset_name    ,   args.dataset_config    ,   split=args.split    )

						if args.limit is not None:
												UpperCAmelCase         =     min(len(lowerCamelCase_    )    ,   args.limit    )
												UpperCAmelCase         =     dataset.select(range(lowerCamelCase_    )    )
												print(F'Limiting the dataset to {args.limit} entries.'    )

						UpperCAmelCase         =     AutoTokenizer.from_pretrained(args.tokenizer_name_or_path    )

						# Handle output directory creation.
						# For serializing into a Google Cloud Storage Bucket, one needs to first
						# create a bucket.
						if "gs" not in args.output_dir:
												if not os.path.exists(args.output_dir    ):
																		os.makedirs(args.output_dir    )
												UpperCAmelCase         =     os.path.join(args.output_dir    ,   args.split    )
												if not os.path.exists(lowerCamelCase_    ):
																		os.makedirs(lowerCamelCase_    )
						else:
												UpperCAmelCase         =     os.path.join(args.output_dir    ,   args.split    )

						# Tokenize the whole dataset at once.
						UpperCAmelCase         =     tokenize_function(lowerCamelCase_    )
						UpperCAmelCase         =     dataset.map(lowerCamelCase_    ,   batched=lowerCamelCase_    ,   num_proc=4    ,   remove_columns=["text"]    )

						# We need to concatenate all our texts together, and then split the result
						# into chunks of a fixed size, which we will call block_size. To do this, we
						# will use the map method again, with the option batched=True. When we use batched=True,
						# the function we pass to map() will be passed multiple inputs at once, allowing us
						# to group them into more or fewer examples than we had in the input.
						# This allows us to create our new fixed-length samples. The advantage of this
						# method is that we don't lose a whole lot of content from the dataset compared to the
						# case where we simply tokenize with a pre-defined max_length.

						def group_texts(lowerCamelCase_    ):
												# Concatenate all texts.
												UpperCAmelCase         =     {k: sum(examples[k]    ,   []    ) for k in examples.keys()}
												UpperCAmelCase         =     len(concatenated_examples[list(examples.keys()    )[0]]    )
												# We drop the small remainder, though you could add padding instead if the model supports it
												# In this, as in all things, we advise you to follow your heart 🫀
												UpperCAmelCase         =     (total_length // args.max_length) * args.max_length
												# Split by chunks of max_len.
												UpperCAmelCase         =     {
												    k: [t[i : i + args.max_length] for i in range(0    ,   lowerCamelCase_    ,   args.max_length    )]
												    for k, t in concatenated_examples.items()
												}
												return result

						UpperCAmelCase         =     dataset_tokenized.map(lowerCamelCase_    ,   batched=lowerCamelCase_    ,   batch_size=1_000    ,   num_proc=4    )

						UpperCAmelCase         =     0
						UpperCAmelCase         =     0
						for shard in range(0    ,   len(lowerCamelCase_    )    ,   args.shard_size    ):
												UpperCAmelCase         =     grouped_dataset[shard : shard + args.shard_size]
												UpperCAmelCase         =     len(dataset_snapshot["input_ids"]    )
												UpperCAmelCase         =     os.path.join(lowerCamelCase_    ,   F'dataset-{shard_count}-{records_containing}.tfrecord'    )
												UpperCAmelCase         =     get_serialized_examples(lowerCamelCase_    )

												with tf.io.TFRecordWriter(lowerCamelCase_    ) as out_file:
																		for i in range(len(lowerCamelCase_    )    ):
																								UpperCAmelCase         =     serialized_examples[i]
																								out_file.write(lowerCamelCase_    )
																		print("Wrote file {} containing {} records".format(lowerCamelCase_    ,   lowerCamelCase_    )    )

												shard_count += 1
												total_records += records_containing

						with open(F'split-{args.split}-records-count.txt'    ,   "w"    ) as f:
												print(F'Total {args.split} records: {total_records}'    ,   file=lowerCamelCase_    )


if __name__ == "__main__":
		__lowerCamelCase     :							Dict  							=				parse_args()
		main(args)








from dataclasses import dataclass
from typing import Dict, Optional, Tuple, Union

import torch
import torch.nn as nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput, apply_forward_hook
from .attention_processor import AttentionProcessor, AttnProcessor
from .modeling_utils import ModelMixin
from .vae import Decoder, DecoderOutput, DiagonalGaussianDistribution, Encoder





@dataclass
class  __magic_name__   (							A__						):
		lowercase      :       "DiagonalGaussianDistribution"





class  __magic_name__   (							A__,  A__						):
		lowercase      :       Union[str, Any]          =True






		@register_to_config
		def __init__(						self      :     List[str]				,		UpperCamelCase__      :     int = 3				,		UpperCamelCase__      :     int = 3				,		UpperCamelCase__      :     Tuple[str] = ("DownEncoderBlock2D",)				,		UpperCamelCase__      :     Tuple[str] = ("UpDecoderBlock2D",)				,		UpperCamelCase__      :     Tuple[int] = (64,)				,		UpperCamelCase__      :     int = 1				,		UpperCamelCase__      :     str = "silu"				,		UpperCamelCase__      :     int = 4				,		UpperCamelCase__      :     int = 32				,		UpperCamelCase__      :     int = 32				,		UpperCamelCase__      :     float = 0.1_82_15				,		)				->					List[str]:
								'''simple docstring'''







								super().__init__()

								# pass init params to Encoder
								UpperCAmelCase         =     Encoder(
								    in_channels=UpperCamelCase__				,		out_channels=UpperCamelCase__				,		down_block_types=UpperCamelCase__				,		block_out_channels=UpperCamelCase__				,		layers_per_block=UpperCamelCase__				,		act_fn=UpperCamelCase__				,		norm_num_groups=UpperCamelCase__				,		double_z=UpperCamelCase__				,		)

								# pass init params to Decoder
								UpperCAmelCase         =     Decoder(
								    in_channels=UpperCamelCase__				,		out_channels=UpperCamelCase__				,		up_block_types=UpperCamelCase__				,		block_out_channels=UpperCamelCase__				,		layers_per_block=UpperCamelCase__				,		norm_num_groups=UpperCamelCase__				,		act_fn=UpperCamelCase__				,		)

								UpperCAmelCase         =     nn.Convad(2 * latent_channels				,		2 * latent_channels				,		1      )
								UpperCAmelCase         =     nn.Convad(UpperCamelCase__				,		UpperCamelCase__				,		1      )

								UpperCAmelCase         =     False
								UpperCAmelCase         =     False

								# only relevant if vae tiling is enabled
								UpperCAmelCase         =     self.config.sample_size
								UpperCAmelCase         =     (
								    self.config.sample_size[0]
								    if isinstance(self.config.sample_size				,		(list, tuple)      )
								    else self.config.sample_size
								)
								UpperCAmelCase         =     int(sample_size / (2 ** (len(self.config.block_out_channels      ) - 1))      )
								UpperCAmelCase         =     0.25






		def SCREAMING_SNAKE_CASE_			(						self      :     Dict				,		UpperCamelCase__      :     Optional[int]				,		UpperCamelCase__      :     int=False      )				->					Optional[int]:
								'''simple docstring'''







								if isinstance(UpperCamelCase__				,		(Encoder, Decoder)      ):
														UpperCAmelCase         =     value






		def SCREAMING_SNAKE_CASE_			(						self      :     Optional[int]				,		UpperCamelCase__      :     bool = True      )				->					Tuple:
								'''simple docstring'''







								UpperCAmelCase         =     use_tiling






		def SCREAMING_SNAKE_CASE_			(						self      :     List[Any]      )				->					Tuple:
								'''simple docstring'''







								self.enable_tiling(UpperCamelCase__      )






		def SCREAMING_SNAKE_CASE_			(						self      :     Any      )				->					Tuple:
								'''simple docstring'''







								UpperCAmelCase         =     True






		def SCREAMING_SNAKE_CASE_			(						self      :     Dict      )				->					List[str]:
								'''simple docstring'''







								UpperCAmelCase         =     False






		@property
		# Copied from diffusers.models.unet_2d_condition.UNet2DConditionModel.attn_processors
		def SCREAMING_SNAKE_CASE_			(						self      :     str      )				->					Dict[str, AttentionProcessor]:
								'''simple docstring'''







								UpperCAmelCase         =     {}

								def fn_recursive_add_processors(UpperCamelCase__      :     str				,		UpperCamelCase__      :     torch.nn.Module				,		UpperCamelCase__      :     Dict[str, AttentionProcessor]      ):
														if hasattr(UpperCamelCase__				,		"set_processor"      ):
																				UpperCAmelCase         =     module.processor

														for sub_name, child in module.named_children():
																				fn_recursive_add_processors(F'{name}.{sub_name}'				,		UpperCamelCase__				,		UpperCamelCase__      )

														return processors

								for name, module in self.named_children():
														fn_recursive_add_processors(UpperCamelCase__				,		UpperCamelCase__				,		UpperCamelCase__      )

								return processors






		def SCREAMING_SNAKE_CASE_			(						self      :     Union[str, Any]				,		UpperCamelCase__      :     Union[AttentionProcessor, Dict[str, AttentionProcessor]]      )				->					List[Any]:
								'''simple docstring'''







								UpperCAmelCase         =     len(self.attn_processors.keys()      )

								if isinstance(UpperCamelCase__				,		UpperCamelCase__      ) and len(UpperCamelCase__      ) != count:
														raise ValueError(
														    F'A dict of processors was passed, but the number of processors {len(UpperCamelCase__      )} does not match the'
														    F' number of attention layers: {count}. Please make sure to pass {count} processor classes.'      )

								def fn_recursive_attn_processor(UpperCamelCase__      :     str				,		UpperCamelCase__      :     torch.nn.Module				,		UpperCamelCase__      :     List[Any]      ):
														if hasattr(UpperCamelCase__				,		"set_processor"      ):
																				if not isinstance(UpperCamelCase__				,		UpperCamelCase__      ):
																										module.set_processor(UpperCamelCase__      )
																				else:
																										module.set_processor(processor.pop(F'{name}.processor'      )      )

														for sub_name, child in module.named_children():
																				fn_recursive_attn_processor(F'{name}.{sub_name}'				,		UpperCamelCase__				,		UpperCamelCase__      )

								for name, module in self.named_children():
														fn_recursive_attn_processor(UpperCamelCase__				,		UpperCamelCase__				,		UpperCamelCase__      )






		def SCREAMING_SNAKE_CASE_			(						self      :     Dict      )				->					Dict:
								'''simple docstring'''







								self.set_attn_processor(AttnProcessor()      )






		@apply_forward_hook
		def SCREAMING_SNAKE_CASE_			(						self      :     Any				,		UpperCamelCase__      :     torch.FloatTensor				,		UpperCamelCase__      :     bool = True      )				->					AutoencoderKLOutput:
								'''simple docstring'''







								if self.use_tiling and (x.shape[-1] > self.tile_sample_min_size or x.shape[-2] > self.tile_sample_min_size):
														return self.tiled_encode(UpperCamelCase__				,		return_dict=UpperCamelCase__      )

								if self.use_slicing and x.shape[0] > 1:
														UpperCAmelCase         =     [self.encoder(UpperCamelCase__      ) for x_slice in x.split(1      )]
														UpperCAmelCase         =     torch.cat(UpperCamelCase__      )
								else:
														UpperCAmelCase         =     self.encoder(UpperCamelCase__      )

								UpperCAmelCase         =     self.quant_conv(UpperCamelCase__      )
								UpperCAmelCase         =     DiagonalGaussianDistribution(UpperCamelCase__      )

								if not return_dict:
														return (posterior,)

								return AutoencoderKLOutput(latent_dist=UpperCamelCase__      )






		def SCREAMING_SNAKE_CASE_			(						self      :     List[str]				,		UpperCamelCase__      :     torch.FloatTensor				,		UpperCamelCase__      :     bool = True      )				->					Union[DecoderOutput, torch.FloatTensor]:
								'''simple docstring'''







								if self.use_tiling and (z.shape[-1] > self.tile_latent_min_size or z.shape[-2] > self.tile_latent_min_size):
														return self.tiled_decode(UpperCamelCase__				,		return_dict=UpperCamelCase__      )

								UpperCAmelCase         =     self.post_quant_conv(UpperCamelCase__      )
								UpperCAmelCase         =     self.decoder(UpperCamelCase__      )

								if not return_dict:
														return (dec,)

								return DecoderOutput(sample=UpperCamelCase__      )






		@apply_forward_hook
		def SCREAMING_SNAKE_CASE_			(						self      :     Any				,		UpperCamelCase__      :     torch.FloatTensor				,		UpperCamelCase__      :     bool = True      )				->					Union[DecoderOutput, torch.FloatTensor]:
								'''simple docstring'''







								if self.use_slicing and z.shape[0] > 1:
														UpperCAmelCase         =     [self._decode(UpperCamelCase__      ).sample for z_slice in z.split(1      )]
														UpperCAmelCase         =     torch.cat(UpperCamelCase__      )
								else:
														UpperCAmelCase         =     self._decode(UpperCamelCase__      ).sample

								if not return_dict:
														return (decoded,)

								return DecoderOutput(sample=UpperCamelCase__      )






		def SCREAMING_SNAKE_CASE_			(						self      :     Dict				,		UpperCamelCase__      :     Any				,		UpperCamelCase__      :     Tuple				,		UpperCamelCase__      :     Tuple      )				->					List[Any]:
								'''simple docstring'''







								UpperCAmelCase         =     min(a.shape[2]				,		b.shape[2]				,		UpperCamelCase__      )
								for y in range(UpperCamelCase__      ):
														UpperCAmelCase         =     a[:, :, -blend_extent + y, :] * (1 - y / blend_extent) + b[:, :, y, :] * (y / blend_extent)
								return b






		def SCREAMING_SNAKE_CASE_			(						self      :     str				,		UpperCamelCase__      :     Optional[Any]				,		UpperCamelCase__      :     Union[str, Any]				,		UpperCamelCase__      :     int      )				->					List[str]:
								'''simple docstring'''







								UpperCAmelCase         =     min(a.shape[3]				,		b.shape[3]				,		UpperCamelCase__      )
								for x in range(UpperCamelCase__      ):
														UpperCAmelCase         =     a[:, :, :, -blend_extent + x] * (1 - x / blend_extent) + b[:, :, :, x] * (x / blend_extent)
								return b






		def SCREAMING_SNAKE_CASE_			(						self      :     str				,		UpperCamelCase__      :     torch.FloatTensor				,		UpperCamelCase__      :     bool = True      )				->					AutoencoderKLOutput:
								'''simple docstring'''







								UpperCAmelCase         =     int(self.tile_sample_min_size * (1 - self.tile_overlap_factor)      )
								UpperCAmelCase         =     int(self.tile_latent_min_size * self.tile_overlap_factor      )
								UpperCAmelCase         =     self.tile_latent_min_size - blend_extent

								# Split the image into 512x512 tiles and encode them separately.
								UpperCAmelCase         =     []
								for i in range(0				,		x.shape[2]				,		UpperCamelCase__      ):
														UpperCAmelCase         =     []
														for j in range(0				,		x.shape[3]				,		UpperCamelCase__      ):
																				UpperCAmelCase         =     x[:, :, i : i + self.tile_sample_min_size, j : j + self.tile_sample_min_size]
																				UpperCAmelCase         =     self.encoder(UpperCamelCase__      )
																				UpperCAmelCase         =     self.quant_conv(UpperCamelCase__      )
																				row.append(UpperCamelCase__      )
														rows.append(UpperCamelCase__      )
								UpperCAmelCase         =     []
								for i, row in enumerate(UpperCamelCase__      ):
														UpperCAmelCase         =     []
														for j, tile in enumerate(UpperCamelCase__      ):
																				# blend the above tile and the left tile
																				# to the current tile and add the current tile to the result row
																				if i > 0:
																										UpperCAmelCase         =     self.blend_v(rows[i - 1][j]				,		UpperCamelCase__				,		UpperCamelCase__      )
																				if j > 0:
																										UpperCAmelCase         =     self.blend_h(row[j - 1]				,		UpperCamelCase__				,		UpperCamelCase__      )
																				result_row.append(tile[:, :, :row_limit, :row_limit]      )
														result_rows.append(torch.cat(UpperCamelCase__				,		dim=3      )      )

								UpperCAmelCase         =     torch.cat(UpperCamelCase__				,		dim=2      )
								UpperCAmelCase         =     DiagonalGaussianDistribution(UpperCamelCase__      )

								if not return_dict:
														return (posterior,)

								return AutoencoderKLOutput(latent_dist=UpperCamelCase__      )






		def SCREAMING_SNAKE_CASE_			(						self      :     List[Any]				,		UpperCamelCase__      :     torch.FloatTensor				,		UpperCamelCase__      :     bool = True      )				->					Union[DecoderOutput, torch.FloatTensor]:
								'''simple docstring'''







								UpperCAmelCase         =     int(self.tile_latent_min_size * (1 - self.tile_overlap_factor)      )
								UpperCAmelCase         =     int(self.tile_sample_min_size * self.tile_overlap_factor      )
								UpperCAmelCase         =     self.tile_sample_min_size - blend_extent

								# Split z into overlapping 64x64 tiles and decode them separately.
								# The tiles have an overlap to avoid seams between tiles.
								UpperCAmelCase         =     []
								for i in range(0				,		z.shape[2]				,		UpperCamelCase__      ):
														UpperCAmelCase         =     []
														for j in range(0				,		z.shape[3]				,		UpperCamelCase__      ):
																				UpperCAmelCase         =     z[:, :, i : i + self.tile_latent_min_size, j : j + self.tile_latent_min_size]
																				UpperCAmelCase         =     self.post_quant_conv(UpperCamelCase__      )
																				UpperCAmelCase         =     self.decoder(UpperCamelCase__      )
																				row.append(UpperCamelCase__      )
														rows.append(UpperCamelCase__      )
								UpperCAmelCase         =     []
								for i, row in enumerate(UpperCamelCase__      ):
														UpperCAmelCase         =     []
														for j, tile in enumerate(UpperCamelCase__      ):
																				# blend the above tile and the left tile
																				# to the current tile and add the current tile to the result row
																				if i > 0:
																										UpperCAmelCase         =     self.blend_v(rows[i - 1][j]				,		UpperCamelCase__				,		UpperCamelCase__      )
																				if j > 0:
																										UpperCAmelCase         =     self.blend_h(row[j - 1]				,		UpperCamelCase__				,		UpperCamelCase__      )
																				result_row.append(tile[:, :, :row_limit, :row_limit]      )
														result_rows.append(torch.cat(UpperCamelCase__				,		dim=3      )      )

								UpperCAmelCase         =     torch.cat(UpperCamelCase__				,		dim=2      )
								if not return_dict:
														return (dec,)

								return DecoderOutput(sample=UpperCamelCase__      )







		def SCREAMING_SNAKE_CASE_			(						self      :     List[str]				,		UpperCamelCase__      :     torch.FloatTensor				,		UpperCamelCase__      :     bool = False				,		UpperCamelCase__      :     bool = True				,		UpperCamelCase__      :     Optional[torch.Generator] = None				,		)				->					Union[DecoderOutput, torch.FloatTensor]:
								'''simple docstring'''







								UpperCAmelCase         =     sample
								UpperCAmelCase         =     self.encode(UpperCamelCase__      ).latent_dist
								if sample_posterior:
														UpperCAmelCase         =     posterior.sample(generator=UpperCamelCase__      )
								else:
														UpperCAmelCase         =     posterior.mode()
								UpperCAmelCase         =     self.decode(UpperCamelCase__      ).sample

								if not return_dict:
														return (dec,)

								return DecoderOutput(sample=UpperCamelCase__      )



