



"""simple docstring"""





import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


lowercase  							=					logging.getLogger(__name__)




def 	UpperCAmelCase     (		):

		'''simple docstring'''


		_UpperCAmelCase							 =		argparse.ArgumentParser(
		    description='Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'       )
		parser.add_argument(
		    '--dataset_name'     ,				type=A     ,				default='wikitext'     ,				help='Name of the training. Explore datasets at: hf.co/datasets.'     ,				)
		parser.add_argument(
		    '--dataset_config'     ,				type=A     ,				default='wikitext-103-raw-v1'     ,				help='Configuration name of the dataset.'       )
		parser.add_argument(
		    '--tokenizer_name_or_path'     ,				type=A     ,				default='sayakpaul/unigram-tokenizer-wikitext'     ,				help='Tokenizer identifier. Can be a local filepath or a Hub identifier.'     ,				)
		parser.add_argument(
		    '--shard_size'     ,				type=A     ,				default=1000     ,				help='Number of entries to go in a single shard.'     ,				)
		parser.add_argument('--split'     ,				type=A     ,				default='train'     ,				choices=['train', 'test', 'validation']       )
		parser.add_argument(
		    '--limit'     ,				default=A     ,				type=A     ,				help='Limit the number of shards (used for debugging).'     ,				)
		parser.add_argument(
		    '--max_length'     ,				type=A     ,				default=512     ,				help='Maximum sequence length. For training on TPUs, it helps to have a maximum'
		    ' sequence length that is a multiple of 8.'     ,				)
		parser.add_argument(
		    '--output_dir'     ,				default='tf-tpu'     ,				type=A     ,				help='Output directory where the TFRecord shards will be saved. If the'
		    ' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'
		    ' shards will be directly saved to a Google Cloud Storage bucket.'     ,				)

		_UpperCAmelCase							 =		parser.parse_args()
		return args




def 	UpperCAmelCase     (		A	:						Tuple       ):

		'''simple docstring'''


		def fn(A	:						int       ):
				return tokenizer(examples['text']       )

		return fn




def 	UpperCAmelCase     (		A	:						int       ):

		'''simple docstring'''


		_UpperCAmelCase							 =		[]
		for i in range(len(tokenized_data['input_ids']       )       ):
				_UpperCAmelCase							 =		{
				    'input_ids': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['input_ids'][i]       )       ),
				    'attention_mask': tf.train.Feature(
				        intaa_list=tf.train.IntaaList(value=tokenized_data['attention_mask'][i]       )       ),
				}
				_UpperCAmelCase							 =		tf.train.Features(feature=A       )
				_UpperCAmelCase							 =		tf.train.Example(features=A       )
				_UpperCAmelCase							 =		example.SerializeToString()
				records.append(A       )
		return records




def 	UpperCAmelCase     (		A	:						Union[str, Any]       ):

		'''simple docstring'''


		_UpperCAmelCase							 =		datasets.load_dataset(args.dataset_name     ,				args.dataset_config     ,				split=args.split       )

		if args.limit is not None:
				_UpperCAmelCase							 =		min(len(A       )     ,				args.limit       )
				_UpperCAmelCase							 =		dataset.select(range(A       )       )
				print(f'Limiting the dataset to {args.limit} entries.'       )

		_UpperCAmelCase							 =		AutoTokenizer.from_pretrained(args.tokenizer_name_or_path       )

		# Handle output directory creation.
		# For serializing into a Google Cloud Storage Bucket, one needs to first
		# create a bucket.
		if "gs" not in args.output_dir:
				if not os.path.exists(args.output_dir       ):
						os.makedirs(args.output_dir       )
				_UpperCAmelCase							 =		os.path.join(args.output_dir     ,				args.split       )
				if not os.path.exists(A       ):
						os.makedirs(A       )
		else:
				_UpperCAmelCase							 =		os.path.join(args.output_dir     ,				args.split       )

		# Tokenize the whole dataset at once.
		_UpperCAmelCase							 =		tokenize_function(A       )
		_UpperCAmelCase							 =		dataset.map(A     ,				batched=A     ,				num_proc=4     ,				remove_columns=['text']       )

		# We need to concatenate all our texts together, and then split the result
		# into chunks of a fixed size, which we will call block_size. To do this, we
		# will use the map method again, with the option batched=True. When we use batched=True,
		# the function we pass to map() will be passed multiple inputs at once, allowing us
		# to group them into more or fewer examples than we had in the input.
		# This allows us to create our new fixed-length samples. The advantage of this
		# method is that we don't lose a whole lot of content from the dataset compared to the
		# case where we simply tokenize with a pre-defined max_length.

		def group_texts(A	:						Any       ):
				# Concatenate all texts.
				_UpperCAmelCase							 =		{k: sum(examples[k]     ,				[]       ) for k in examples.keys()}
				_UpperCAmelCase							 =		len(concatenated_examples[list(examples.keys()       )[0]]       )
				# We drop the small remainder, though you could add padding instead if the model supports it
				# In this, as in all things, we advise you to follow your heart 🫀
				_UpperCAmelCase							 =		(total_length // args.max_length) * args.max_length
				# Split by chunks of max_len.
				_UpperCAmelCase							 =		{
				    k: [t[i : i + args.max_length] for i in range(0     ,				A     ,				args.max_length       )]
				    for k, t in concatenated_examples.items()
				}
				return result

		_UpperCAmelCase							 =		dataset_tokenized.map(A     ,				batched=A     ,				batch_size=1000     ,				num_proc=4       )

		_UpperCAmelCase							 =		0
		_UpperCAmelCase							 =		0
		for shard in range(0     ,				len(A       )     ,				args.shard_size       ):
				_UpperCAmelCase							 =		grouped_dataset[shard : shard + args.shard_size]
				_UpperCAmelCase							 =		len(dataset_snapshot['input_ids']       )
				_UpperCAmelCase							 =		os.path.join(A     ,				f'dataset-{shard_count}-{records_containing}.tfrecord'       )
				_UpperCAmelCase							 =		get_serialized_examples(A       )

				with tf.io.TFRecordWriter(A       ) as out_file:
						for i in range(len(A       )       ):
								_UpperCAmelCase							 =		serialized_examples[i]
								out_file.write(A       )
						print('Wrote file {} containing {} records'.format(A     ,				A       )       )

				shard_count += 1
				total_records += records_containing

		with open(f'split-{args.split}-records-count.txt'     ,				'w'       ) as f:
				print(f'Total {args.split} records: {total_records}'     ,				file=A       )


if __name__ == "__main__":
					lowercase  							=					parse_args()
					main(args)







"""simple docstring"""





import gc
import unittest

import numpy as np
import torch

from diffusers import AutoencoderKL, DDIMScheduler, DiTPipeline, DPMSolverMultistepScheduler, TransformeraDModel
from diffusers.utils import is_xformers_available, load_numpy, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..pipeline_params import (
    CLASS_CONDITIONED_IMAGE_GENERATION_BATCH_PARAMS,
    CLASS_CONDITIONED_IMAGE_GENERATION_PARAMS,
)
from ..test_pipelines_common import PipelineTesterMixin


enable_full_determinism()




class 			lowercase__			(      A,					unittest.TestCase    ):
							'''simple docstring'''







							_UpperCAmelCase      				=					DiTPipeline
							_UpperCAmelCase      				=					CLASS_CONDITIONED_IMAGE_GENERATION_PARAMS
							_UpperCAmelCase      				=					PipelineTesterMixin.required_optional_params - {
							    '''latents''',
							    '''num_images_per_prompt''',
							    '''callback''',
							    '''callback_steps''',
							}
							_UpperCAmelCase      				=					CLASS_CONDITIONED_IMAGE_GENERATION_BATCH_PARAMS
							_UpperCAmelCase      				=					False






							def        lowerCamelCase_   (  self						)			->							str:
									torch.manual_seed(0						)
									_UpperCAmelCase							 =		TransformeraDModel(
									    sample_size=16	,       num_layers=2	,       patch_size=4	,       attention_head_dim=8	,       num_attention_heads=2	,       in_channels=4	,       out_channels=8	,       attention_bias=snake_case	,       activation_fn='gelu-approximate'	,       num_embeds_ada_norm=1000	,       norm_type='ada_norm_zero'	,       norm_elementwise_affine=snake_case	,       )
									_UpperCAmelCase							 =		AutoencoderKL()
									_UpperCAmelCase							 =		DDIMScheduler()
									_UpperCAmelCase							 =		{'transformer': transformer.eval(), 'vae': vae.eval(), 'scheduler': scheduler}
									return components






							def        lowerCamelCase_   (  self	,       snake_case	,       snake_case=0						)			->							Optional[Any]:
									if str(snake_case						).startswith('mps'						):
											_UpperCAmelCase							 =		torch.manual_seed(snake_case						)
									else:
											_UpperCAmelCase							 =		torch.Generator(device=snake_case						).manual_seed(snake_case						)
									_UpperCAmelCase							 =		{
									    'class_labels': [1],
									    'generator': generator,
									    'num_inference_steps': 2,
									    'output_type': 'numpy',
									}
									return inputs






							def        lowerCamelCase_   (  self						)			->							List[Any]:
									_UpperCAmelCase							 =		'cpu'

									_UpperCAmelCase							 =		self.get_dummy_components()
									_UpperCAmelCase							 =		self.pipeline_class(**snake_case						)
									pipe.to(snake_case						)
									pipe.set_progress_bar_config(disable=snake_case						)

									_UpperCAmelCase							 =		self.get_dummy_inputs(snake_case						)
									_UpperCAmelCase							 =		pipe(**snake_case						).images
									_UpperCAmelCase							 =		image[0, -3:, -3:, -1]

									self.assertEqual(image.shape	,       (1, 16, 16, 3)						)
									_UpperCAmelCase							 =		np.array([0.2946, 0.6601, 0.4329, 0.3296, 0.4144, 0.5319, 0.7273, 0.5013, 0.4457]						)
									_UpperCAmelCase							 =		np.abs(image_slice.flatten() - expected_slice						).max()
									self.assertLessEqual(snake_case	,       1E-3						)






							def        lowerCamelCase_   (  self						)			->							Any:
									self._test_inference_batch_single_identical(relax_max_difference=snake_case	,       expected_max_diff=1E-3						)






							@unittest.skipIf(
							    torch_device != 'cuda' or not is_xformers_available()	,       reason='XFormers attention is only available with CUDA and `xformers` installed'	,       )
							def        lowerCamelCase_   (  self						)			->							Optional[int]:
									self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=1E-3						)







@require_torch_gpu
@slow
class 			lowercase__			(      unittest.TestCase    ):
							'''simple docstring'''







							def        lowerCamelCase_   (  self						)			->							int:
									super().tearDown()
									gc.collect()
									torch.cuda.empty_cache()






							def        lowerCamelCase_   (  self						)			->							int:
									_UpperCAmelCase							 =		torch.manual_seed(0						)

									_UpperCAmelCase							 =		DiTPipeline.from_pretrained('facebook/DiT-XL-2-256'						)
									pipe.to('cuda'						)

									_UpperCAmelCase							 =		['vase', 'umbrella', 'white shark', 'white wolf']
									_UpperCAmelCase							 =		pipe.get_label_ids(snake_case						)

									_UpperCAmelCase							 =		pipe(snake_case	,       generator=snake_case	,       num_inference_steps=40	,       output_type='np'						).images

									for word, image in zip(snake_case	,       snake_case						):
											_UpperCAmelCase							 =		load_numpy(
											    f'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/dit/{word}.npy'						)
											assert np.abs((expected_image - image).max()						) < 1E-2






							def        lowerCamelCase_   (  self						)			->							int:
									_UpperCAmelCase							 =		DiTPipeline.from_pretrained('facebook/DiT-XL-2-512'						)
									_UpperCAmelCase							 =		DPMSolverMultistepScheduler.from_config(pipe.scheduler.config						)
									pipe.to('cuda'						)

									_UpperCAmelCase							 =		['vase', 'umbrella']
									_UpperCAmelCase							 =		pipe.get_label_ids(snake_case						)

									_UpperCAmelCase							 =		torch.manual_seed(0						)
									_UpperCAmelCase							 =		pipe(snake_case	,       generator=snake_case	,       num_inference_steps=25	,       output_type='np'						).images

									for word, image in zip(snake_case	,       snake_case						):
											_UpperCAmelCase							 =		load_numpy(
											    'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'
											    f'/dit/{word}_512.npy'						)

											assert np.abs((expected_image - image).max()						) < 1E-1


