


import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


A							:					Optional[int]     						=     logging.getLogger(__name__)





def 	_lowerCAmelCase       (    )						->	int:


						'''simple docstring'''






						__snake_case				=		argparse.ArgumentParser(
						    description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset." )
						parser.add_argument(
						    "--dataset_name"      ,	type=_lowerCAmelCase      ,	default="wikitext"      ,	help="Name of the training. Explore datasets at: hf.co/datasets."      ,	)
						parser.add_argument(
						    "--dataset_config"      ,	type=_lowerCAmelCase      ,	default="wikitext-103-raw-v1"      ,	help="Configuration name of the dataset." )
						parser.add_argument(
						    "--tokenizer_name_or_path"      ,	type=_lowerCAmelCase      ,	default="sayakpaul/unigram-tokenizer-wikitext"      ,	help="Tokenizer identifier. Can be a local filepath or a Hub identifier."      ,	)
						parser.add_argument(
						    "--shard_size"      ,	type=_lowerCAmelCase      ,	default=1000      ,	help="Number of entries to go in a single shard."      ,	)
						parser.add_argument("--split"      ,	type=_lowerCAmelCase      ,	default="train"      ,	choices=["train", "test", "validation"] )
						parser.add_argument(
						    "--limit"      ,	default=_lowerCAmelCase      ,	type=_lowerCAmelCase      ,	help="Limit the number of shards (used for debugging)."      ,	)
						parser.add_argument(
						    "--max_length"      ,	type=_lowerCAmelCase      ,	default=512      ,	help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
						    " sequence length that is a multiple of 8."      ,	)
						parser.add_argument(
						    "--output_dir"      ,	default="tf-tpu"      ,	type=_lowerCAmelCase      ,	help="Output directory where the TFRecord shards will be saved. If the"
						    " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
						    " shards will be directly saved to a Google Cloud Storage bucket."      ,	)

						__snake_case				=		parser.parse_args()
						return args





def 	_lowerCAmelCase       (    _lowerCAmelCase )						->	int:


						'''simple docstring'''






						def fn(_lowerCAmelCase ):
												return tokenizer(examples["text"] )

						return fn





def 	_lowerCAmelCase       (    _lowerCAmelCase )						->	Tuple:


						'''simple docstring'''






						__snake_case				=		[]
						for i in range(len(tokenized_data["input_ids"] ) ):
												__snake_case				=		{
												    "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i] ) ),
												    "attention_mask": tf.train.Feature(
												        intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i] ) ),
												}
												__snake_case				=		tf.train.Features(feature=_lowerCAmelCase )
												__snake_case				=		tf.train.Example(features=_lowerCAmelCase )
												__snake_case				=		example.SerializeToString()
												records.append(_lowerCAmelCase )
						return records





def 	_lowerCAmelCase       (    _lowerCAmelCase )						->	List[Any]:


						'''simple docstring'''






						__snake_case				=		datasets.load_dataset(args.dataset_name      ,	args.dataset_config      ,	split=args.split )

						if args.limit is not None:
												__snake_case				=		min(len(_lowerCAmelCase )      ,	args.limit )
												__snake_case				=		dataset.select(range(_lowerCAmelCase ) )
												print(F'''Limiting the dataset to {args.limit} entries.''' )

						__snake_case				=		AutoTokenizer.from_pretrained(args.tokenizer_name_or_path )

						# Handle output directory creation.
						# For serializing into a Google Cloud Storage Bucket, one needs to first
						# create a bucket.
						if "gs" not in args.output_dir:
												if not os.path.exists(args.output_dir ):
																		os.makedirs(args.output_dir )
												__snake_case				=		os.path.join(args.output_dir      ,	args.split )
												if not os.path.exists(_lowerCAmelCase ):
																		os.makedirs(_lowerCAmelCase )
						else:
												__snake_case				=		os.path.join(args.output_dir      ,	args.split )

						# Tokenize the whole dataset at once.
						__snake_case				=		tokenize_function(_lowerCAmelCase )
						__snake_case				=		dataset.map(_lowerCAmelCase      ,	batched=_lowerCAmelCase      ,	num_proc=4      ,	remove_columns=["text"] )

						# We need to concatenate all our texts together, and then split the result
						# into chunks of a fixed size, which we will call block_size. To do this, we
						# will use the map method again, with the option batched=True. When we use batched=True,
						# the function we pass to map() will be passed multiple inputs at once, allowing us
						# to group them into more or fewer examples than we had in the input.
						# This allows us to create our new fixed-length samples. The advantage of this
						# method is that we don't lose a whole lot of content from the dataset compared to the
						# case where we simply tokenize with a pre-defined max_length.

						def group_texts(_lowerCAmelCase ):
												# Concatenate all texts.
												__snake_case				=		{k: sum(examples[k]      ,	[] ) for k in examples.keys()}
												__snake_case				=		len(concatenated_examples[list(examples.keys() )[0]] )
												# We drop the small remainder, though you could add padding instead if the model supports it
												# In this, as in all things, we advise you to follow your heart 🫀
												__snake_case				=		(total_length // args.max_length) * args.max_length
												# Split by chunks of max_len.
												__snake_case				=		{
												    k: [t[i : i + args.max_length] for i in range(0      ,	_lowerCAmelCase      ,	args.max_length )]
												    for k, t in concatenated_examples.items()
												}
												return result

						__snake_case				=		dataset_tokenized.map(_lowerCAmelCase      ,	batched=_lowerCAmelCase      ,	batch_size=1000      ,	num_proc=4 )

						__snake_case				=		0
						__snake_case				=		0
						for shard in range(0      ,	len(_lowerCAmelCase )      ,	args.shard_size ):
												__snake_case				=		grouped_dataset[shard : shard + args.shard_size]
												__snake_case				=		len(dataset_snapshot["input_ids"] )
												__snake_case				=		os.path.join(_lowerCAmelCase      ,	F'''dataset-{shard_count}-{records_containing}.tfrecord''' )
												__snake_case				=		get_serialized_examples(_lowerCAmelCase )

												with tf.io.TFRecordWriter(_lowerCAmelCase ) as out_file:
																		for i in range(len(_lowerCAmelCase ) ):
																								__snake_case				=		serialized_examples[i]
																								out_file.write(_lowerCAmelCase )
																		print("Wrote file {} containing {} records".format(_lowerCAmelCase      ,	_lowerCAmelCase ) )

												shard_count += 1
												total_records += records_containing

						with open(F'''split-{args.split}-records-count.txt'''      ,	"w" ) as f:
												print(F'''Total {args.split} records: {total_records}'''      ,	file=_lowerCAmelCase )


if __name__ == "__main__":
	A							:					Any     						=     parse_args()
	main(args)









from collections import OrderedDict
from typing import Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


A							:					Union[str, Any]     						=     logging.get_logger(__name__)

A							:					Tuple     						=     {
    'facebook/data2vec-vision-base-ft': (
        'https://huggingface.co/facebook/data2vec-vision-base-ft/resolve/main/config.json'
    ),
}






class     UpperCamelCase(  _a   ):
					snake_case_		:				List[Any]										=	"""data2vec-vision"""


					def __init__(						self	:		Any     ,    SCREAMING_SNAKE_CASE	:		List[str]=7_6_8     ,    SCREAMING_SNAKE_CASE	:		Tuple=1_2     ,    SCREAMING_SNAKE_CASE	:		int=1_2     ,    SCREAMING_SNAKE_CASE	:		List[str]=3_0_7_2     ,    SCREAMING_SNAKE_CASE	:		Tuple="gelu"     ,    SCREAMING_SNAKE_CASE	:		Optional[Any]=0.0     ,    SCREAMING_SNAKE_CASE	:		List[Any]=0.0     ,    SCREAMING_SNAKE_CASE	:		Any=0.02     ,    SCREAMING_SNAKE_CASE	:		List[str]=1e-1_2     ,    SCREAMING_SNAKE_CASE	:		Optional[int]=2_2_4     ,    SCREAMING_SNAKE_CASE	:		Optional[int]=1_6     ,    SCREAMING_SNAKE_CASE	:		str=3     ,    SCREAMING_SNAKE_CASE	:		Optional[Any]=False     ,    SCREAMING_SNAKE_CASE	:		Any=False     ,    SCREAMING_SNAKE_CASE	:		Dict=False     ,    SCREAMING_SNAKE_CASE	:		int=False     ,    SCREAMING_SNAKE_CASE	:		Tuple=0.1     ,    SCREAMING_SNAKE_CASE	:		Any=0.1     ,    SCREAMING_SNAKE_CASE	:		Any=True     ,    SCREAMING_SNAKE_CASE	:		List[str]=[3, 5, 7, 1_1]     ,    SCREAMING_SNAKE_CASE	:		int=[1, 2, 3, 6]     ,    SCREAMING_SNAKE_CASE	:		List[str]=True     ,    SCREAMING_SNAKE_CASE	:		Tuple=0.4     ,    SCREAMING_SNAKE_CASE	:		Optional[Any]=2_5_6     ,    SCREAMING_SNAKE_CASE	:		Tuple=1     ,    SCREAMING_SNAKE_CASE	:		Optional[int]=False     ,    SCREAMING_SNAKE_CASE	:		List[Any]=2_5_5     ,    **SCREAMING_SNAKE_CASE	:		Any     ,    )			->	List[str]:




											'''simple docstring'''

											super().__init__(**SCREAMING_SNAKE_CASE						)

											__snake_case				=		hidden_size
											__snake_case				=		num_hidden_layers
											__snake_case				=		num_attention_heads
											__snake_case				=		intermediate_size
											__snake_case				=		hidden_act
											__snake_case				=		hidden_dropout_prob
											__snake_case				=		attention_probs_dropout_prob
											__snake_case				=		initializer_range
											__snake_case				=		layer_norm_eps

											__snake_case				=		image_size
											__snake_case				=		patch_size
											__snake_case				=		num_channels
											__snake_case				=		use_mask_token
											__snake_case				=		use_absolute_position_embeddings
											__snake_case				=		use_relative_position_bias
											__snake_case				=		use_shared_relative_position_bias
											__snake_case				=		layer_scale_init_value
											__snake_case				=		drop_path_rate
											__snake_case				=		use_mean_pooling
											# decode head attributes (semantic segmentation)
											__snake_case				=		out_indices
											__snake_case				=		pool_scales
											# auxiliary head attributes (semantic segmentation)
											__snake_case				=		use_auxiliary_head
											__snake_case				=		auxiliary_loss_weight
											__snake_case				=		auxiliary_channels
											__snake_case				=		auxiliary_num_convs
											__snake_case				=		auxiliary_concat_input
											__snake_case				=		semantic_loss_ignore_index






class     UpperCamelCase(  _a   ):
					snake_case_		:				Any										=	version.parse("""1.11"""   )


					@property
					def 							SCREAMING_SNAKE_CASE_  (						self	:		List[str]						)			->	Mapping[str, Mapping[int, str]]:




											'''simple docstring'''

											return OrderedDict(
											    [
											        ("pixel_values", {0: "batch", 1: "num_channels", 2: "height", 3: "width"}),
											    ]						)






					@property
					def 							SCREAMING_SNAKE_CASE_  (						self	:		Union[str, Any]						)			->	float:




											'''simple docstring'''

											return 1e-4





