



'''simple docstring'''



import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


lowerCAmelCase	:		List[Any]				=   logging.getLogger(__name__)



def   __lowerCAmelCase					(							):






	'''simple docstring'''




	__lowerCAmelCase												=  argparse.ArgumentParser(
	    description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."					)
	parser.add_argument(
	    "--dataset_name"				,		type=lowerCamelCase				,		default="wikitext"				,		help="Name of the training. Explore datasets at: hf.co/datasets."				,		)
	parser.add_argument(
	    "--dataset_config"				,		type=lowerCamelCase				,		default="wikitext-103-raw-v1"				,		help="Configuration name of the dataset."					)
	parser.add_argument(
	    "--tokenizer_name_or_path"				,		type=lowerCamelCase				,		default="sayakpaul/unigram-tokenizer-wikitext"				,		help="Tokenizer identifier. Can be a local filepath or a Hub identifier."				,		)
	parser.add_argument(
	    "--shard_size"				,		type=lowerCamelCase				,		default=10_00				,		help="Number of entries to go in a single shard."				,		)
	parser.add_argument("--split"				,		type=lowerCamelCase				,		default="train"				,		choices=["train", "test", "validation"]					)
	parser.add_argument(
	    "--limit"				,		default=lowerCamelCase				,		type=lowerCamelCase				,		help="Limit the number of shards (used for debugging)."				,		)
	parser.add_argument(
	    "--max_length"				,		type=lowerCamelCase				,		default=5_12				,		help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
	    " sequence length that is a multiple of 8."				,		)
	parser.add_argument(
	    "--output_dir"				,		default="tf-tpu"				,		type=lowerCamelCase				,		help="Output directory where the TFRecord shards will be saved. If the"
	    " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
	    " shards will be directly saved to a Google Cloud Storage bucket."				,		)

	__lowerCAmelCase												=  parser.parse_args()
	return args



def   __lowerCAmelCase					(							lowerCamelCase   :						int					):






	'''simple docstring'''




	def fn(lowerCamelCase   :						Optional[Any]					):
		return tokenizer(examples["text"]					)

	return fn



def   __lowerCAmelCase					(							lowerCamelCase   :						List[str]					):






	'''simple docstring'''




	__lowerCAmelCase												=  []
	for i in range(len(tokenized_data["input_ids"]					)					):
		__lowerCAmelCase												=  {
		    "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]					)					),
		    "attention_mask": tf.train.Feature(
		        intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]					)					),
		}
		__lowerCAmelCase												=  tf.train.Features(feature=lowerCamelCase					)
		__lowerCAmelCase												=  tf.train.Example(features=lowerCamelCase					)
		__lowerCAmelCase												=  example.SerializeToString()
		records.append(lowerCamelCase					)
	return records



def   __lowerCAmelCase					(							lowerCamelCase   :						Optional[Any]					):






	'''simple docstring'''




	__lowerCAmelCase												=  datasets.load_dataset(args.dataset_name				,		args.dataset_config				,		split=args.split					)

	if args.limit is not None:
		__lowerCAmelCase												=  min(len(lowerCamelCase					)				,		args.limit					)
		__lowerCAmelCase												=  dataset.select(range(lowerCamelCase					)					)
		print(f'''Limiting the dataset to {args.limit} entries.'''					)

	__lowerCAmelCase												=  AutoTokenizer.from_pretrained(args.tokenizer_name_or_path					)

	# Handle output directory creation.
	# For serializing into a Google Cloud Storage Bucket, one needs to first
	# create a bucket.
	if "gs" not in args.output_dir:
		if not os.path.exists(args.output_dir					):
			os.makedirs(args.output_dir					)
		__lowerCAmelCase												=  os.path.join(args.output_dir				,		args.split					)
		if not os.path.exists(lowerCamelCase					):
			os.makedirs(lowerCamelCase					)
	else:
		__lowerCAmelCase												=  os.path.join(args.output_dir				,		args.split					)

	# Tokenize the whole dataset at once.
	__lowerCAmelCase												=  tokenize_function(lowerCamelCase					)
	__lowerCAmelCase												=  dataset.map(lowerCamelCase				,		batched=lowerCamelCase				,		num_proc=4				,		remove_columns=["text"]					)

	# We need to concatenate all our texts together, and then split the result
	# into chunks of a fixed size, which we will call block_size. To do this, we
	# will use the map method again, with the option batched=True. When we use batched=True,
	# the function we pass to map() will be passed multiple inputs at once, allowing us
	# to group them into more or fewer examples than we had in the input.
	# This allows us to create our new fixed-length samples. The advantage of this
	# method is that we don't lose a whole lot of content from the dataset compared to the
	# case where we simply tokenize with a pre-defined max_length.

	def group_texts(lowerCamelCase   :						Optional[Any]					):
		# Concatenate all texts.
		__lowerCAmelCase												=  {k: sum(examples[k]				,		[]					) for k in examples.keys()}
		__lowerCAmelCase												=  len(concatenated_examples[list(examples.keys()					)[0]]					)
		# We drop the small remainder, though you could add padding instead if the model supports it
		# In this, as in all things, we advise you to follow your heart 🫀
		__lowerCAmelCase												=  (total_length // args.max_length) * args.max_length
		# Split by chunks of max_len.
		__lowerCAmelCase												=  {
		    k: [t[i : i + args.max_length] for i in range(0				,		lowerCamelCase				,		args.max_length					)]
		    for k, t in concatenated_examples.items()
		}
		return result

	__lowerCAmelCase												=  dataset_tokenized.map(lowerCamelCase				,		batched=lowerCamelCase				,		batch_size=10_00				,		num_proc=4					)

	__lowerCAmelCase												=  0
	__lowerCAmelCase												=  0
	for shard in range(0				,		len(lowerCamelCase					)				,		args.shard_size					):
		__lowerCAmelCase												=  grouped_dataset[shard : shard + args.shard_size]
		__lowerCAmelCase												=  len(dataset_snapshot["input_ids"]					)
		__lowerCAmelCase												=  os.path.join(lowerCamelCase				,		f'''dataset-{shard_count}-{records_containing}.tfrecord'''					)
		__lowerCAmelCase												=  get_serialized_examples(lowerCamelCase					)

		with tf.io.TFRecordWriter(lowerCamelCase					) as out_file:
			for i in range(len(lowerCamelCase					)					):
				__lowerCAmelCase												=  serialized_examples[i]
				out_file.write(lowerCamelCase					)
			print("Wrote file {} containing {} records".format(lowerCamelCase				,		lowerCamelCase					)					)

		shard_count += 1
		total_records += records_containing

	with open(f'''split-{args.split}-records-count.txt'''				,		"w"					) as f:
		print(f'''Total {args.split} records: {total_records}'''				,		file=lowerCamelCase					)


if __name__ == "__main__":
	lowerCAmelCase	:		Optional[Any]				=   parse_args()
	main(args)




'''simple docstring'''



import re



def   __lowerCAmelCase					(							lowerCamelCase   :						str					):






	'''simple docstring'''




	__lowerCAmelCase												=  re.compile(
	    r"^(?:0|94|\+94|0{2}94)" r"7(0|1|2|4|5|6|7|8)" r"(-| |)" r"\d{7}$"					)

	return bool(re.search(lowerCamelCase				,		lowerCamelCase					)					)


if __name__ == "__main__":
	lowerCAmelCase	:		Optional[Any]				=   '''0094702343221'''

	print(is_sri_lankan_phone_number(phone))