





'''simple docstring'''





import json
import os
from functools import lru_cache
from typing import Dict, List, Optional, Tuple, Union

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...tokenization_utils_base import BatchEncoding, EncodedInput
from ...utils import PaddingStrategy, logging


_lowerCamelCase   				=			logging.get_logger(__name__)


_lowerCamelCase   				=			{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}

# See all LED models at https://huggingface.co/models?filter=LED
_lowerCamelCase   				=			{
    'vocab_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json',
    },
    'merges_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json',
    },
}

_lowerCamelCase   				=			{
    'allenai/led-base-16384': 16384,
}






@lru_cache()
# Copied from transformers.models.bart.tokenization_bart.bytes_to_unicode
def 				SCREAMING_SNAKE_CASE__							(       ):
		SCREAMING_SNAKE_CASE__			=      (
		    list(range(ord("""!"""    )					,   ord("""~"""    ) + 1    )    ) + list(range(ord("""¡"""    )					,   ord("""¬"""    ) + 1    )    ) + list(range(ord("""®"""    )					,   ord("""ÿ"""    ) + 1    )    )
		)
		SCREAMING_SNAKE_CASE__			=      bs[:]
		SCREAMING_SNAKE_CASE__			=      0
		for b in range(2**8    ):
				if b not in bs:
						bs.append(__A    )
						cs.append(2**8 + n    )
						n += 1
		SCREAMING_SNAKE_CASE__			=      [chr(__A    ) for n in cs]
		return dict(zip(__A					,   __A    )    )






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							str    ):
		SCREAMING_SNAKE_CASE__			=      set()
		SCREAMING_SNAKE_CASE__			=      word[0]
		for char in word[1:]:
				pairs.add((prev_char, char)    )
				SCREAMING_SNAKE_CASE__			=      char
		return pairs




class   UpperCamelCase_       (     __A    ):
							lowerCamelCase_           =					VOCAB_FILES_NAMES
							lowerCamelCase_           =					PRETRAINED_VOCAB_FILES_MAP
							lowerCamelCase_           =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowerCamelCase_           =					["input_ids", "attention_mask"]


							def __init__(							self    :Tuple							,				__A    :int							,				__A    :str							,				__A    :str="replace"							,				__A    :int="<s>"							,				__A    :Optional[int]="</s>"							,				__A    :Optional[int]="</s>"							,				__A    :List[Any]="<s>"							,				__A    :str="<unk>"							,				__A    :Dict="<pad>"							,				__A    :Union[str, Any]="<mask>"							,				__A    :str=False							,				**__A    :int							,				)   ->   Dict:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      AddedToken(__A							,				lstrip=__A							,				rstrip=__A		) if isinstance(__A							,				__A		) else bos_token
									SCREAMING_SNAKE_CASE__			=      AddedToken(__A							,				lstrip=__A							,				rstrip=__A		) if isinstance(__A							,				__A		) else eos_token
									SCREAMING_SNAKE_CASE__			=      AddedToken(__A							,				lstrip=__A							,				rstrip=__A		) if isinstance(__A							,				__A		) else sep_token
									SCREAMING_SNAKE_CASE__			=      AddedToken(__A							,				lstrip=__A							,				rstrip=__A		) if isinstance(__A							,				__A		) else cls_token
									SCREAMING_SNAKE_CASE__			=      AddedToken(__A							,				lstrip=__A							,				rstrip=__A		) if isinstance(__A							,				__A		) else unk_token
									SCREAMING_SNAKE_CASE__			=      AddedToken(__A							,				lstrip=__A							,				rstrip=__A		) if isinstance(__A							,				__A		) else pad_token

									# Mask token behave like a normal word, i.e. include the space before it
									SCREAMING_SNAKE_CASE__			=      AddedToken(__A							,				lstrip=__A							,				rstrip=__A		) if isinstance(__A							,				__A		) else mask_token

									super().__init__(
									    errors=__A							,				bos_token=__A							,				eos_token=__A							,				unk_token=__A							,				sep_token=__A							,				cls_token=__A							,				pad_token=__A							,				mask_token=__A							,				add_prefix_space=__A							,				**__A							,				)

									with open(__A							,				encoding="""utf-8"""		) as vocab_handle:
											SCREAMING_SNAKE_CASE__			=      json.load(__A		)
									SCREAMING_SNAKE_CASE__			=      {v: k for k, v in self.encoder.items()}
									SCREAMING_SNAKE_CASE__			=      errors  # how to handle errors in decoding
									SCREAMING_SNAKE_CASE__			=      bytes_to_unicode()
									SCREAMING_SNAKE_CASE__			=      {v: k for k, v in self.byte_encoder.items()}
									with open(__A							,				encoding="""utf-8"""		) as merges_handle:
											SCREAMING_SNAKE_CASE__			=      merges_handle.read().split("""\n"""		)[1:-1]
									SCREAMING_SNAKE_CASE__			=      [tuple(merge.split()		) for merge in bpe_merges]
									SCREAMING_SNAKE_CASE__			=      dict(zip(__A							,				range(len(__A		)		)		)		)
									SCREAMING_SNAKE_CASE__			=      {}
									SCREAMING_SNAKE_CASE__			=      add_prefix_space

									# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
									SCREAMING_SNAKE_CASE__			=      re.compile(r"""\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""		)


							@property
							# Copied from transformers.models.bart.tokenization_bart.BartTokenizer.vocab_size
							def      _snake_case  (							self    :Union[str, Any]		)   ->   Optional[int]:



									"""simple docstring"""



									return len(self.encoder		)


							def      _snake_case  (							self    :Union[str, Any]		)   ->   Union[str, Any]:



									"""simple docstring"""



									return dict(self.encoder							,				**self.added_tokens_encoder		)


							def      _snake_case  (							self    :int							,				__A    :str		)   ->   Optional[Any]:



									"""simple docstring"""



									if token in self.cache:
											return self.cache[token]
									SCREAMING_SNAKE_CASE__			=      tuple(__A		)
									SCREAMING_SNAKE_CASE__			=      get_pairs(__A		)

									if not pairs:
											return token

									while True:
											SCREAMING_SNAKE_CASE__			=      min(__A							,				key=lambda __A		: self.bpe_ranks.get(__A							,				float("""inf"""		)		)		)
											if bigram not in self.bpe_ranks:
													break
											SCREAMING_SNAKE_CASE__			=      bigram
											SCREAMING_SNAKE_CASE__			=      []
											SCREAMING_SNAKE_CASE__			=      0
											while i < len(__A		):
													try:
															SCREAMING_SNAKE_CASE__			=      word.index(__A							,				__A		)
													except ValueError:
															new_word.extend(word[i:]		)
															break
													else:
															new_word.extend(word[i:j]		)
															SCREAMING_SNAKE_CASE__			=      j

													if word[i] == first and i < len(__A		) - 1 and word[i + 1] == second:
															new_word.append(first + second		)
															i += 2
													else:
															new_word.append(word[i]		)
															i += 1
											SCREAMING_SNAKE_CASE__			=      tuple(__A		)
											SCREAMING_SNAKE_CASE__			=      new_word
											if len(__A		) == 1:
													break
											else:
													SCREAMING_SNAKE_CASE__			=      get_pairs(__A		)
									SCREAMING_SNAKE_CASE__			=      ''' '''.join(__A		)
									SCREAMING_SNAKE_CASE__			=      word
									return word


							def      _snake_case  (							self    :int							,				__A    :Optional[Any]		)   ->   Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      []
									for token in re.findall(self.pat							,				__A		):
											SCREAMING_SNAKE_CASE__			=      ''''''.join(
											    self.byte_encoder[b] for b in token.encode("""utf-8"""		)		)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
											bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__A		).split(""" """		)		)
									return bpe_tokens


							def      _snake_case  (							self    :Union[str, Any]							,				__A    :Union[str, Any]		)   ->   List[str]:



									"""simple docstring"""



									return self.encoder.get(__A							,				self.encoder.get(self.unk_token		)		)


							def      _snake_case  (							self    :Dict							,				__A    :Optional[int]		)   ->   Any:



									"""simple docstring"""



									return self.decoder.get(__A		)


							def      _snake_case  (							self    :List[str]							,				__A    :List[Any]		)   ->   Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      ''''''.join(__A		)
									SCREAMING_SNAKE_CASE__			=      bytearray([self.byte_decoder[c] for c in text]		).decode("""utf-8"""							,				errors=self.errors		)
									return text


							def      _snake_case  (							self    :Any							,				__A    :str							,				__A    :Optional[str] = None		)   ->   Tuple[str]:



									"""simple docstring"""



									if not os.path.isdir(__A		):
											logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''		)
											return
									SCREAMING_SNAKE_CASE__			=      os.path.join(
									    __A							,				(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]		)
									SCREAMING_SNAKE_CASE__			=      os.path.join(
									    __A							,				(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]		)

									with open(__A							,				"""w"""							,				encoding="""utf-8"""		) as f:
											f.write(json.dumps(self.encoder							,				indent=2							,				sort_keys=__A							,				ensure_ascii=__A		) + """\n"""		)

									SCREAMING_SNAKE_CASE__			=      0
									with open(__A							,				"""w"""							,				encoding="""utf-8"""		) as writer:
											writer.write("""#version: 0.2\n"""		)
											for bpe_tokens, token_index in sorted(self.bpe_ranks.items()							,				key=lambda __A		: kv[1]		):
													if index != token_index:
															logger.warning(
															    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
															    """ Please check that the tokenizer is not corrupted!"""		)
															SCREAMING_SNAKE_CASE__			=      token_index
													writer.write(""" """.join(__A		) + """\n"""		)
													index += 1

									return vocab_file, merge_file


							def      _snake_case  (							self    :str							,				__A    :List[int]							,				__A    :Optional[List[int]] = None		)   ->   List[int]:



									"""simple docstring"""



									if token_ids_a is None:
											return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
									SCREAMING_SNAKE_CASE__			=      [self.cls_token_id]
									SCREAMING_SNAKE_CASE__			=      [self.sep_token_id]
									return cls + token_ids_a + sep + sep + token_ids_a + sep


							def      _snake_case  (							self    :Optional[int]							,				__A    :List[int]							,				__A    :Optional[List[int]] = None							,				__A    :bool = False		)   ->   List[int]:



									"""simple docstring"""



									if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=__A							,				token_ids_a=__A							,				already_has_special_tokens=__A		)

									if token_ids_a is None:
											return [1] + ([0] * len(__A		)) + [1]
									return [1] + ([0] * len(__A		)) + [1, 1] + ([0] * len(__A		)) + [1]


							def      _snake_case  (							self    :Any							,				__A    :List[int]							,				__A    :Optional[List[int]] = None		)   ->   List[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      [self.sep_token_id]
									SCREAMING_SNAKE_CASE__			=      [self.cls_token_id]

									if token_ids_a is None:
											return len(cls + token_ids_a + sep		) * [0]
									return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]


							def      _snake_case  (							self    :Optional[int]							,				__A    :Optional[Any]							,				__A    :Optional[int]=False							,				**__A    :Optional[Any]		)   ->   str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      kwargs.pop("""add_prefix_space"""							,				self.add_prefix_space		)
									if (is_split_into_words or add_prefix_space) and (len(__A		) > 0 and not text[0].isspace()):
											SCREAMING_SNAKE_CASE__			=      ''' ''' + text
									return (text, kwargs)


							def      _snake_case  (							self    :Any							,				__A    :Union[Dict[str, EncodedInput], BatchEncoding]							,				__A    :Optional[int] = None							,				__A    :PaddingStrategy = PaddingStrategy.DO_NOT_PAD							,				__A    :Optional[int] = None							,				__A    :Optional[bool] = None							,				)   ->   dict:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      super()._pad(
									    encoded_inputs=__A							,				max_length=__A							,				padding_strategy=__A							,				pad_to_multiple_of=__A							,				return_attention_mask=__A							,				)

									# Load from model defaults
									if return_attention_mask is None:
											SCREAMING_SNAKE_CASE__			=      '''attention_mask''' in self.model_input_names

									if return_attention_mask and "global_attention_mask" in encoded_inputs:
											SCREAMING_SNAKE_CASE__			=      encoded_inputs[self.model_input_names[0]]
											# `global_attention_mask` need to have the same length as other (sequential) inputs.
											SCREAMING_SNAKE_CASE__			=      len(encoded_inputs["""global_attention_mask"""]		) != len(__A		)

											if needs_to_be_padded:
													SCREAMING_SNAKE_CASE__			=      len(__A		) - len(encoded_inputs["""global_attention_mask"""]		)

													if self.padding_side == "right":
															# Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
															SCREAMING_SNAKE_CASE__			=      (
															    encoded_inputs['''global_attention_mask'''] + [-1] * difference
															)
													elif self.padding_side == "left":
															SCREAMING_SNAKE_CASE__			=      [-1] * difference + encoded_inputs[
															    '''global_attention_mask'''
															]
													else:
															raise ValueError("""Invalid padding strategy:""" + str(self.padding_side		)		)

									return encoded_inputs






import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


_lowerCamelCase   				=			logging.getLogger(__name__)






def 				SCREAMING_SNAKE_CASE__							(       ):
		SCREAMING_SNAKE_CASE__			=      argparse.ArgumentParser(
		    description="""Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."""    )
		parser.add_argument(
		    """--dataset_name"""					,   type=UpperCamelCase__					,   default="""wikitext"""					,   help="""Name of the training. Explore datasets at: hf.co/datasets."""					,   )
		parser.add_argument(
		    """--dataset_config"""					,   type=UpperCamelCase__					,   default="""wikitext-103-raw-v1"""					,   help="""Configuration name of the dataset."""    )
		parser.add_argument(
		    """--tokenizer_name_or_path"""					,   type=UpperCamelCase__					,   default="""sayakpaul/unigram-tokenizer-wikitext"""					,   help="""Tokenizer identifier. Can be a local filepath or a Hub identifier."""					,   )
		parser.add_argument(
		    """--shard_size"""					,   type=UpperCamelCase__					,   default=1_000					,   help="""Number of entries to go in a single shard."""					,   )
		parser.add_argument("""--split"""					,   type=UpperCamelCase__					,   default="""train"""					,   choices=["""train""", """test""", """validation"""]    )
		parser.add_argument(
		    """--limit"""					,   default=UpperCamelCase__					,   type=UpperCamelCase__					,   help="""Limit the number of shards (used for debugging)."""					,   )
		parser.add_argument(
		    """--max_length"""					,   type=UpperCamelCase__					,   default=512					,   help="""Maximum sequence length. For training on TPUs, it helps to have a maximum"""
		    """ sequence length that is a multiple of 8."""					,   )
		parser.add_argument(
		    """--output_dir"""					,   default="""tf-tpu"""					,   type=UpperCamelCase__					,   help="""Output directory where the TFRecord shards will be saved. If the"""
		    """ path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"""
		    """ shards will be directly saved to a Google Cloud Storage bucket."""					,   )

		SCREAMING_SNAKE_CASE__			=      parser.parse_args()
		return args






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							List[Any]    ):
		def fn(UpperCamelCase__:							Any    ):
				return tokenizer(examples["""text"""]    )

		return fn






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							Any    ):
		SCREAMING_SNAKE_CASE__			=      []
		for i in range(len(tokenized_data["""input_ids"""]    )    ):
				SCREAMING_SNAKE_CASE__			=      {
				    """input_ids""": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["""input_ids"""][i]    )    ),
				    """attention_mask""": tf.train.Feature(
				        intaa_list=tf.train.IntaaList(value=tokenized_data["""attention_mask"""][i]    )    ),
				}
				SCREAMING_SNAKE_CASE__			=      tf.train.Features(feature=UpperCamelCase__    )
				SCREAMING_SNAKE_CASE__			=      tf.train.Example(features=UpperCamelCase__    )
				SCREAMING_SNAKE_CASE__			=      example.SerializeToString()
				records.append(UpperCamelCase__    )
		return records






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							List[str]    ):
		SCREAMING_SNAKE_CASE__			=      datasets.load_dataset(args.dataset_name					,   args.dataset_config					,   split=args.split    )

		if args.limit is not None:
				SCREAMING_SNAKE_CASE__			=      min(len(UpperCamelCase__    )					,   args.limit    )
				SCREAMING_SNAKE_CASE__			=      dataset.select(range(UpperCamelCase__    )    )
				print(f'''Limiting the dataset to {args.limit} entries.'''    )

		SCREAMING_SNAKE_CASE__			=      AutoTokenizer.from_pretrained(args.tokenizer_name_or_path    )

		# Handle output directory creation.
		# For serializing into a Google Cloud Storage Bucket, one needs to first
		# create a bucket.
		if "gs" not in args.output_dir:
				if not os.path.exists(args.output_dir    ):
						os.makedirs(args.output_dir    )
				SCREAMING_SNAKE_CASE__			=      os.path.join(args.output_dir					,   args.split    )
				if not os.path.exists(UpperCamelCase__    ):
						os.makedirs(UpperCamelCase__    )
		else:
				SCREAMING_SNAKE_CASE__			=      os.path.join(args.output_dir					,   args.split    )

		# Tokenize the whole dataset at once.
		SCREAMING_SNAKE_CASE__			=      tokenize_function(UpperCamelCase__    )
		SCREAMING_SNAKE_CASE__			=      dataset.map(UpperCamelCase__					,   batched=UpperCamelCase__					,   num_proc=4					,   remove_columns=["""text"""]    )

		# We need to concatenate all our texts together, and then split the result
		# into chunks of a fixed size, which we will call block_size. To do this, we
		# will use the map method again, with the option batched=True. When we use batched=True,
		# the function we pass to map() will be passed multiple inputs at once, allowing us
		# to group them into more or fewer examples than we had in the input.
		# This allows us to create our new fixed-length samples. The advantage of this
		# method is that we don't lose a whole lot of content from the dataset compared to the
		# case where we simply tokenize with a pre-defined max_length.

		def group_texts(UpperCamelCase__:							int    ):
				# Concatenate all texts.
				SCREAMING_SNAKE_CASE__			=      {k: sum(examples[k]					,   []    ) for k in examples.keys()}
				SCREAMING_SNAKE_CASE__			=      len(concatenated_examples[list(examples.keys()    )[0]]    )
				# We drop the small remainder, though you could add padding instead if the model supports it
				# In this, as in all things, we advise you to follow your heart 🫀
				SCREAMING_SNAKE_CASE__			=      (total_length // args.max_length) * args.max_length
				# Split by chunks of max_len.
				SCREAMING_SNAKE_CASE__			=      {
				    k: [t[i : i + args.max_length] for i in range(0					,   UpperCamelCase__					,   args.max_length    )]
				    for k, t in concatenated_examples.items()
				}
				return result

		SCREAMING_SNAKE_CASE__			=      dataset_tokenized.map(UpperCamelCase__					,   batched=UpperCamelCase__					,   batch_size=1_000					,   num_proc=4    )

		SCREAMING_SNAKE_CASE__			=      0
		SCREAMING_SNAKE_CASE__			=      0
		for shard in range(0					,   len(UpperCamelCase__    )					,   args.shard_size    ):
				SCREAMING_SNAKE_CASE__			=      grouped_dataset[shard : shard + args.shard_size]
				SCREAMING_SNAKE_CASE__			=      len(dataset_snapshot["""input_ids"""]    )
				SCREAMING_SNAKE_CASE__			=      os.path.join(UpperCamelCase__					,   f'''dataset-{shard_count}-{records_containing}.tfrecord'''    )
				SCREAMING_SNAKE_CASE__			=      get_serialized_examples(UpperCamelCase__    )

				with tf.io.TFRecordWriter(UpperCamelCase__    ) as out_file:
						for i in range(len(UpperCamelCase__    )    ):
								SCREAMING_SNAKE_CASE__			=      serialized_examples[i]
								out_file.write(UpperCamelCase__    )
						print("""Wrote file {} containing {} records""".format(UpperCamelCase__					,   UpperCamelCase__    )    )

				shard_count += 1
				total_records += records_containing

		with open(f'''split-{args.split}-records-count.txt'''					,   """w"""    ) as f:
				print(f'''Total {args.split} records: {total_records}'''					,   file=UpperCamelCase__    )


if __name__ == "__main__":
	_lowerCamelCase   				=			parse_args()
	main(args)