





'''simple docstring'''



from collections.abc import Generator
from math import sin




def    _lowerCAmelCase						(				lowerCamelCase_			:  bytes				):
			if len(lowerCamelCase_				) != 3_2:
						raise ValueError('''Input must be of length 32'''				)

			__lowercase										=							b''''''
			for i in [3, 2, 1, 0]:
						little_endian += string_aa[8 * i : 8 * i + 8]
			return little_endian




def    _lowerCAmelCase						(				lowerCamelCase_			:  int				):
			if i < 0:
						raise ValueError('''Input must be non-negative'''				)

			__lowercase										=							format(lowerCamelCase_		,       '''08x'''				)[-8:]
			__lowercase										=							b''''''
			for i in [3, 2, 1, 0]:
						little_endian_hex += hex_rep[2 * i : 2 * i + 2].encode('''utf-8'''				)
			return little_endian_hex




def    _lowerCAmelCase						(				lowerCamelCase_			:  bytes				):
			__lowercase										=							b''''''
			for char in message:
						bit_string += format(lowerCamelCase_		,       '''08b'''				).encode('''utf-8'''				)
			__lowercase										=							format(len(lowerCamelCase_				)		,       '''064b'''				).encode('''utf-8'''				)

			# Pad bit_string to a multiple of 512 chars
			bit_string += b"1"
			while len(lowerCamelCase_				) % 5_1_2 != 4_4_8:
						bit_string += b"0"
			bit_string += to_little_endian(start_len[3_2:]				) + to_little_endian(start_len[:3_2]				)

			return bit_string




def    _lowerCAmelCase						(				lowerCamelCase_			:  bytes				):
			if len(lowerCamelCase_				) % 5_1_2 != 0:
						raise ValueError('''Input must have length that\'s a multiple of 512'''				)

			for pos in range(0		,       len(lowerCamelCase_				)		,       5_1_2				):
						__lowercase										=							bit_string[pos : pos + 5_1_2]
						__lowercase										=							[]
						for i in range(0		,       5_1_2		,       3_2				):
									block_words.append(int(to_little_endian(block[i : i + 3_2]				)		,       2				)				)
						yield block_words




def    _lowerCAmelCase						(				lowerCamelCase_			:  int				):
			if i < 0:
						raise ValueError('''Input must be non-negative'''				)

			__lowercase										=							format(lowerCamelCase_		,       '''032b'''				)
			__lowercase										=							''''''
			for c in i_str:
						new_str += "1" if c == "0" else "0"
			return int(lowerCamelCase_		,       2				)




def    _lowerCAmelCase						(				lowerCamelCase_			:  int		,       lowerCamelCase_			:  int				):
			return (a + b) % 2**3_2




def    _lowerCAmelCase						(				lowerCamelCase_			:  int		,       lowerCamelCase_			:  int				):
			if i < 0:
						raise ValueError('''Input must be non-negative'''				)
			if shift < 0:
						raise ValueError('''Shift must be non-negative'''				)
			return ((i << shift) ^ (i >> (3_2 - shift))) % 2**3_2




def    _lowerCAmelCase						(				lowerCamelCase_			:  bytes				):
			__lowercase										=							preprocess(lowerCamelCase_				)

			__lowercase										=							[int(2**3_2 * abs(sin(i + 1				)				)				) for i in range(6_4				)]

			# Starting states
			__lowercase										=							0x6745_2301
			__lowercase										=							0xefcd_ab89
			__lowercase										=							0x98ba_dcfe
			__lowercase										=							0x1032_5476

			__lowercase										=							[
			    7,
			    1_2,
			    1_7,
			    2_2,
			    7,
			    1_2,
			    1_7,
			    2_2,
			    7,
			    1_2,
			    1_7,
			    2_2,
			    7,
			    1_2,
			    1_7,
			    2_2,
			    5,
			    9,
			    1_4,
			    2_0,
			    5,
			    9,
			    1_4,
			    2_0,
			    5,
			    9,
			    1_4,
			    2_0,
			    5,
			    9,
			    1_4,
			    2_0,
			    4,
			    1_1,
			    1_6,
			    2_3,
			    4,
			    1_1,
			    1_6,
			    2_3,
			    4,
			    1_1,
			    1_6,
			    2_3,
			    4,
			    1_1,
			    1_6,
			    2_3,
			    6,
			    1_0,
			    1_5,
			    2_1,
			    6,
			    1_0,
			    1_5,
			    2_1,
			    6,
			    1_0,
			    1_5,
			    2_1,
			    6,
			    1_0,
			    1_5,
			    2_1,
			]

			# Process bit string in chunks, each with 16 32-char words
			for block_words in get_block_words(lowerCamelCase_				):
						__lowercase										=							aa
						__lowercase										=							ba
						__lowercase										=							ca
						__lowercase										=							da

						# Hash current chunk
						for i in range(6_4				):
									if i <= 1_5:
												# f = (b & c) | (not_32(b) & d)     # Alternate definition for f
												__lowercase										=							d ^ (b & (c ^ d))
												__lowercase										=							i
									elif i <= 3_1:
												# f = (d & b) | (not_32(d) & c)     # Alternate definition for f
												__lowercase										=							c ^ (d & (b ^ c))
												__lowercase										=							(5 * i + 1) % 1_6
									elif i <= 4_7:
												__lowercase										=							b ^ c ^ d
												__lowercase										=							(3 * i + 5) % 1_6
									else:
												__lowercase										=							c ^ (b | not_aa(lowerCamelCase_				))
												__lowercase										=							(7 * i) % 1_6
									__lowercase										=							(f + a + added_consts[i] + block_words[g]) % 2**3_2
									__lowercase										=							d
									__lowercase										=							c
									__lowercase										=							b
									__lowercase										=							sum_aa(lowerCamelCase_		,       left_rotate_aa(lowerCamelCase_		,       shift_amounts[i]				)				)

						# Add hashed chunk to running total
						__lowercase										=							sum_aa(lowerCamelCase_		,       lowerCamelCase_				)
						__lowercase										=							sum_aa(lowerCamelCase_		,       lowerCamelCase_				)
						__lowercase										=							sum_aa(lowerCamelCase_		,       lowerCamelCase_				)
						__lowercase										=							sum_aa(lowerCamelCase_		,       lowerCamelCase_				)

			__lowercase										=							reformat_hex(lowerCamelCase_				) + reformat_hex(lowerCamelCase_				) + reformat_hex(lowerCamelCase_				) + reformat_hex(lowerCamelCase_				)
			return digest


if __name__ == "__main__":
	import doctest

	doctest.testmod()












'''simple docstring'''



import json
from typing import Iterator, List, Union

from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, trainers
from tokenizers.implementations.base_tokenizer import BaseTokenizer
from tokenizers.models import Unigram
from tokenizers.processors import TemplateProcessing






class 							__lowercase       (      lowerCAmelCase__						):







	'''simple docstring'''

	def __init__(self     ,_lowerCamelCase = "▁"     ,_lowerCamelCase = True     ,_lowerCamelCase = "<unk>"     ,_lowerCamelCase = "</s>"     ,_lowerCamelCase = "<pad>"     ,)							->							List[Any]:







				'''simple docstring'''





				__lowercase										=							{
				    '''pad''': {'''id''': 0, '''token''': pad_token},
				    '''eos''': {'''id''': 1, '''token''': eos_token},
				    '''unk''': {'''id''': 2, '''token''': unk_token},
				}

				__lowercase										=							[None] * len(self.special_tokens )
				for token_dict in self.special_tokens.values():
							__lowercase										=							token_dict['''token''']

				__lowercase										=							Tokenizer(Unigram() )

				__lowercase										=							normalizers.Sequence(
				    [
				        normalizers.Nmt(),
				        normalizers.NFKC(),
				        normalizers.Replace(Regex(''' {2,}''' )     ,''' ''' ),
				        normalizers.Lowercase(),
				    ] )
				__lowercase										=							pre_tokenizers.Sequence(
				    [
				        pre_tokenizers.Metaspace(replacement=_lowerCamelCase     ,add_prefix_space=_lowerCamelCase ),
				        pre_tokenizers.Digits(individual_digits=_lowerCamelCase ),
				        pre_tokenizers.Punctuation(),
				    ] )
				__lowercase										=							decoders.Metaspace(replacement=_lowerCamelCase     ,add_prefix_space=_lowerCamelCase )

				__lowercase										=							TemplateProcessing(
				    single=f"$A {self.special_tokens['eos']['token']}"     ,special_tokens=[(self.special_tokens['''eos''']['''token'''], self.special_tokens['''eos''']['''id'''])]     ,)

				__lowercase										=							{
				    '''model''': '''SentencePieceUnigram''',
				    '''replacement''': replacement,
				    '''add_prefix_space''': add_prefix_space,
				}

				super().__init__(_lowerCamelCase     ,_lowerCamelCase )


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase = 8000     ,_lowerCamelCase = True     ,)							->							Union[str, Any]:







				'''simple docstring'''





				__lowercase										=							trainers.UnigramTrainer(
				    vocab_size=_lowerCamelCase     ,special_tokens=self.special_tokens_list     ,show_progress=_lowerCamelCase     ,)

				if isinstance(_lowerCamelCase     ,_lowerCamelCase ):
							__lowercase										=							[files]
				self._tokenizer.train(_lowerCamelCase     ,trainer=_lowerCamelCase )

				self.add_unk_id()


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase = 8000     ,_lowerCamelCase = True     ,)							->							List[str]:







				'''simple docstring'''





				__lowercase										=							trainers.UnigramTrainer(
				    vocab_size=_lowerCamelCase     ,special_tokens=self.special_tokens_list     ,show_progress=_lowerCamelCase     ,)

				self._tokenizer.train_from_iterator(_lowerCamelCase     ,trainer=_lowerCamelCase )

				self.add_unk_id()


	def 						_UpperCAmelCase     (self )							->							Tuple:







				'''simple docstring'''





				__lowercase										=							json.loads(self._tokenizer.to_str() )

				__lowercase										=							self.special_tokens['''unk''']['''id''']

				__lowercase										=							Tokenizer.from_str(json.dumps(_lowerCamelCase ) )





