



import re

import jax.numpy as jnp
from flax.traverse_util import flatten_dict, unflatten_dict
from jax.random import PRNGKey

from ..utils import logging


lowercase_								=     logging.get_logger(__name__)



def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				r"\w+[.]\d+"
			lowercase__					  =				re.findall(SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  )
			for pat in pats:
						lowercase__					  =				key.replace(SCREAMING_SNAKE_CASE_						,  "_".join(pat.split("."  )  )  )
			return key



def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  ):
			lowercase__					  =				pt_tuple_key[:-1] + ("scale",)
			if (
			    any("norm" in str_ for str_ in pt_tuple_key  )
			    and (pt_tuple_key[-1] == "bias")
			    and (pt_tuple_key[:-1] + ("bias",) not in random_flax_state_dict)
			    and (pt_tuple_key[:-1] + ("scale",) in random_flax_state_dict)
			):
						lowercase__					  =				pt_tuple_key[:-1] + ("scale",)
						return renamed_pt_tuple_key, pt_tensor
			elif pt_tuple_key[-1] in ["weight", "gamma"] and pt_tuple_key[:-1] + ("scale",) in random_flax_state_dict:
						lowercase__					  =				pt_tuple_key[:-1] + ("scale",)
						return renamed_pt_tuple_key, pt_tensor

			# embedding
			if pt_tuple_key[-1] == "weight" and pt_tuple_key[:-1] + ("embedding",) in random_flax_state_dict:
						lowercase__					  =				pt_tuple_key[:-1] + ("embedding",)
						return renamed_pt_tuple_key, pt_tensor

			# conv layer
			lowercase__					  =				pt_tuple_key[:-1] + ("kernel",)
			if pt_tuple_key[-1] == "weight" and pt_tensor.ndim == 4:
						lowercase__					  =				pt_tensor.transpose(2						,  3						,  1						,  0  )
						return renamed_pt_tuple_key, pt_tensor

			# linear layer
			lowercase__					  =				pt_tuple_key[:-1] + ("kernel",)
			if pt_tuple_key[-1] == "weight":
						lowercase__					  =				pt_tensor.T
						return renamed_pt_tuple_key, pt_tensor

			# old PyTorch layer norm weight
			lowercase__					  =				pt_tuple_key[:-1] + ("weight",)
			if pt_tuple_key[-1] == "gamma":
						return renamed_pt_tuple_key, pt_tensor

			# old PyTorch layer norm bias
			lowercase__					  =				pt_tuple_key[:-1] + ("bias",)
			if pt_tuple_key[-1] == "beta":
						return renamed_pt_tuple_key, pt_tensor

			return pt_tuple_key, pt_tensor




def __lowerCAmelCase (     SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_=42  ):
			# Step 1: Convert pytorch tensor to numpy
			lowercase__					  =				{k: v.numpy() for k, v in pt_state_dict.items()}

			# Step 2: Since the model is stateless, get random Flax params
			lowercase__					  =				flax_model.init_weights(PRNGKey(SCREAMING_SNAKE_CASE_  )  )

			lowercase__					  =				flatten_dict(SCREAMING_SNAKE_CASE_  )
			lowercase__					  =				{}

			# Need to change some parameters name to match Flax names
			for pt_key, pt_tensor in pt_state_dict.items():
						lowercase__					  =				rename_key(SCREAMING_SNAKE_CASE_  )
						lowercase__					  =				tuple(renamed_pt_key.split("."  )  )

						# Correctly rename weight parameters
						lowercase__  ,       lowercase__					  =				rename_key_and_reshape_tensor(SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_						,  SCREAMING_SNAKE_CASE_  )

						if flax_key in random_flax_state_dict:
									if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
												raise ValueError(
												    f'''PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape '''
												    f'''{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}.'''  )

        # also add unexpected weight so that warning is thrown
						lowercase__					  =				jnp.asarray(SCREAMING_SNAKE_CASE_  )

			return unflatten_dict(SCREAMING_SNAKE_CASE_  )









import json
from typing import Iterator, List, Union

from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, trainers
from tokenizers.implementations.base_tokenizer import BaseTokenizer
from tokenizers.models import Unigram
from tokenizers.processors import TemplateProcessing
class 				_snake_case			(					lowercase__):




				def __init__(						self					:						Optional[Any],					__lowercase					:						str = "▁",					__lowercase					:						bool = True,					__lowercase					:						Union[str, AddedToken] = "<unk>",					__lowercase					:						Union[str, AddedToken] = "</s>",					__lowercase					:						Union[str, AddedToken] = "<pad>",					):
							lowercase__					  =				{
							    "pad": {"id": 0, "token": pad_token},
							    "eos": {"id": 1, "token": eos_token},
							    "unk": {"id": 2, "token": unk_token},
							}

							lowercase__					  =				[None] * len(self.special_tokens						)
							for token_dict in self.special_tokens.values():
										lowercase__					  =				token_dict["token"]

							lowercase__					  =				Tokenizer(Unigram()						)

							lowercase__					  =				normalizers.Sequence(
							    [
							        normalizers.Nmt(),
							        normalizers.NFKC(),
							        normalizers.Replace(Regex(" {2,}"						),					" "						),
							        normalizers.Lowercase(),
							    ]						)
							lowercase__					  =				pre_tokenizers.Sequence(
							    [
							        pre_tokenizers.Metaspace(replacement=__lowercase,					add_prefix_space=__lowercase						),
							        pre_tokenizers.Digits(individual_digits=__lowercase						),
							        pre_tokenizers.Punctuation(),
							    ]						)
							lowercase__					  =				decoders.Metaspace(replacement=__lowercase,					add_prefix_space=__lowercase						)

							lowercase__					  =				TemplateProcessing(
							    single=F'''$A {self.special_tokens["eos"]["token"]}''',					special_tokens=[(self.special_tokens["eos"]["token"], self.special_tokens["eos"]["id"])],					)

							lowercase__					  =				{
							    "model": "SentencePieceUnigram",
							    "replacement": replacement,
							    "add_prefix_space": add_prefix_space,
							}

							super().__init__(__lowercase,					__lowercase						)




				def 							A__      (						self					:						Union[str, Any],					__lowercase					:						Union[str, List[str]],					__lowercase					:						int = 8000,					__lowercase					:						bool = True,					):
							lowercase__					  =				trainers.UnigramTrainer(
							    vocab_size=__lowercase,					special_tokens=self.special_tokens_list,					show_progress=__lowercase,					)

							if isinstance(__lowercase,					__lowercase						):
										lowercase__					  =				[files]
							self._tokenizer.train(__lowercase,					trainer=__lowercase						)

							self.add_unk_id()




				def 							A__      (						self					:						List[Any],					__lowercase					:						Union[Iterator[str], Iterator[Iterator[str]]],					__lowercase					:						int = 8000,					__lowercase					:						bool = True,					):
							lowercase__					  =				trainers.UnigramTrainer(
							    vocab_size=__lowercase,					special_tokens=self.special_tokens_list,					show_progress=__lowercase,					)

							self._tokenizer.train_from_iterator(__lowercase,					trainer=__lowercase						)

							self.add_unk_id()




				def 							A__      (						self					:						str						):
							lowercase__					  =				json.loads(self._tokenizer.to_str()						)

							lowercase__					  =				self.special_tokens["unk"]["id"]

							lowercase__					  =				Tokenizer.from_str(json.dumps(__lowercase						)						)




