



import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowercase_	       =			'''▁'''

lowercase_	       =			{'''vocab_file''': '''spiece.model'''}

lowercase_	       =			{
    '''vocab_file''': {'''google/pegasus-xsum''': '''https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model'''}
}

lowercase_	       =			{
    '''google/pegasus-xsum''': 5_12,
}


lowercase_	       =			logging.get_logger(__name__)






class 						__a (     SCREAMING_SNAKE_CASE				):
	SCREAMING_SNAKE_CASE       =       VOCAB_FILES_NAMES

	SCREAMING_SNAKE_CASE       =       VOCAB_FILES_NAMES
	SCREAMING_SNAKE_CASE       =       PRETRAINED_VOCAB_FILES_MAP
	SCREAMING_SNAKE_CASE       =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	SCREAMING_SNAKE_CASE       =       ["input_ids", "attention_mask"]



	def __init__(						self							:    int				,	snake_case_							:    Dict				,	snake_case_							:    Optional[int]="<pad>"				,	snake_case_							:    Tuple="</s>"				,	snake_case_							:    Any="<unk>"				,	snake_case_							:    Any="<mask_2>"				,	snake_case_							:    List[Any]="<mask_1>"				,	snake_case_							:    Any=None				,	snake_case_							:    str=1_03				,	snake_case_							:    Optional[Dict[str, Any]] = None				,	**snake_case_							:    str				,	)->							None:
						__lowerCAmelCase	   =offset
						if additional_special_tokens is not None:
											if not isinstance(snake_case_				,	snake_case_):
																raise TypeError(
																    F"""additional_special_tokens should be of type {type(snake_case_)}, but is"""
																    F""" {type(snake_case_)}""")

											__lowerCAmelCase	   =(
											    ([mask_token_sent] + additional_special_tokens)
											    if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
											    else additional_special_tokens
											)
											# fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
											additional_special_tokens_extended += [
											    F"""<unk_{i}>""" for i in range(len(snake_case_)				,	self.offset - 1)
											]

											if len(set(snake_case_)) != len(snake_case_):
																raise ValueError(
																    """Please make sure that the provided additional_special_tokens do not contain an incorrectly"""
																    F""" shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}.""")
											__lowerCAmelCase	   =additional_special_tokens_extended
						else:
											__lowerCAmelCase	   =[mask_token_sent] if mask_token_sent is not None else []
											additional_special_tokens += [F"""<unk_{i}>""" for i in range(2				,	self.offset)]

						__lowerCAmelCase	   ={} if sp_model_kwargs is None else sp_model_kwargs

						super().__init__(
						    eos_token=snake_case_				,	unk_token=snake_case_				,	mask_token=snake_case_				,	pad_token=snake_case_				,	mask_token_sent=snake_case_				,	offset=snake_case_				,	additional_special_tokens=snake_case_				,	sp_model_kwargs=self.sp_model_kwargs				,	**snake_case_				,	)
						__lowerCAmelCase	   =mask_token_sent
						__lowerCAmelCase	   =vocab_file
						__lowerCAmelCase	   =spm.SentencePieceProcessor(**self.sp_model_kwargs)
						self.sp_model.Load(snake_case_)

						# add special tokens to encoder dict
						__lowerCAmelCase	   ={
						    0: self.pad_token,
						    1: self.eos_token,
						}

						if self.mask_token_sent is not None:
											self.encoder.update(
											    {
											        2: self.mask_token_sent,
											        3: self.mask_token,
											    })

						if self.offset > 0:
											# entries 2-104 are only used for pretraining and called <mask_1>, <mask_2>, unk_2, ...unk_102
											# mask_token_sent is already added to list -> so start at 1
											self.encoder.update({i + 3: additional_special_tokens[i] for i in range(1				,	self.offset - 1)})

						__lowerCAmelCase	   ={v: k for k, v in self.encoder.items()}



	@property
	def 		UpperCamelCase							(						self							:    int)->							int:
						return len(self.sp_model) + self.offset



	def 		UpperCamelCase							(						self							:    Tuple)->							Dict[str, int]:
						__lowerCAmelCase	   ={self.convert_ids_to_tokens(snake_case_): i for i in range(self.vocab_size)}
						vocab.update(self.added_tokens_encoder)
						return vocab



	def __getstate__(						self							:    List[Any])->							Optional[int]:
						__lowerCAmelCase	   =self.__dict__.copy()
						__lowerCAmelCase	   =None
						return state



	def __setstate__(						self							:    Tuple				,	snake_case_							:    int)->							Dict:
						__lowerCAmelCase	   =d

						# for backward compatibility
						if not hasattr(self				,	"""sp_model_kwargs"""):
											__lowerCAmelCase	   ={}

						__lowerCAmelCase	   =spm.SentencePieceProcessor(**self.sp_model_kwargs)
						self.sp_model.Load(self.vocab_file)



	def 		UpperCamelCase							(						self							:    Tuple				,	snake_case_							:    str)->							List[str]:
						return self.sp_model.encode(snake_case_				,	out_type=snake_case_)



	def 		UpperCamelCase							(						self							:    Dict				,	snake_case_							:    str)->							int:
						if token in self.decoder:
											return self.decoder[token]
						elif token in self.added_tokens_decoder:
											return self.added_tokens_decoder[token]
						__lowerCAmelCase	   =self.sp_model.piece_to_id(snake_case_)
						return sp_id + self.offset



	def 		UpperCamelCase							(						self							:    str				,	snake_case_							:    int)->							str:
						if index in self.encoder:
											return self.encoder[index]
						elif index in self.added_tokens_encoder:
											return self.added_tokens_encoder[index]
						else:
											__lowerCAmelCase	   =self.sp_model.IdToPiece(index - self.offset)
						return token



	def 		UpperCamelCase							(						self							:    Optional[Any]				,	snake_case_							:    Any)->							Dict:
						__lowerCAmelCase	   =[]
						__lowerCAmelCase	   =""""""
						for token in tokens:
											# make sure that special tokens are not decoded using sentencepiece model
											if token in self.all_special_tokens:
																out_string += self.sp_model.decode(snake_case_) + token
																__lowerCAmelCase	   =[]
											else:
																current_sub_tokens.append(snake_case_)
						out_string += self.sp_model.decode(snake_case_)
						return out_string.strip()



	def 		UpperCamelCase							(						self							:    Optional[Any]				,	snake_case_							:    str=False)->							List[str]:
						return 1



	def 		UpperCamelCase							(						self							:    Any				,	snake_case_							:    str)->							List[str]:
						__lowerCAmelCase	   =set(self.all_special_ids)  # call it once instead of inside list comp
						all_special_ids.remove(self.unk_token_id)  # <unk> is only sometimes special

						return [1 if x in all_special_ids else 0 for x in seq]



	def 		UpperCamelCase							(						self							:    Any				,	snake_case_							:    List				,	snake_case_							:    Optional[List] = None				,	snake_case_							:    bool = False)->							List[int]:
						if already_has_special_tokens:
											return self._special_token_mask(snake_case_)
						elif token_ids_a is None:
											return self._special_token_mask(snake_case_) + [1]
						else:
											return self._special_token_mask(token_ids_a + token_ids_a) + [1]



	def 		UpperCamelCase							(						self							:    List[str]				,	snake_case_							:    List[str]				,	snake_case_							:    str=None)->							List[int]:
						if token_ids_a is None:
											return token_ids_a + [self.eos_token_id]
						# We don't expect to process pairs, but leave the pair logic for API consistency
						return token_ids_a + token_ids_a + [self.eos_token_id]



	def 		UpperCamelCase							(						self							:    Optional[int]				,	snake_case_							:    str				,	snake_case_							:    Optional[str] = None)->							Tuple[str]:
						if not os.path.isdir(snake_case_):
											logger.error(F"""Vocabulary path ({save_directory}) should be a directory""")
											return
						__lowerCAmelCase	   =os.path.join(
						    snake_case_				,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""])

						if os.path.abspath(self.vocab_file) != os.path.abspath(snake_case_) and os.path.isfile(self.vocab_file):
											copyfile(self.vocab_file				,	snake_case_)
						elif not os.path.isfile(self.vocab_file):
											with open(snake_case_				,	"""wb""") as fi:
																__lowerCAmelCase	   =self.sp_model.serialized_model_proto()
																fi.write(snake_case_)

						return (out_vocab_file,)









import time
from dataclasses import dataclass
from multiprocessing import Pool
from unittest import TestCase
from unittest.mock import patch

import multiprocess
import numpy as np
import pytest

from datasets.utils.py_utils import (
    NestedDataStructure,
    asdict,
    iflatmap_unordered,
    map_nested,
    temp_seed,
    temporary_assignment,
    zip_dict,
)

from .utils import require_tf, require_torch






def 					__lowerCAmelCase		(  __lowerCamelCase	:    List[Any]	)  ->						int:  # picklable for multiprocessing
					return x.sum()






def 					__lowerCAmelCase		(  __lowerCamelCase	:    Any	)  ->						Any:  # picklable for multiprocessing
					return i + 1






@dataclass
class 						__a :
	SCREAMING_SNAKE_CASE       =       42
	SCREAMING_SNAKE_CASE       =       42






class 						__a (     SCREAMING_SNAKE_CASE				):



	def 		UpperCamelCase							(						self							:    List[Any])->							Tuple:
						__lowerCAmelCase	   ={}
						__lowerCAmelCase	   =[]
						__lowerCAmelCase	   =1
						__lowerCAmelCase	   =[1, 2]
						__lowerCAmelCase	   ={"""a""": 1, """b""": 2}
						__lowerCAmelCase	   ={"""a""": [1, 2], """b""": [3, 4]}
						__lowerCAmelCase	   ={"""a""": {"""1""": 1}, """b""": 2}
						__lowerCAmelCase	   ={"""a""": 1, """b""": 2, """c""": 3, """d""": 4}
						__lowerCAmelCase	   ={}
						__lowerCAmelCase	   =[]
						__lowerCAmelCase	   =2
						__lowerCAmelCase	   =[2, 3]
						__lowerCAmelCase	   ={"""a""": 2, """b""": 3}
						__lowerCAmelCase	   ={"""a""": [2, 3], """b""": [4, 5]}
						__lowerCAmelCase	   ={"""a""": {"""1""": 2}, """b""": 3}
						__lowerCAmelCase	   ={"""a""": 2, """b""": 3, """c""": 4, """d""": 5}
						self.assertEqual(map_nested(snake_case_				,	snake_case_)				,	snake_case_)
						self.assertEqual(map_nested(snake_case_				,	snake_case_)				,	snake_case_)
						self.assertEqual(map_nested(snake_case_				,	snake_case_)				,	snake_case_)
						self.assertEqual(map_nested(snake_case_				,	snake_case_)				,	snake_case_)
						self.assertEqual(map_nested(snake_case_				,	snake_case_)				,	snake_case_)
						self.assertEqual(map_nested(snake_case_				,	snake_case_)				,	snake_case_)
						self.assertEqual(map_nested(snake_case_				,	snake_case_)				,	snake_case_)
						self.assertEqual(map_nested(snake_case_				,	snake_case_)				,	snake_case_)

						__lowerCAmelCase	   =2
						self.assertEqual(map_nested(snake_case_				,	snake_case_				,	num_proc=snake_case_)				,	snake_case_)
						self.assertEqual(map_nested(snake_case_				,	snake_case_				,	num_proc=snake_case_)				,	snake_case_)
						self.assertEqual(map_nested(snake_case_				,	snake_case_				,	num_proc=snake_case_)				,	snake_case_)
						self.assertEqual(map_nested(snake_case_				,	snake_case_				,	num_proc=snake_case_)				,	snake_case_)
						self.assertEqual(map_nested(snake_case_				,	snake_case_				,	num_proc=snake_case_)				,	snake_case_)
						self.assertEqual(map_nested(snake_case_				,	snake_case_				,	num_proc=snake_case_)				,	snake_case_)
						self.assertEqual(map_nested(snake_case_				,	snake_case_				,	num_proc=snake_case_)				,	snake_case_)
						self.assertEqual(map_nested(snake_case_				,	snake_case_				,	num_proc=snake_case_)				,	snake_case_)

						__lowerCAmelCase	   ={"""a""": np.eye(2), """b""": np.zeros(3), """c""": np.ones(2)}
						__lowerCAmelCase	   ={"""a""": 2, """b""": 0, """c""": 2}
						__lowerCAmelCase	   ={
						    """a""": np.eye(2).astype(snake_case_),
						    """b""": np.zeros(3).astype(snake_case_),
						    """c""": np.ones(2).astype(snake_case_),
						}
						self.assertEqual(map_nested(snake_case_				,	snake_case_				,	map_numpy=snake_case_)				,	snake_case_)
						self.assertEqual(
						    {k: v.tolist() for k, v in map_nested(snake_case_				,	snake_case_				,	map_numpy=snake_case_).items()}				,	{k: v.tolist() for k, v in expected_map_nested_sna_int.items()}				,	)
						self.assertEqual(map_nested(snake_case_				,	snake_case_				,	map_numpy=snake_case_				,	num_proc=snake_case_)				,	snake_case_)
						self.assertEqual(
						    {k: v.tolist() for k, v in map_nested(snake_case_				,	snake_case_				,	map_numpy=snake_case_				,	num_proc=snake_case_).items()}				,	{k: v.tolist() for k, v in expected_map_nested_sna_int.items()}				,	)
						with self.assertRaises(snake_case_):  # can't pickle a local lambda
											map_nested(lambda snake_case_: x + 1				,	snake_case_				,	num_proc=snake_case_)



	def 		UpperCamelCase							(						self							:    Any)->							int:
						__lowerCAmelCase	   ={"""a""": 1, """b""": 2}
						__lowerCAmelCase	   ={"""a""": 3, """b""": 4}
						__lowerCAmelCase	   ={"""a""": 5, """b""": 6}
						__lowerCAmelCase	   =sorted([("""a""", (1, 3, 5)), ("""b""", (2, 4, 6))])
						self.assertEqual(sorted(zip_dict(snake_case_				,	snake_case_				,	snake_case_))				,	snake_case_)



	def 		UpperCamelCase							(						self							:    Optional[Any])->							Optional[int]:






						class 						__a :
							SCREAMING_SNAKE_CASE       =       "bar"


						__lowerCAmelCase	   =Foo()
						self.assertEqual(foo.my_attr				,	"""bar""")
						with temporary_assignment(snake_case_				,	"""my_attr"""				,	"""BAR"""):
											self.assertEqual(foo.my_attr				,	"""BAR""")
						self.assertEqual(foo.my_attr				,	"""bar""")






@pytest.mark.parametrize(
    """iterable_length, num_proc, expected_num_proc"""				,   [
        (1, None, 1),
        (1, 1, 1),
        (2, None, 1),
        (2, 1, 1),
        (2, 2, 1),
        (2, 3, 1),
        (3, 2, 1),
        (16, 16, 16),
        (16, 17, 16),
        (17, 16, 16),
    ]				,   )
def 					__lowerCAmelCase		(  __lowerCamelCase	:    List[str]				,   __lowerCamelCase	:    Any				,   __lowerCamelCase	:    Dict	)  ->						Union[str, Any]:
					with patch("""datasets.utils.py_utils._single_map_nested"""	) as mock_single_map_nested, patch(
					    """datasets.parallel.parallel.Pool"""	) as mock_multiprocessing_pool:
										__lowerCAmelCase	   ={f"""{i}""": i for i in range(__lowerCamelCase	)}
										__lowerCAmelCase	   =map_nested(lambda __lowerCamelCase	: x + 10				,   __lowerCamelCase				,   num_proc=__lowerCamelCase				,   parallel_min_length=16	)
										if expected_num_proc == 1:
															assert mock_single_map_nested.called
															assert not mock_multiprocessing_pool.called
										else:
															assert not mock_single_map_nested.called
															assert mock_multiprocessing_pool.called
															assert mock_multiprocessing_pool.call_args[0][0] == expected_num_proc






class 						__a (     SCREAMING_SNAKE_CASE				):



	@require_tf
	def 		UpperCamelCase							(						self							:    Tuple)->							Optional[int]:
						import tensorflow as tf
						from tensorflow.keras import layers

						__lowerCAmelCase	   =layers.Dense(2)

						def gen_random_output():
											__lowerCAmelCase	   =tf.random.uniform((1, 3))
											return model(snake_case_).numpy()

						with temp_seed(42				,	set_tensorflow=snake_case_):
											__lowerCAmelCase	   =gen_random_output()
						with temp_seed(42				,	set_tensorflow=snake_case_):
											__lowerCAmelCase	   =gen_random_output()
						__lowerCAmelCase	   =gen_random_output()

						np.testing.assert_equal(snake_case_				,	snake_case_)
						self.assertGreater(np.abs(outa - outa).sum()				,	0)



	@require_torch
	def 		UpperCamelCase							(						self							:    Dict)->							Dict:
						import torch

						def gen_random_output():
											__lowerCAmelCase	   =torch.nn.Linear(3				,	2)
											__lowerCAmelCase	   =torch.rand(1				,	3)
											return model(snake_case_).detach().numpy()

						with temp_seed(42				,	set_pytorch=snake_case_):
											__lowerCAmelCase	   =gen_random_output()
						with temp_seed(42				,	set_pytorch=snake_case_):
											__lowerCAmelCase	   =gen_random_output()
						__lowerCAmelCase	   =gen_random_output()

						np.testing.assert_equal(snake_case_				,	snake_case_)
						self.assertGreater(np.abs(outa - outa).sum()				,	0)



	def 		UpperCamelCase							(						self							:    Union[str, Any])->							List[Any]:
						def gen_random_output():
											return np.random.rand(1				,	3)

						with temp_seed(42):
											__lowerCAmelCase	   =gen_random_output()
						with temp_seed(42):
											__lowerCAmelCase	   =gen_random_output()
						__lowerCAmelCase	   =gen_random_output()

						np.testing.assert_equal(snake_case_				,	snake_case_)
						self.assertGreater(np.abs(outa - outa).sum()				,	0)






@pytest.mark.parametrize("""input_data"""				,   [{}]	)
def 					__lowerCAmelCase		(  __lowerCamelCase	:    List[str]	)  ->						List[str]:
					__lowerCAmelCase	   =NestedDataStructure(__lowerCamelCase	).data
					assert output_data == input_data






@pytest.mark.parametrize(
    """data, expected_output"""				,   [
        ({}, []),
        ([], []),
        ("""foo""", ["""foo"""]),
        (["""foo""", """bar"""], ["""foo""", """bar"""]),
        ([["""foo""", """bar"""]], ["""foo""", """bar"""]),
        ([[["""foo"""], ["""bar"""]]], ["""foo""", """bar"""]),
        ([[["""foo"""], """bar"""]], ["""foo""", """bar"""]),
        ({"""a""": 1, """b""": 2}, [1, 2]),
        ({"""a""": [1, 2], """b""": [3, 4]}, [1, 2, 3, 4]),
        ({"""a""": [[1, 2]], """b""": [[3, 4]]}, [1, 2, 3, 4]),
        ({"""a""": [[1, 2]], """b""": [3, 4]}, [1, 2, 3, 4]),
        ({"""a""": [[[1], [2]]], """b""": [[[3], [4]]]}, [1, 2, 3, 4]),
        ({"""a""": [[[1], [2]]], """b""": [[3, 4]]}, [1, 2, 3, 4]),
        ({"""a""": [[[1], [2]]], """b""": [3, 4]}, [1, 2, 3, 4]),
        ({"""a""": [[[1], [2]]], """b""": [3, [4]]}, [1, 2, 3, 4]),
        ({"""a""": {"""1""": 1}, """b""": 2}, [1, 2]),
        ({"""a""": {"""1""": [1]}, """b""": 2}, [1, 2]),
        ({"""a""": {"""1""": [1]}, """b""": [2]}, [1, 2]),
    ]				,   )
def 					__lowerCAmelCase		(  __lowerCamelCase	:    List[Any]				,   __lowerCamelCase	:    int	)  ->						List[str]:
					__lowerCAmelCase	   =NestedDataStructure(__lowerCamelCase	).flatten()
					assert output == expected_output






def 					__lowerCAmelCase		(  )  ->						int:
					__lowerCAmelCase	   =A(x=1				,   y="""foobar"""	)
					__lowerCAmelCase	   ={"""x""": 1, """y""": """foobar"""}
					assert asdict(__lowerCamelCase	) == expected_output

					__lowerCAmelCase	   ={"""a""": {"""b""": A(x=10				,   y="""foo"""	)}, """c""": [A(x=20				,   y="""bar"""	)]}
					__lowerCAmelCase	   ={"""a""": {"""b""": {"""x""": 10, """y""": """foo"""}}, """c""": [{"""x""": 20, """y""": """bar"""}]}
					assert asdict(__lowerCamelCase	) == expected_output

					with pytest.raises(__lowerCamelCase	):
										asdict([1, A(x=10				,   y="""foo"""	)]	)






def 					__lowerCAmelCase		(  __lowerCamelCase	:    str	)  ->						Dict:
					return text.split()






def 					__lowerCAmelCase		(  __lowerCamelCase	:    List[Any]	)  ->						Any:
					yield (time.time(), content)
					time.sleep(2	)
					yield (time.time(), content)






def 					__lowerCAmelCase		(  )  ->						Any:
					with Pool(2	) as pool:
										__lowerCAmelCase	   =list(iflatmap_unordered(__lowerCamelCase				,   _split_text				,   kwargs_iterable=[{"""text""": """hello there"""}] * 10	)	)
										assert out.count("""hello"""	) == 10
										assert out.count("""there"""	) == 10
										assert len(__lowerCamelCase	) == 20

					# check multiprocess from pathos (uses dill for pickling)
					with multiprocess.Pool(2	) as pool:
										__lowerCAmelCase	   =list(iflatmap_unordered(__lowerCamelCase				,   _split_text				,   kwargs_iterable=[{"""text""": """hello there"""}] * 10	)	)
										assert out.count("""hello"""	) == 10
										assert out.count("""there"""	) == 10
										assert len(__lowerCamelCase	) == 20

					# check that we get items as fast as possible
					with Pool(2	) as pool:
										__lowerCAmelCase	   =[]
										for yield_time, content in iflatmap_unordered(
										    __lowerCamelCase				,   _aseconds_generator_of_aitems_with_timing				,   kwargs_iterable=[{"""content""": """a"""}, {"""content""": """b"""}]	):
															assert yield_time < time.time() + 0.1, "we should each item directly after it was yielded"
															out.append(__lowerCamelCase	)
										assert out.count("""a"""	) == 2
										assert out.count("""b"""	) == 2
										assert len(__lowerCamelCase	) == 4




