





import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SPIECE_UNDERLINE, is_sentencepiece_available
from transformers.models.speech_to_text import SpeechaTextTokenizer
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__lowercase       :Dict       =  get_tests_dir("fixtures/test_sentencepiece.model")

if is_sentencepiece_available():
				import sentencepiece as sp


__lowercase       :Optional[int]       =  5
__lowercase       :Optional[Any]       =  10



@require_sentencepiece
@require_tokenizers
class    _a     (				lowercase__					,							unittest.TestCase							):






							"""simple docstring"""

							snake_case_							 =						SpeechaTextTokenizer
							snake_case_							 =						False
							snake_case_							 =						True

							def      A_  (      self				:     Union[str, Any]    )	->Optional[Any]:
														super().setUp()

														SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					sp.SentencePieceProcessor()
														spm_model.Load(a    )
														SCREAMING_SNAKE_CASE__     :				Any   =					["<s>", "<pad>", "</s>", "<unk>"]

														vocab += [spm_model.IdToPiece(id_    ) for id_ in range(len(a    )    )]
														SCREAMING_SNAKE_CASE__     :				List[str]   =					dict(zip(a							,	range(len(a    )    )    )    )

														SCREAMING_SNAKE_CASE__     :				str   =					Path(self.tmpdirname    )
														save_json(a							,	save_dir / VOCAB_FILES_NAMES["vocab_file"]    )
														if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
																					copyfile(a							,	save_dir / VOCAB_FILES_NAMES["spm_file"]    )

														SCREAMING_SNAKE_CASE__     :				Optional[int]   =					SpeechaTextTokenizer.from_pretrained(self.tmpdirname    )
														tokenizer.save_pretrained(self.tmpdirname    )

							def      A_  (      self				:     Optional[int]    )	->Optional[int]:
														SCREAMING_SNAKE_CASE__     :				Dict   =					"<pad>"
														SCREAMING_SNAKE_CASE__     :				int   =					1

														self.assertEqual(self.get_tokenizer()._convert_token_to_id(a    )							,	a    )
														self.assertEqual(self.get_tokenizer()._convert_id_to_token(a    )							,	a    )

							def      A_  (      self				:     Dict    )	->List[str]:
														SCREAMING_SNAKE_CASE__     :				Union[str, Any]   =					list(self.get_tokenizer().get_vocab().keys()    )

														self.assertEqual(vocab_keys[0]							,	"<s>"    )
														self.assertEqual(vocab_keys[1]							,	"<pad>"    )
														self.assertEqual(vocab_keys[-1]							,	"j"    )
														self.assertEqual(len(a    )							,	10_01    )

							def      A_  (      self				:     List[Any]    )	->Tuple:
														self.assertEqual(self.get_tokenizer().vocab_size							,	10_01    )

							def      A_  (      self				:     int    )	->List[str]:
														SCREAMING_SNAKE_CASE__     :				str   =					SpeechaTextTokenizer.from_pretrained(self.tmpdirname    )

														SCREAMING_SNAKE_CASE__     :				Dict   =					tokenizer.tokenize("This is a test"    )
														self.assertListEqual(a							,	["▁This", "▁is", "▁a", "▁t", "est"]    )

														self.assertListEqual(
														    tokenizer.convert_tokens_to_ids(a    )							,	[2_89, 50, 14, 1_74, 3_86]							,	)

														SCREAMING_SNAKE_CASE__     :				Any   =					tokenizer.tokenize("I was born in 92000, and this is falsé."    )
														self.assertListEqual(
														    a							,	[SPIECE_UNDERLINE + "I", SPIECE_UNDERLINE + "was", SPIECE_UNDERLINE + "b", "or", "n", SPIECE_UNDERLINE + "in", SPIECE_UNDERLINE + "", "9", "2", "0", "0", "0", ",", SPIECE_UNDERLINE + "and", SPIECE_UNDERLINE + "this", SPIECE_UNDERLINE + "is", SPIECE_UNDERLINE + "f", "al", "s", "é", "."]							,	)
														SCREAMING_SNAKE_CASE__     :				str   =					tokenizer.convert_tokens_to_ids(a    )
														self.assertListEqual(a							,	[12, 25, 88, 59, 28, 23, 11, 4, 6_06, 3_51, 3_51, 3_51, 7, 16, 70, 50, 76, 84, 10, 4, 8]    )

														SCREAMING_SNAKE_CASE__     :				List[str]   =					tokenizer.convert_ids_to_tokens(a    )
														self.assertListEqual(
														    a							,	[SPIECE_UNDERLINE + "I", SPIECE_UNDERLINE + "was", SPIECE_UNDERLINE + "b", "or", "n", SPIECE_UNDERLINE + "in", SPIECE_UNDERLINE + "", "<unk>", "2", "0", "0", "0", ",", SPIECE_UNDERLINE + "and", SPIECE_UNDERLINE + "this", SPIECE_UNDERLINE + "is", SPIECE_UNDERLINE + "f", "al", "s", "<unk>", "."]							,	)






							@slow
							def      A_  (      self				:     List[str]    )	->Tuple:
														# fmt: off
														SCREAMING_SNAKE_CASE__     :				Tuple   =					{"input_ids": [[37_91, 7_97, 31, 11, 64, 7_97, 31, 24_29, 4_33, 12, 11_76, 12, 20, 7_86, 9_15, 1_42, 24_13, 2_40, 37, 32_38, 7_97, 31, 11, 35, 93, 9_15, 1_42, 24_13, 2_40, 37, 55_40, 5_67, 12_76, 93, 37, 6_10, 40, 62, 4_55, 6_57, 10_42, 1_23, 7_80, 1_77, 37, 3_09, 2_41, 12_98, 5_14, 20, 2_92, 27_37, 1_14, 24_69, 2_41, 85, 64, 3_02, 5_48, 5_28, 4_23, 4, 5_09, 4_06, 4_23, 37, 6_01, 4, 7_77, 3_02, 5_48, 5_28, 4_23, 2_84, 4, 33_88, 5_11, 4_59, 4, 35_55, 40, 3_21, 3_02, 7_05, 4, 33_88, 5_11, 5_83, 3_26, 5, 5, 5, 62, 33_10, 5_60, 1_77, 26_80, 2_17, 15_08, 32, 31, 8_53, 4_18, 64, 5_83, 5_11, 16_05, 62, 35, 93, 5_60, 1_77, 26_80, 2_17, 15_08, 15_21, 64, 5_83, 5_11, 5_19, 62, 20, 15_15, 7_64, 20, 1_49, 2_61, 56_25, 79_72, 20, 55_40, 5_67, 12_76, 93, 39_25, 16_75, 11, 15, 8_02, 79_72, 5_76, 2_17, 15_08, 11, 35, 93, 12_53, 24_41, 15, 2_89, 6_52, 31, 4_16, 3_21, 38_42, 1_15, 40, 9_11, 8, 4_76, 6_19, 4, 3_80, 1_42, 4_23, 3_35, 2_40, 35, 93, 2_64, 8, 11, 3_35, 5_69, 4_20, 1_63, 5, 2], [2_60, 5_48, 5_28, 4_23, 20, 4_51, 20, 26_81, 11_53, 34_34, 20, 55_40, 37, 5_67, 1_26, 12_53, 24_41, 33_76, 4_49, 2_10, 4_31, 15_63, 1_77, 7_67, 55_40, 11, 12_03, 4_72, 11, 29_53, 6_85, 2_85, 3_64, 7_06, 11_53, 20, 67_99, 20, 28_69, 20, 44_64, 1_26, 40, 24_29, 20, 10_40, 8_66, 26_64, 4_18, 20, 3_18, 20, 17_26, 1_86, 20, 2_65, 5_22, 35, 93, 21_91, 46_34, 20, 10_40, 12, 67_99, 15, 2_28, 23_56, 1_42, 31, 11, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [25_75, 26_66, 6_84, 15_82, 11_76, 12, 6_27, 1_49, 6_19, 20, 49_02, 5_63, 11, 20, 1_49, 2_61, 34_20, 23_56, 1_74, 1_42, 47_14, 1_31, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
														# fmt: on

														self.tokenizer_integration_test_util(
														    expected_encoding=a							,	model_name="facebook/s2t-small-mustc-en-de-st"							,	revision="a14f04cf0776c02f62a8cb800cf7909e15ea23ad"							,	)







@require_sentencepiece
class    _a     (				unittest.TestCase							):






							"""simple docstring"""

							snake_case_							 =						"valhalla/s2t_mustc_multilinguial_medium"

							snake_case_							 =						"C'est trop cool"
							snake_case_							 =						"Esto es genial"

							@classmethod
							def      A_  (      cls				:     List[Any]    )	->Optional[int]:
														SCREAMING_SNAKE_CASE__     :				SpeechaTextTokenizer   =					SpeechaTextTokenizer.from_pretrained(cls.checkpoint_name    )
														return cls

							def      A_  (      self				:     Any    )	->Dict:
														self.assertEqual(self.tokenizer.lang_code_to_id["pt"]							,	4    )
														self.assertEqual(self.tokenizer.lang_code_to_id["ru"]							,	6    )
														self.assertEqual(self.tokenizer.lang_code_to_id["it"]							,	9    )
														self.assertEqual(self.tokenizer.lang_code_to_id["de"]							,	11    )

							def      A_  (      self				:     Union[str, Any]    )	->str:
														self.assertEqual(self.tokenizer.vocab_size							,	1_00_00    )

							def      A_  (      self				:     int    )	->Union[str, Any]:
														self.assertIn(a							,	self.tokenizer.all_special_ids    )
														SCREAMING_SNAKE_CASE__     :				List[Any]   =					[ES_CODE, 4, 16_01, 47, 76_47, 2]
														SCREAMING_SNAKE_CASE__     :				List[Any]   =					self.tokenizer.decode(a							,	skip_special_tokens=a    )
														SCREAMING_SNAKE_CASE__     :				str   =					self.tokenizer.decode(generated_ids[1:]							,	skip_special_tokens=a    )
														self.assertEqual(a							,	a    )
														self.assertNotIn(self.tokenizer.eos_token							,	a    )

							def      A_  (      self				:     List[str]    )	->Union[str, Any]:
														SCREAMING_SNAKE_CASE__     :				Tuple   =					"fr"
														SCREAMING_SNAKE_CASE__     :				Any   =					self.tokenizer(self.french_text    ).input_ids
														self.assertEqual(encoded[0]							,	a    )
														self.assertEqual(encoded[-1]							,	self.tokenizer.eos_token_id    )






							def      A_  (      self				:     int    )	->Optional[int]:
														SCREAMING_SNAKE_CASE__     :				int   =					"fr"
														self.assertListEqual(self.tokenizer.prefix_tokens							,	[FR_CODE]    )

														SCREAMING_SNAKE_CASE__     :				Optional[Any]   =					"es"
														self.assertListEqual(self.tokenizer.prefix_tokens							,	[ES_CODE]    )






# tests directory-specific settings - this file is run automatically
# by pytest before any tests are run

import doctest
import sys
import warnings
from os.path import abspath, dirname, join

import _pytest

from transformers.testing_utils import HfDoctestModule, HfDocTestParser


# allow having multiple repository checkouts and not needing to remember to rerun
# 'pip install -e .[dev]' when switching between checkouts and running tests.
__lowercase       :List[Any]       =  abspath(join(dirname(__file__), "src"))
sys.path.insert(1, git_repo_path)

# silence FutureWarning warnings in tests since often we can't act on them until
# they become normal warnings - i.e. the tests still need to test the current functionality
warnings.simplefilter(action="ignore", category=FutureWarning)







def       UpperCAmelCase							(     _lowerCamelCase      :  int     ):





						'''simple docstring'''


						config.addinivalue_line(
						    "markers"			,   "is_pt_tf_cross_test: mark test to run only when PT and TF interactions are tested"     )
						config.addinivalue_line(
						    "markers"			,   "is_pt_flax_cross_test: mark test to run only when PT and FLAX interactions are tested"     )
						config.addinivalue_line("markers"			,   "is_pipeline_test: mark test to run only when pipelines are tested"     )
						config.addinivalue_line("markers"			,   "is_staging_test: mark test to run only in the staging environment"     )
						config.addinivalue_line("markers"			,   "accelerate_tests: mark test that require accelerate"     )
						config.addinivalue_line("markers"			,   "tool_tests: mark the tool tests that are run on their specific schedule"     )







def       UpperCAmelCase							(     _lowerCamelCase      :  str     ):





						'''simple docstring'''


						from transformers.testing_utils import pytest_addoption_shared

						pytest_addoption_shared(_lowerCamelCase     )







def       UpperCAmelCase							(     _lowerCamelCase      :  Tuple     ):





						'''simple docstring'''


						from transformers.testing_utils import pytest_terminal_summary_main

						SCREAMING_SNAKE_CASE__     :				List[str]   =					terminalreporter.config.getoption("--make-reports"     )
						if make_reports:
												pytest_terminal_summary_main(_lowerCamelCase			,   id=_lowerCamelCase     )







def       UpperCAmelCase							(     _lowerCamelCase      :  List[Any]			,   _lowerCamelCase      :  Dict     ):





						'''simple docstring'''


						if exitstatus == 5:
												SCREAMING_SNAKE_CASE__     :				List[str]   =					0


# Doctest custom flag to ignore output.
__lowercase       :Optional[Any]       =  doctest.register_optionflag("IGNORE_RESULT")

__lowercase       :Dict       =  doctest.OutputChecker



class    _a     (				lowercase__							):






							"""simple docstring"""

							def      A_  (      self				:     Dict							,	a				:     List[str]							,	a				:     Dict							,	a				:     int    )	->Optional[Any]:
													if IGNORE_RESULT & optionflags:
																			return True
													return OutputChecker.check_output(self							,	a							,	a							,	a    )







__lowercase       :Any       =  CustomOutputChecker
__lowercase       :Any       =  HfDoctestModule



__lowercase       :int       =  HfDocTestParser