






import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SPIECE_UNDERLINE, is_sentencepiece_available
from transformers.models.speech_to_text import SpeechaTextTokenizer
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


snake_case    				=			get_tests_dir("""fixtures/test_sentencepiece.model""")

if is_sentencepiece_available():
   import sentencepiece as sp


snake_case    				=			5
snake_case    				=			10


@require_sentencepiece
@require_tokenizers
class  SCREAMING_SNAKE_CASE			(			lowerCAmelCase      ,		unittest.TestCase					):
     '''simple docstring'''


     UpperCamelCase_							:		Union[str, Any]			=	SpeechaTextTokenizer
     UpperCamelCase_							:		List[Any]			=	False
     UpperCamelCase_							:		Dict			=	True


     def 				_A       (				self       :	List[Any]      ):
       super().setUp()

       SCREAMING_SNAKE_CASE       :      Tuple							=    sp.SentencePieceProcessor()
       spm_model.Load(UpperCAmelCase_      )
       SCREAMING_SNAKE_CASE       :      Any							=    ["<s>", "<pad>", "</s>", "<unk>"]

       vocab += [spm_model.IdToPiece(id_      ) for id_ in range(len(UpperCAmelCase_      )      )]
       SCREAMING_SNAKE_CASE       :      Any							=    dict(zip(UpperCAmelCase_					, range(len(UpperCAmelCase_      )      )      )      )

       SCREAMING_SNAKE_CASE       :      Tuple							=    Path(self.tmpdirname      )
       save_json(UpperCAmelCase_					, save_dir / VOCAB_FILES_NAMES["vocab_file"]      )
       if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
         copyfile(UpperCAmelCase_					, save_dir / VOCAB_FILES_NAMES["spm_file"]      )

       SCREAMING_SNAKE_CASE       :      int							=    SpeechaTextTokenizer.from_pretrained(self.tmpdirname      )
       tokenizer.save_pretrained(self.tmpdirname      )


     def 				_A       (				self       :	Optional[Any]      ):
       SCREAMING_SNAKE_CASE       :      Optional[int]							=    "<pad>"
       SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    1

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCAmelCase_      )					, UpperCAmelCase_      )
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCAmelCase_      )					, UpperCAmelCase_      )


     def 				_A       (				self       :	Any      ):
       SCREAMING_SNAKE_CASE       :      Optional[Any]							=    list(self.get_tokenizer().get_vocab().keys()      )

       self.assertEqual(vocab_keys[0]					, "<s>"      )
       self.assertEqual(vocab_keys[1]					, "<pad>"      )
       self.assertEqual(vocab_keys[-1]					, "j"      )
       self.assertEqual(len(UpperCAmelCase_      )					, 1001      )


     def 				_A       (				self       :	List[str]      ):
       self.assertEqual(self.get_tokenizer().vocab_size					, 1001      )


     def 				_A       (				self       :	int      ):
       SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    SpeechaTextTokenizer.from_pretrained(self.tmpdirname      )

       SCREAMING_SNAKE_CASE       :      List[str]							=    tokenizer.tokenize("This is a test"      )
       self.assertListEqual(UpperCAmelCase_					, ["▁This", "▁is", "▁a", "▁t", "est"]      )

       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(UpperCAmelCase_      )					, [289, 50, 14, 174, 386]					, )

       SCREAMING_SNAKE_CASE       :      Optional[Any]							=    tokenizer.tokenize("I was born in 92000, and this is falsé."      )
       self.assertListEqual(
           UpperCAmelCase_					, [SPIECE_UNDERLINE + "I", SPIECE_UNDERLINE + "was", SPIECE_UNDERLINE + "b", "or", "n", SPIECE_UNDERLINE + "in", SPIECE_UNDERLINE + "", "9", "2", "0", "0", "0", ",", SPIECE_UNDERLINE + "and", SPIECE_UNDERLINE + "this", SPIECE_UNDERLINE + "is", SPIECE_UNDERLINE + "f", "al", "s", "é", "."]					, )
       SCREAMING_SNAKE_CASE       :      List[str]							=    tokenizer.convert_tokens_to_ids(UpperCAmelCase_      )
       self.assertListEqual(UpperCAmelCase_					, [12, 25, 88, 59, 28, 23, 11, 4, 606, 351, 351, 351, 7, 16, 70, 50, 76, 84, 10, 4, 8]      )

       SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    tokenizer.convert_ids_to_tokens(UpperCAmelCase_      )
       self.assertListEqual(
           UpperCAmelCase_					, [SPIECE_UNDERLINE + "I", SPIECE_UNDERLINE + "was", SPIECE_UNDERLINE + "b", "or", "n", SPIECE_UNDERLINE + "in", SPIECE_UNDERLINE + "", "<unk>", "2", "0", "0", "0", ",", SPIECE_UNDERLINE + "and", SPIECE_UNDERLINE + "this", SPIECE_UNDERLINE + "is", SPIECE_UNDERLINE + "f", "al", "s", "<unk>", "."]					, )






     @slow
     def 				_A       (				self       :	int      ):
       # fmt: off
       SCREAMING_SNAKE_CASE       :      Dict							=    {"input_ids": [[3791, 797, 31, 11, 64, 797, 31, 2429, 433, 12, 1176, 12, 20, 786, 915, 142, 2413, 240, 37, 3238, 797, 31, 11, 35, 93, 915, 142, 2413, 240, 37, 5540, 567, 1276, 93, 37, 610, 40, 62, 455, 657, 1042, 123, 780, 177, 37, 309, 241, 1298, 514, 20, 292, 2737, 114, 2469, 241, 85, 64, 302, 548, 528, 423, 4, 509, 406, 423, 37, 601, 4, 777, 302, 548, 528, 423, 284, 4, 3388, 511, 459, 4, 3555, 40, 321, 302, 705, 4, 3388, 511, 583, 326, 5, 5, 5, 62, 3310, 560, 177, 2680, 217, 1508, 32, 31, 853, 418, 64, 583, 511, 1605, 62, 35, 93, 560, 177, 2680, 217, 1508, 1521, 64, 583, 511, 519, 62, 20, 1515, 764, 20, 149, 261, 5625, 7972, 20, 5540, 567, 1276, 93, 3925, 1675, 11, 15, 802, 7972, 576, 217, 1508, 11, 35, 93, 1253, 2441, 15, 289, 652, 31, 416, 321, 3842, 115, 40, 911, 8, 476, 619, 4, 380, 142, 423, 335, 240, 35, 93, 264, 8, 11, 335, 569, 420, 163, 5, 2], [260, 548, 528, 423, 20, 451, 20, 2681, 1153, 3434, 20, 5540, 37, 567, 126, 1253, 2441, 3376, 449, 210, 431, 1563, 177, 767, 5540, 11, 1203, 472, 11, 2953, 685, 285, 364, 706, 1153, 20, 6799, 20, 2869, 20, 4464, 126, 40, 2429, 20, 1040, 866, 2664, 418, 20, 318, 20, 1726, 186, 20, 265, 522, 35, 93, 2191, 4634, 20, 1040, 12, 6799, 15, 228, 2356, 142, 31, 11, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [2575, 2666, 684, 1582, 1176, 12, 627, 149, 619, 20, 4902, 563, 11, 20, 149, 261, 3420, 2356, 174, 142, 4714, 131, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
       # fmt: on

       self.tokenizer_integration_test_util(
           expected_encoding=UpperCAmelCase_					, model_name="facebook/s2t-small-mustc-en-de-st"					, revision="a14f04cf0776c02f62a8cb800cf7909e15ea23ad"					, )





@require_sentencepiece
class  SCREAMING_SNAKE_CASE			(			unittest.TestCase					):
     '''simple docstring'''


     UpperCamelCase_							:		Dict			=	'''valhalla/s2t_mustc_multilinguial_medium'''

     UpperCamelCase_							:		int			=	'''C\'est trop cool'''
     UpperCamelCase_							:		Any			=	'''Esto es genial'''


     @classmethod
     def 				_A       (				cls       :	int      ):
       SCREAMING_SNAKE_CASE       :      SpeechaTextTokenizer							=    SpeechaTextTokenizer.from_pretrained(cls.checkpoint_name      )
       return cls


     def 				_A       (				self       :	str      ):
       self.assertEqual(self.tokenizer.lang_code_to_id["pt"]					, 4      )
       self.assertEqual(self.tokenizer.lang_code_to_id["ru"]					, 6      )
       self.assertEqual(self.tokenizer.lang_code_to_id["it"]					, 9      )
       self.assertEqual(self.tokenizer.lang_code_to_id["de"]					, 11      )


     def 				_A       (				self       :	List[Any]      ):
       self.assertEqual(self.tokenizer.vocab_size					, 1_0000      )


     def 				_A       (				self       :	Tuple      ):
       self.assertIn(UpperCAmelCase_					, self.tokenizer.all_special_ids      )
       SCREAMING_SNAKE_CASE       :      Dict							=    [ES_CODE, 4, 1601, 47, 7647, 2]
       SCREAMING_SNAKE_CASE       :      List[Any]							=    self.tokenizer.decode(UpperCAmelCase_					, skip_special_tokens=UpperCAmelCase_      )
       SCREAMING_SNAKE_CASE       :      Tuple							=    self.tokenizer.decode(generated_ids[1:]					, skip_special_tokens=UpperCAmelCase_      )
       self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_      )
       self.assertNotIn(self.tokenizer.eos_token					, UpperCAmelCase_      )


     def 				_A       (				self       :	Optional[int]      ):
       SCREAMING_SNAKE_CASE       :      Optional[int]							=    "fr"
       SCREAMING_SNAKE_CASE       :      Any							=    self.tokenizer(self.french_text      ).input_ids
       self.assertEqual(encoded[0]					, UpperCAmelCase_      )
       self.assertEqual(encoded[-1]					, self.tokenizer.eos_token_id      )






     def 				_A       (				self       :	Union[str, Any]      ):
       SCREAMING_SNAKE_CASE       :      int							=    "fr"
       self.assertListEqual(self.tokenizer.prefix_tokens					, [FR_CODE]      )

       SCREAMING_SNAKE_CASE       :      Tuple							=    "es"
       self.assertListEqual(self.tokenizer.prefix_tokens					, [ES_CODE]      )














import gc
import unittest

import numpy as np
import torch

from diffusers import StableDiffusionKDiffusionPipeline
from diffusers.utils import slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu


enable_full_determinism()


@slow
@require_torch_gpu
class  SCREAMING_SNAKE_CASE			(			unittest.TestCase					):
     '''simple docstring'''


     def 				_A       (				self       :	List[Any]      ):
       # clean up the VRAM after each test
       super().tearDown()
       gc.collect()
       torch.cuda.empty_cache()


     def 				_A       (				self       :	str      ):
       SCREAMING_SNAKE_CASE       :      List[str]							=    StableDiffusionKDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4"      )
       SCREAMING_SNAKE_CASE       :      int							=    sd_pipe.to(UpperCAmelCase_      )
       sd_pipe.set_progress_bar_config(disable=UpperCAmelCase_      )

       sd_pipe.set_scheduler("sample_euler"      )

       SCREAMING_SNAKE_CASE       :      str							=    "A painting of a squirrel eating a burger"
       SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    torch.manual_seed(0      )
       SCREAMING_SNAKE_CASE       :      Any							=    sd_pipe([prompt]					, generator=UpperCAmelCase_					, guidance_scale=9.0					, num_inference_steps=20					, output_type="np"      )

       SCREAMING_SNAKE_CASE       :      List[str]							=    output.images

       SCREAMING_SNAKE_CASE       :      str							=    image[0, -3:, -3:, -1]

       assert image.shape == (1, 512, 512, 3)
       SCREAMING_SNAKE_CASE       :      int							=    np.array([0.0_447, 0.0_492, 0.0_468, 0.0_408, 0.0_383, 0.0_408, 0.0_354, 0.0_380, 0.0_339]      )

       assert np.abs(image_slice.flatten() - expected_slice      ).max() < 1E-2


     def 				_A       (				self       :	List[Any]      ):
       SCREAMING_SNAKE_CASE       :      str							=    StableDiffusionKDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-1-base"      )
       SCREAMING_SNAKE_CASE       :      str							=    sd_pipe.to(UpperCAmelCase_      )
       sd_pipe.set_progress_bar_config(disable=UpperCAmelCase_      )

       sd_pipe.set_scheduler("sample_euler"      )

       SCREAMING_SNAKE_CASE       :      List[str]							=    "A painting of a squirrel eating a burger"
       SCREAMING_SNAKE_CASE       :      Dict							=    torch.manual_seed(0      )
       SCREAMING_SNAKE_CASE       :      Optional[int]							=    sd_pipe([prompt]					, generator=UpperCAmelCase_					, guidance_scale=9.0					, num_inference_steps=20					, output_type="np"      )

       SCREAMING_SNAKE_CASE       :      Optional[int]							=    output.images

       SCREAMING_SNAKE_CASE       :      Dict							=    image[0, -3:, -3:, -1]

       assert image.shape == (1, 512, 512, 3)
       SCREAMING_SNAKE_CASE       :      int							=    np.array([0.1_237, 0.1_320, 0.1_438, 0.1_359, 0.1_390, 0.1_132, 0.1_277, 0.1_175, 0.1_112]      )

       assert np.abs(image_slice.flatten() - expected_slice      ).max() < 5E-1






     def 				_A       (				self       :	Optional[int]      ):
       SCREAMING_SNAKE_CASE       :      List[str]							=    StableDiffusionKDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-1-base"      )
       SCREAMING_SNAKE_CASE       :      Any							=    sd_pipe.to(UpperCAmelCase_      )
       sd_pipe.set_progress_bar_config(disable=UpperCAmelCase_      )

       sd_pipe.set_scheduler("sample_dpmpp_2m"      )

       SCREAMING_SNAKE_CASE       :      List[str]							=    "A painting of a squirrel eating a burger"
       SCREAMING_SNAKE_CASE       :      Any							=    torch.manual_seed(0      )
       SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    sd_pipe(
           [prompt]					, generator=UpperCAmelCase_					, guidance_scale=7.5					, num_inference_steps=15					, output_type="np"					, use_karras_sigmas=UpperCAmelCase_					, )

       SCREAMING_SNAKE_CASE       :      Optional[Any]							=    output.images

       SCREAMING_SNAKE_CASE       :      List[Any]							=    image[0, -3:, -3:, -1]

       assert image.shape == (1, 512, 512, 3)
       SCREAMING_SNAKE_CASE       :      Union[str, Any]							=    np.array(
           [0.11_381_689, 0.12_112_921, 0.1_389_457, 0.12_549_606, 0.1_244_964, 0.10_831_517, 0.11_562_866, 0.10_867_816, 0.10_499_048]      )

       assert np.abs(image_slice.flatten() - expected_slice      ).max() < 1E-2






