





'''simple docstring'''







import json
import os
import unittest

from transformers.models.blenderbot_small.tokenization_blenderbot_small import (
    VOCAB_FILES_NAMES,
    BlenderbotSmallTokenizer,
)

from ...test_tokenization_common import TokenizerTesterMixin
class 			_lowercase						(       __a				,		unittest.TestCase					):
     _UpperCAmelCase										=					BlenderbotSmallTokenizer
     _UpperCAmelCase										=					False







     def     UpperCamelCase			( self    ) ->	int:
      super().setUp()

      snake_case				=			['''__start__''', '''adapt''', '''act''', '''ap@@''', '''te''', '''__end__''', '''__unk__''']
      snake_case				=			dict(zip(A__    ,   range(len(A__    )    )    )    )

      snake_case				=			['''#version: 0.2''', '''a p''', '''t e</w>''', '''ap t</w>''', '''a d''', '''ad apt</w>''', '''a c''', '''ac t</w>''', '''''']
      snake_case				=			{'''unk_token''': '''__unk__''', '''bos_token''': '''__start__''', '''eos_token''': '''__end__'''}

      snake_case				=			os.path.join(self.tmpdirname    ,   VOCAB_FILES_NAMES['''vocab_file''']    )
      snake_case				=			os.path.join(self.tmpdirname    ,   VOCAB_FILES_NAMES['''merges_file''']    )
      with open(self.vocab_file    ,   '''w'''    ,   encoding='''utf-8'''    ) as fp:
       fp.write(json.dumps(A__    ) + '''\n'''    )
      with open(self.merges_file    ,   '''w'''    ,   encoding='''utf-8'''    ) as fp:
       fp.write('''\n'''.join(A__    )    )







     def     UpperCamelCase			( self    ,   **A__    ) ->	Tuple:
      kwargs.update(self.special_tokens_map    )
      return BlenderbotSmallTokenizer.from_pretrained(self.tmpdirname    ,   **A__    )







     def     UpperCamelCase			( self    ,   A__    ) ->	Dict:
      snake_case				=			'''adapt act apte'''
      snake_case				=			'''adapt act apte'''
      return input_text, output_text







     def     UpperCamelCase			( self    ) ->	Dict:
      snake_case				=			BlenderbotSmallTokenizer(self.vocab_file    ,   self.merges_file    ,   **self.special_tokens_map    )
      snake_case				=			'''adapt act apte'''
      snake_case				=			['''adapt''', '''act''', '''ap@@''', '''te''']
      snake_case				=			tokenizer.tokenize(A__    )
      self.assertListEqual(A__    ,   A__    )

      snake_case				=			[tokenizer.bos_token] + tokens + [tokenizer.eos_token]

      snake_case				=			[0, 1, 2, 3, 4, 5]
      self.assertListEqual(tokenizer.convert_tokens_to_ids(A__    )    ,   A__    )







     def     UpperCamelCase			( self    ) ->	Optional[Any]:
      snake_case				=			BlenderbotSmallTokenizer.from_pretrained('''facebook/blenderbot-90M'''    )
      assert tok('''sam'''    ).input_ids == [13_84]
      snake_case				=			'''I am a small frog.'''
      snake_case				=			tok([src_text]    ,   padding=A__    ,   truncation=A__    )['''input_ids''']
      snake_case				=			tok.batch_decode(A__    ,   skip_special_tokens=A__    ,   clean_up_tokenization_spaces=A__    )[0]
      assert src_text != decoded  # I wish it did!
      assert decoded == "i am a small frog ."







     def     UpperCamelCase			( self    ) ->	str:
      snake_case				=			BlenderbotSmallTokenizer.from_pretrained('''facebook/blenderbot-90M'''    )
      snake_case				=			'''I am a small frog .'''
      snake_case				=			'''.'''
      snake_case				=			tok(A__    )['''input_ids''']
      snake_case				=			tok(A__    )['''input_ids''']

      assert encoded[-1] == encoded_dot[0]








'''simple docstring'''







from ...processing_utils import ProcessorMixin
class 			_lowercase						(       __a					):
     _UpperCAmelCase										=					'''WhisperFeatureExtractor'''
     _UpperCAmelCase										=					'''WhisperTokenizer'''







     def __init__( self    ,   A__    ,   A__    ) ->	Optional[Any]:
      super().__init__(A__    ,   A__    )
      snake_case				=			self.feature_extractor
      snake_case				=			False







     def     UpperCamelCase			( self    ,   A__=None    ,   A__=None    ,   A__=True    ) ->	Union[str, Any]:
      return self.tokenizer.get_decoder_prompt_ids(task=A__    ,   language=A__    ,   no_timestamps=A__    )







     def __call__( self    ,   *A__    ,   **A__    ) ->	Dict:
      # For backward compatibility
      if self._in_target_context_manager:
       return self.current_processor(*A__    ,   **A__    )

      snake_case				=			kwargs.pop('''audio'''    ,   A__    )
      snake_case				=			kwargs.pop('''sampling_rate'''    ,   A__    )
      snake_case				=			kwargs.pop('''text'''    ,   A__    )
      if len(A__    ) > 0:
       snake_case				=			args[0]
       snake_case				=			args[1:]

      if audio is None and text is None:
       raise ValueError('''You need to specify either an `audio` or `text` input to process.'''    )

      if audio is not None:
       snake_case				=			self.feature_extractor(A__    ,   *A__    ,   sampling_rate=A__    ,   **A__    )
      if text is not None:
       snake_case				=			self.tokenizer(A__    ,   **A__    )

      if text is None:
       return inputs

      elif audio is None:
       return encodings
      else:
       snake_case				=			encodings['''input_ids''']
       return inputs







     def     UpperCamelCase			( self    ,   *A__    ,   **A__    ) ->	Optional[Any]:
      return self.tokenizer.batch_decode(*A__    ,   **A__    )







     def     UpperCamelCase			( self    ,   *A__    ,   **A__    ) ->	str:
      return self.tokenizer.decode(*A__    ,   **A__    )







     def     UpperCamelCase			( self    ,   A__    ,   A__="np"    ) ->	Optional[Any]:
      return self.tokenizer.get_prompt_ids(A__    ,   return_tensors=A__    )

