
'''simple docstring'''

import json
import os
import shutil
import tempfile
import unittest

from transformers import BatchEncoding, CanineTokenizer
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.tokenization_utils import AddedToken
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


class  _UpperCAmelCase ( _UpperCamelCase	,unittest.TestCase    ):

     """simple docstring"""

     a_    	=					CanineTokenizer
     a_    	=					False


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       super().setUp()
       a_      :	Optional[int]      =							CanineTokenizer()
       tokenizer.save_pretrained(self.tmpdirname )


     @cached_property
     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       return CanineTokenizer.from_pretrained("""google/canine-s""" )


     def 							_lowerCAmelCase  (      self							,		**lowerCAmelCase_ ):

       '''simple docstring'''

       a_      :	Dict      =							self.tokenizer_class.from_pretrained(self.tmpdirname							,		**_UpperCAmelCase )
       a_      :	Union[str, Any]      =							10_24
       return tokenizer


     @require_torch
     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	Optional[Any]      =							self.canine_tokenizer
       a_      :	int      =							['''Life is like a box of chocolates.''', '''You never know what you\'re gonna get.''']
       # fmt: off
       a_      :	Dict      =							[5_73_44, 76, 1_05, 1_02, 1_01, 32, 1_05, 1_15, 32, 1_08, 1_05, 1_07, 1_01, 32, 97, 32, 98, 1_11, 1_20, 32, 1_11, 1_02, 32, 99, 1_04, 1_11, 99, 1_11, 1_08, 97, 1_16, 1_01, 1_15, 46, 5_73_45, 0, 0, 0, 0]
       # fmt: on
       a_      :	Optional[int]      =							tokenizer(_UpperCAmelCase							,		padding=_UpperCAmelCase							,		return_tensors="""pt""" )
       self.assertIsInstance(_UpperCAmelCase							,		_UpperCAmelCase )

       a_      :	str      =							list(batch.input_ids.numpy()[0] )

       self.assertListEqual(_UpperCAmelCase							,		_UpperCAmelCase )

       self.assertEqual((2, 39)							,		batch.input_ids.shape )
       self.assertEqual((2, 39)							,		batch.attention_mask.shape )


     @require_torch
     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	int      =							self.canine_tokenizer
       a_      :	Dict      =							['''Once there was a man.''', '''He wrote a test in HuggingFace Tranformers.''']
       a_      :	List[str]      =							tokenizer(_UpperCAmelCase							,		padding=_UpperCAmelCase							,		return_tensors="""pt""" )
       # check if input_ids, attention_mask and token_type_ids are returned
       self.assertIn("""input_ids"""							,		_UpperCAmelCase )
       self.assertIn("""attention_mask"""							,		_UpperCAmelCase )
       self.assertIn("""token_type_ids"""							,		_UpperCAmelCase )


     @require_torch
     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	Optional[Any]      =							self.canine_tokenizer
       a_      :	Optional[Any]      =							[
           '''What\'s the weater?''',
           '''It\'s about 25 degrees.''',
       ]
       a_      :	Optional[int]      =							tokenizer(
           text_target=_UpperCAmelCase							,		max_length=32							,		padding="""max_length"""							,		truncation=_UpperCAmelCase							,		return_tensors="""pt""" )
       self.assertEqual(32							,		targets["""input_ids"""].shape[1] )


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	List[Any]      =							self.get_tokenizers()
       for tokenizer in tokenizers:
         with self.subTest(f'''{tokenizer.__class__.__name__}''' ):
           self.assertNotEqual(tokenizer.model_max_length							,		42 )

        # Now let's start the test
       a_      :	Dict      =							self.get_tokenizers()
       for tokenizer in tokenizers:
         with self.subTest(f'''{tokenizer.__class__.__name__}''' ):
           # Isolate this from the other tests because we save additional tokens/etc
           a_      :	Any      =							tempfile.mkdtemp()

           a_      :	Union[str, Any]      =							''' He is very happy, UNwant\u00E9d,running'''
           a_      :	Optional[int]      =							tokenizer.encode(_UpperCAmelCase							,		add_special_tokens=_UpperCAmelCase )
           tokenizer.save_pretrained(_UpperCAmelCase )

           a_      :	int      =							tokenizer.__class__.from_pretrained(_UpperCAmelCase )
           a_      :	Any      =							after_tokenizer.encode(_UpperCAmelCase							,		add_special_tokens=_UpperCAmelCase )
           self.assertListEqual(_UpperCAmelCase							,		_UpperCAmelCase )

           shutil.rmtree(_UpperCAmelCase )

       a_      :	Dict      =							self.get_tokenizers(model_max_length=42 )
       for tokenizer in tokenizers:
         with self.subTest(f'''{tokenizer.__class__.__name__}''' ):
           # Isolate this from the other tests because we save additional tokens/etc
           a_      :	List[str]      =							tempfile.mkdtemp()

           a_      :	str      =							''' He is very happy, UNwant\u00E9d,running'''

           a_      :	Tuple      =							tokenizer.additional_special_tokens

           # We can add a new special token for Canine as follows:
           a_      :	Optional[int]      =							chr(0xE007 )
           additional_special_tokens.append(_UpperCAmelCase )
           tokenizer.add_special_tokens({"""additional_special_tokens""": additional_special_tokens} )
           a_      :	List[str]      =							tokenizer.encode(_UpperCAmelCase							,		add_special_tokens=_UpperCAmelCase )
           tokenizer.save_pretrained(_UpperCAmelCase )

           a_      :	Optional[int]      =							tokenizer.__class__.from_pretrained(_UpperCAmelCase )
           a_      :	List[Any]      =							after_tokenizer.encode(_UpperCAmelCase							,		add_special_tokens=_UpperCAmelCase )
           self.assertListEqual(_UpperCAmelCase							,		_UpperCAmelCase )
           self.assertIn(_UpperCAmelCase							,		after_tokenizer.additional_special_tokens )
           self.assertEqual(after_tokenizer.model_max_length							,		42 )

           a_      :	List[Any]      =							tokenizer.__class__.from_pretrained(_UpperCAmelCase							,		model_max_length=43 )
           self.assertEqual(tokenizer.model_max_length							,		43 )

           shutil.rmtree(_UpperCAmelCase )


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	List[str]      =							self.get_tokenizers(do_lower_case=_UpperCAmelCase )
       for tokenizer in tokenizers:
         with self.subTest(f'''{tokenizer.__class__.__name__}''' ):
           a_      :	Union[str, Any]      =							self.get_clean_sequence(_UpperCAmelCase )

           # a special token for Canine can be defined as follows:
           a_      :	Union[str, Any]      =							0xE005
           a_      :	Any      =							chr(_UpperCAmelCase )

           tokenizer.add_special_tokens({"""cls_token""": special_token} )
           a_      :	List[str]      =							tokenizer.encode(_UpperCAmelCase							,		add_special_tokens=_UpperCAmelCase )
           self.assertEqual(len(_UpperCAmelCase )							,		1 )

           a_      :	int      =							tokenizer.decode(ids + encoded_special_token							,		clean_up_tokenization_spaces=_UpperCAmelCase )
           a_      :	Union[str, Any]      =							tokenizer.encode(_UpperCAmelCase							,		add_special_tokens=_UpperCAmelCase )

           a_      :	str      =							tokenizer.encode(_UpperCAmelCase							,		add_special_tokens=_UpperCAmelCase )
           a_      :	Optional[Any]      =							tokenizer.encode(_UpperCAmelCase							,		add_special_tokens=_UpperCAmelCase )
           self.assertEqual(_UpperCAmelCase							,		input_encoded + special_token_id )

           a_      :	Dict      =							tokenizer.decode(_UpperCAmelCase							,		skip_special_tokens=_UpperCAmelCase )
           self.assertTrue(special_token not in decoded )


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	Optional[Any]      =							self.get_tokenizers(do_lower_case=_UpperCAmelCase )
       for tokenizer in tokenizers:
         with self.subTest(f'''{tokenizer.__class__.__name__}''' ):
           a_      :	Optional[Any]      =							chr(0xE005 )
           a_      :	Tuple      =							chr(0xE006 )

           # `add_tokens` method stores special tokens only in `tokenizer.unique_no_split_tokens`. (in tokenization_utils.py)
           tokenizer.add_tokens([SPECIAL_TOKEN_1]							,		special_tokens=_UpperCAmelCase )
           # `add_special_tokens` method stores special tokens in `tokenizer.additional_special_tokens`,
           # which also occur in `tokenizer.all_special_tokens`. (in tokenization_utils_base.py)
           tokenizer.add_special_tokens({"""additional_special_tokens""": [SPECIAL_TOKEN_2]} )

           a_      :	Dict      =							tokenizer.tokenize(_UpperCAmelCase )
           a_      :	Any      =							tokenizer.tokenize(_UpperCAmelCase )

           self.assertEqual(len(_UpperCAmelCase )							,		1 )
           self.assertEqual(len(_UpperCAmelCase )							,		1 )
           self.assertEqual(token_a[0]							,		_UpperCAmelCase )
           self.assertEqual(token_a[0]							,		_UpperCAmelCase )


     @require_tokenizers
     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	Tuple      =							self.get_tokenizers(do_lower_case=_UpperCAmelCase )
       for tokenizer in tokenizers:
         with self.subTest(f'''{tokenizer.__class__.__name__}''' ):
           # a special token for Canine can be defined as follows:
           a_      :	Optional[Any]      =							0xE006
           a_      :	List[str]      =							chr(_UpperCAmelCase )

           a_      :	Union[str, Any]      =							AddedToken(_UpperCAmelCase							,		lstrip=_UpperCAmelCase )
           tokenizer.add_special_tokens({"""additional_special_tokens""": [new_token]} )

           with tempfile.TemporaryDirectory() as tmp_dir_name:
             tokenizer.save_pretrained(_UpperCAmelCase )
             tokenizer.from_pretrained(_UpperCAmelCase )


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	Dict      =							[]
       if self.test_slow_tokenizer:
         tokenizer_list.append((self.tokenizer_class, self.get_tokenizer()) )

       if self.test_rust_tokenizer:
         tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer()) )

       for tokenizer_class, tokenizer_utils in tokenizer_list:
         with tempfile.TemporaryDirectory() as tmp_dir:
           tokenizer_utils.save_pretrained(_UpperCAmelCase )

           with open(os.path.join(_UpperCAmelCase							,		"""special_tokens_map.json""" )							,		encoding="""utf-8""" ) as json_file:
             a_      :	Any      =							json.load(_UpperCAmelCase )

           with open(os.path.join(_UpperCAmelCase							,		"""tokenizer_config.json""" )							,		encoding="""utf-8""" ) as json_file:
             a_      :	Tuple      =							json.load(_UpperCAmelCase )

           # a special token for Canine can be defined as follows:
           a_      :	str      =							0xE006
           a_      :	int      =							chr(_UpperCAmelCase )

           a_      :	Optional[int]      =							[new_token_a]
           a_      :	Tuple      =							[new_token_a]

           with open(os.path.join(_UpperCAmelCase							,		"""special_tokens_map.json""" )							,		"""w"""							,		encoding="""utf-8""" ) as outfile:
             json.dump(_UpperCAmelCase							,		_UpperCAmelCase )
           with open(os.path.join(_UpperCAmelCase							,		"""tokenizer_config.json""" )							,		"""w"""							,		encoding="""utf-8""" ) as outfile:
             json.dump(_UpperCAmelCase							,		_UpperCAmelCase )

           # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
           # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
           # "special_tokens_map.json" files
           a_      :	Union[str, Any]      =							tokenizer_class.from_pretrained(_UpperCAmelCase							,		extra_ids=0 )
           self.assertIn(_UpperCAmelCase							,		tokenizer_without_change_in_init.additional_special_tokens )
           # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
           self.assertEqual(
               [new_token_a]							,		tokenizer_without_change_in_init.convert_ids_to_tokens(
                   tokenizer_without_change_in_init.convert_tokens_to_ids([new_token_a] ) )							,		)

           a_      :	Any      =							0xE007
           a_      :	Optional[int]      =							chr(_UpperCAmelCase )
           # Now we test that we can change the value of additional_special_tokens in the from_pretrained
           a_      :	Dict      =							[AddedToken(_UpperCAmelCase							,		lstrip=_UpperCAmelCase )]
           a_      :	List[Any]      =							tokenizer_class.from_pretrained(
               _UpperCAmelCase							,		additional_special_tokens=_UpperCAmelCase							,		extra_ids=0 )

           self.assertIn(_UpperCAmelCase							,		tokenizer.additional_special_tokens )
           # self.assertIn(new_token_2,tokenizer.get_vocab()) # ByT5Tokenization no vocab
           self.assertEqual(
               [new_token_a]							,		tokenizer.convert_ids_to_tokens(tokenizer.convert_tokens_to_ids([new_token_a] ) ) )


     @require_tokenizers
     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	str      =							self.get_tokenizers(do_lower_case=_UpperCAmelCase )
       for tokenizer in tokenizers:
         with self.subTest(f'''{tokenizer.__class__.__name__}''' ):
           a_      :	List[str]      =							'''hello world'''
           if self.space_between_special_tokens:
             a_      :	Any      =							'''[CLS] hello world [SEP]'''
           else:
             a_      :	Optional[int]      =							input
           a_      :	List[Any]      =							tokenizer.encode(_UpperCAmelCase							,		add_special_tokens=_UpperCAmelCase )
           a_      :	Optional[int]      =							tokenizer.decode(_UpperCAmelCase							,		spaces_between_special_tokens=self.space_between_special_tokens )
           self.assertIn(_UpperCAmelCase							,		[output, output.lower()] )


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       a_      :	Optional[Any]      =							self.get_tokenizers()
       for tokenizer in tokenizers:
         with self.subTest(f'''{tokenizer.__class__.__name__}''' ):
           a_      :	Optional[int]      =							[
               '''bos_token''',
               '''eos_token''',
               '''unk_token''',
               '''sep_token''',
               '''pad_token''',
               '''cls_token''',
               '''mask_token''',
           ]

       a_      :	List[str]      =							'''a'''
       a_      :	Optional[Any]      =							ord(_UpperCAmelCase )

       for attr in attributes_list:
         setattr(_UpperCAmelCase							,		attr + """_id"""							,		_UpperCAmelCase )
         self.assertEqual(getattr(_UpperCAmelCase							,		_UpperCAmelCase )							,		_UpperCAmelCase )
         self.assertEqual(getattr(_UpperCAmelCase							,		attr + """_id""" )							,		_UpperCAmelCase )

         setattr(_UpperCAmelCase							,		attr + """_id"""							,		_UpperCAmelCase )
         self.assertEqual(getattr(_UpperCAmelCase							,		_UpperCAmelCase )							,		_UpperCAmelCase )
         self.assertEqual(getattr(_UpperCAmelCase							,		attr + """_id""" )							,		_UpperCAmelCase )

       setattr(_UpperCAmelCase							,		"""additional_special_tokens_ids"""							,		[] )
       self.assertListEqual(getattr(_UpperCAmelCase							,		"""additional_special_tokens""" )							,		[] )
       self.assertListEqual(getattr(_UpperCAmelCase							,		"""additional_special_tokens_ids""" )							,		[] )

       a_      :	Optional[Any]      =							0xE006
       a_      :	List[Any]      =							chr(_UpperCAmelCase )
       setattr(_UpperCAmelCase							,		"""additional_special_tokens_ids"""							,		[additional_special_token_id] )
       self.assertListEqual(getattr(_UpperCAmelCase							,		"""additional_special_tokens""" )							,		[additional_special_token] )
       self.assertListEqual(getattr(_UpperCAmelCase							,		"""additional_special_tokens_ids""" )							,		[additional_special_token_id] )


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       pass


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       pass


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       pass


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       pass


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       pass


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       pass


     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       pass






     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       pass




'''simple docstring'''

import copy
from dataclasses import dataclass, field
from typing import ClassVar, Dict

from ..features import Audio, Features, Value
from .base import TaskTemplate


@dataclass(frozen=lowerCAmelCase__    )
class  _UpperCAmelCase ( lowerCAmelCase__    ):

     """simple docstring"""

     a_    	=					field(default="automatic-speech-recognition"	,metadata={"include_in_asdict_even_if_is_default": True}    )
     a_    	=					Features({"audio": Audio()}    )
     a_    	=					Features({"transcription": Value("string"    )}    )
     a_    	=					"audio"
     a_    	=					"transcription"


     def 							_lowerCAmelCase  (      self							,		lowerCAmelCase_ ):

       '''simple docstring'''

       if self.audio_column not in features:
         raise ValueError(f'''Column {self.audio_column} is not present in features.''' )
       if not isinstance(features[self.audio_column]							,		lowerCAmelCase_ ):
         raise ValueError(f'''Column {self.audio_column} is not an Audio type.''' )
       a_      :	int      =							copy.deepcopy(self )
       a_      :	Optional[int]      =							self.input_schema.copy()
       a_      :	Any      =							features[self.audio_column]
       a_      :	Any      =							input_schema
       return task_template






     @property
     def 							_lowerCAmelCase  (      self ):

       '''simple docstring'''

       return {self.audio_column: "audio", self.transcription_column: "transcription"}


