


import json
import os
import shutil
import tempfile
import unittest
from multiprocessing import get_context
from pathlib import Path

import datasets
import numpy as np
from datasets import load_dataset
from parameterized import parameterized

from transformers import AutoProcessor
from transformers.models.wavaveca import WavaVecaCTCTokenizer, WavaVecaFeatureExtractor
from transformers.models.wavaveca.tokenization_wavaveca import VOCAB_FILES_NAMES
from transformers.testing_utils import require_pyctcdecode, require_torch, require_torchaudio, slow
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_pyctcdecode_available, is_torch_available

from ..wavaveca.test_feature_extraction_wavaveca import floats_list


if is_pyctcdecode_available():
				from huggingface_hub import snapshot_download
				from pyctcdecode import BeamSearchDecoderCTC

				from transformers.models.wavaveca_with_lm import WavaVecaProcessorWithLM
				from transformers.models.wavaveca_with_lm.processing_wavaveca_with_lm import WavaVecaDecoderWithLMOutput

if is_torch_available():
				from transformers import WavaVecaForCTC







@require_pyctcdecode
class     _lowercase				(							unittest.TestCase):


							"""simple docstring"""




							def 		lowerCAmelCase  (     self							:    List[Any] ):


													'''simple docstring'''



													lowerCamelCase__	:       List[str]					=						"| <pad> <unk> <s> </s> a b c d e f g h i j k".split()
													lowerCamelCase__	:       List[Any]					=						dict(zip(__lowerCamelCase      ,   range(len(__lowerCamelCase ) ) ) )

													lowerCamelCase__	:       Optional[int]					=						{
													    "unk_token": "<unk>",
													    "bos_token": "<s>",
													    "eos_token": "</s>",
													}
													lowerCamelCase__	:       List[Any]					=						{
													    "feature_size": 1,
													    "padding_value": 0.0,
													    "sampling_rate": 16000,
													    "return_attention_mask": False,
													    "do_normalize": True,
													}

													lowerCamelCase__	:       Tuple					=						tempfile.mkdtemp()
													lowerCamelCase__	:       Union[str, Any]					=						os.path.join(self.tmpdirname      ,   VOCAB_FILES_NAMES["vocab_file"] )
													lowerCamelCase__	:       List[str]					=						os.path.join(self.tmpdirname      ,   __lowerCamelCase )
													with open(self.vocab_file      ,   "w"      ,   encoding="utf-8" ) as fp:
																			fp.write(json.dumps(__lowerCamelCase ) + "\n" )

													with open(self.feature_extraction_file      ,   "w"      ,   encoding="utf-8" ) as fp:
																			fp.write(json.dumps(__lowerCamelCase ) + "\n" )

													# load decoder from hub
													lowerCamelCase__	:       Dict					=						"hf-internal-testing/ngram-beam-search-decoder"





							def 		lowerCAmelCase  (     self							:    Union[str, Any]      ,   **__lowerCamelCase							:    List[Any] ):


													'''simple docstring'''



													lowerCamelCase__	:       int					=						self.add_kwargs_tokens_map.copy()
													kwargs.update(__lowerCamelCase )
													return WavaVecaCTCTokenizer.from_pretrained(self.tmpdirname      ,   **__lowerCamelCase )





							def 		lowerCAmelCase  (     self							:    int      ,   **__lowerCamelCase							:    str ):


													'''simple docstring'''



													return WavaVecaFeatureExtractor.from_pretrained(self.tmpdirname      ,   **__lowerCamelCase )





							def 		lowerCAmelCase  (     self							:    int      ,   **__lowerCamelCase							:    Dict ):


													'''simple docstring'''



													return BeamSearchDecoderCTC.load_from_hf_hub(self.decoder_name      ,   **__lowerCamelCase )





							def 		lowerCAmelCase  (     self							:    Union[str, Any] ):


													'''simple docstring'''



													shutil.rmtree(self.tmpdirname )





							def 		lowerCAmelCase  (     self							:    Any ):


													'''simple docstring'''



													lowerCamelCase__	:       Tuple					=						self.get_tokenizer()
													lowerCamelCase__	:       str					=						self.get_feature_extractor()
													lowerCamelCase__	:       Any					=						self.get_decoder()

													lowerCamelCase__	:       Any					=						WavaVecaProcessorWithLM(tokenizer=__lowerCamelCase      ,   feature_extractor=__lowerCamelCase      ,   decoder=__lowerCamelCase )

													processor.save_pretrained(self.tmpdirname )
													lowerCamelCase__	:       str					=						WavaVecaProcessorWithLM.from_pretrained(self.tmpdirname )

													# tokenizer
													self.assertEqual(processor.tokenizer.get_vocab()      ,   tokenizer.get_vocab() )
													self.assertIsInstance(processor.tokenizer      ,   __lowerCamelCase )

													# feature extractor
													self.assertEqual(processor.feature_extractor.to_json_string()      ,   feature_extractor.to_json_string() )
													self.assertIsInstance(processor.feature_extractor      ,   __lowerCamelCase )

													# decoder
													self.assertEqual(processor.decoder._alphabet.labels      ,   decoder._alphabet.labels )
													self.assertEqual(
													    processor.decoder.model_container[decoder._model_key]._unigram_set      ,   decoder.model_container[decoder._model_key]._unigram_set      ,   )
													self.assertIsInstance(processor.decoder      ,   __lowerCamelCase )





							def 		lowerCAmelCase  (     self							:    Optional[int] ):


													'''simple docstring'''



													lowerCamelCase__	:       Optional[int]					=						WavaVecaProcessorWithLM(
													    tokenizer=self.get_tokenizer()      ,   feature_extractor=self.get_feature_extractor()      ,   decoder=self.get_decoder() )
													processor.save_pretrained(self.tmpdirname )

													# make sure that error is thrown when decoder alphabet doesn't match
													lowerCamelCase__	:       str					=						WavaVecaProcessorWithLM.from_pretrained(
													    self.tmpdirname      ,   alpha=5.0      ,   beta=3.0      ,   score_boundary=-7.0      ,   unk_score_offset=3 )

													# decoder
													self.assertEqual(processor.language_model.alpha      ,   5.0 )
													self.assertEqual(processor.language_model.beta      ,   3.0 )
													self.assertEqual(processor.language_model.score_boundary      ,   -7.0 )
													self.assertEqual(processor.language_model.unk_score_offset      ,   3 )





							def 		lowerCAmelCase  (     self							:    str ):


													'''simple docstring'''



													lowerCamelCase__	:       List[str]					=						self.get_tokenizer()
													# add token to trigger raise
													tokenizer.add_tokens(["xx"] )
													with self.assertRaisesRegex(__lowerCamelCase      ,   "include" ):
																			WavaVecaProcessorWithLM(
																			    tokenizer=__lowerCamelCase      ,   feature_extractor=self.get_feature_extractor()      ,   decoder=self.get_decoder() )





							def 		lowerCAmelCase  (     self							:    str ):


													'''simple docstring'''



													lowerCamelCase__	:       Optional[Any]					=						self.get_feature_extractor()
													lowerCamelCase__	:       Dict					=						self.get_tokenizer()
													lowerCamelCase__	:       Union[str, Any]					=						self.get_decoder()

													lowerCamelCase__	:       int					=						WavaVecaProcessorWithLM(tokenizer=__lowerCamelCase      ,   feature_extractor=__lowerCamelCase      ,   decoder=__lowerCamelCase )

													lowerCamelCase__	:       Optional[Any]					=						floats_list((3, 1000) )

													lowerCamelCase__	:       Tuple					=						feature_extractor(__lowerCamelCase      ,   return_tensors="np" )
													lowerCamelCase__	:       List[str]					=						processor(__lowerCamelCase      ,   return_tensors="np" )

													for key in input_feat_extract.keys():
																			self.assertAlmostEqual(input_feat_extract[key].sum()      ,   input_processor[key].sum()      ,   delta=1E-2 )





							def 		lowerCAmelCase  (     self							:    int ):


													'''simple docstring'''



													lowerCamelCase__	:       Union[str, Any]					=						self.get_feature_extractor()
													lowerCamelCase__	:       Tuple					=						self.get_tokenizer()
													lowerCamelCase__	:       Union[str, Any]					=						self.get_decoder()

													lowerCamelCase__	:       Any					=						WavaVecaProcessorWithLM(tokenizer=__lowerCamelCase      ,   feature_extractor=__lowerCamelCase      ,   decoder=__lowerCamelCase )

													lowerCamelCase__	:       Any					=						"This is a test string"

													lowerCamelCase__	:       Union[str, Any]					=						processor(text=__lowerCamelCase )

													lowerCamelCase__	:       List[str]					=						tokenizer(__lowerCamelCase )

													for key in encoded_tok.keys():
																			self.assertListEqual(encoded_tok[key]      ,   encoded_processor[key] )





							def 		lowerCAmelCase  (     self							:    List[Any]      ,   __lowerCamelCase							:    Optional[Any]=(2, 10, 16)      ,   __lowerCamelCase							:    Any=77 ):


													'''simple docstring'''



													np.random.seed(__lowerCamelCase )
													return np.random.rand(*__lowerCamelCase )





							def 		lowerCAmelCase  (     self							:    Optional[Any] ):


													'''simple docstring'''



													lowerCamelCase__	:       Tuple					=						self.get_feature_extractor()
													lowerCamelCase__	:       Any					=						self.get_tokenizer()
													lowerCamelCase__	:       List[str]					=						self.get_decoder()

													lowerCamelCase__	:       int					=						WavaVecaProcessorWithLM(tokenizer=__lowerCamelCase      ,   feature_extractor=__lowerCamelCase      ,   decoder=__lowerCamelCase )

													lowerCamelCase__	:       Union[str, Any]					=						self._get_dummy_logits(shape=(10, 16)      ,   seed=13 )

													lowerCamelCase__	:       Optional[Any]					=						processor.decode(__lowerCamelCase )

													lowerCamelCase__	:       Tuple					=						decoder.decode_beams(__lowerCamelCase )[0]

													self.assertEqual(decoded_decoder[0]      ,   decoded_processor.text )
													self.assertEqual("</s> <s> </s>"      ,   decoded_processor.text )
													self.assertEqual(decoded_decoder[-2]      ,   decoded_processor.logit_score )
													self.assertEqual(decoded_decoder[-1]      ,   decoded_processor.lm_score )





							@parameterized.expand([[None], ["fork"], ["spawn"]] )
							def 		lowerCAmelCase  (     self							:    Optional[int]      ,   __lowerCamelCase							:    str ):


													'''simple docstring'''



													lowerCamelCase__	:       Tuple					=						self.get_feature_extractor()
													lowerCamelCase__	:       Any					=						self.get_tokenizer()
													lowerCamelCase__	:       Dict					=						self.get_decoder()

													lowerCamelCase__	:       Union[str, Any]					=						WavaVecaProcessorWithLM(tokenizer=__lowerCamelCase      ,   feature_extractor=__lowerCamelCase      ,   decoder=__lowerCamelCase )

													lowerCamelCase__	:       Tuple					=						self._get_dummy_logits()

													# note: pool should be instantiated *after* Wav2Vec2ProcessorWithLM.
													#       otherwise, the LM won't be available to the pool's sub-processes.
													# manual logic used to allow parameterized test for both pool=None and pool=Pool(...)
													if pool_context is None:
																			lowerCamelCase__	:       Dict					=						processor.batch_decode(__lowerCamelCase )
													else:
																			with get_context(__lowerCamelCase ).Pool() as pool:
																									lowerCamelCase__	:       str					=						processor.batch_decode(__lowerCamelCase      ,   __lowerCamelCase )

													lowerCamelCase__	:       Optional[int]					=						list(__lowerCamelCase )

													with get_context("fork" ).Pool() as p:
																			lowerCamelCase__	:       Any					=						decoder.decode_beams_batch(__lowerCamelCase      ,   __lowerCamelCase )

													lowerCamelCase__					,     lowerCamelCase__					,     lowerCamelCase__	:       Optional[Any]					=						[], [], []
													for beams in decoded_beams:
																			texts_decoder.append(beams[0][0] )
																			logit_scores_decoder.append(beams[0][-2] )
																			lm_scores_decoder.append(beams[0][-1] )

													self.assertListEqual(__lowerCamelCase      ,   decoded_processor.text )
													self.assertListEqual(["<s> <s> </s>", "<s> <s> <s>"]      ,   decoded_processor.text )
													self.assertListEqual(__lowerCamelCase      ,   decoded_processor.logit_score )
													self.assertListEqual(__lowerCamelCase      ,   decoded_processor.lm_score )





							def 		lowerCAmelCase  (     self							:    int ):


													'''simple docstring'''



													lowerCamelCase__	:       Tuple					=						self.get_feature_extractor()
													lowerCamelCase__	:       Any					=						self.get_tokenizer()
													lowerCamelCase__	:       Any					=						self.get_decoder()

													lowerCamelCase__	:       Tuple					=						WavaVecaProcessorWithLM(tokenizer=__lowerCamelCase      ,   feature_extractor=__lowerCamelCase      ,   decoder=__lowerCamelCase )

													lowerCamelCase__	:       Dict					=						self._get_dummy_logits()

													lowerCamelCase__	:       List[Any]					=						15
													lowerCamelCase__	:       List[str]					=						-2_0.0
													lowerCamelCase__	:       Optional[Any]					=						-4.0

													lowerCamelCase__	:       str					=						processor.batch_decode(
													    __lowerCamelCase      ,   beam_width=__lowerCamelCase      ,   beam_prune_logp=__lowerCamelCase      ,   token_min_logp=__lowerCamelCase      ,   )
													lowerCamelCase__	:       Dict					=						decoded_processor_out.text

													lowerCamelCase__	:       Optional[Any]					=						list(__lowerCamelCase )

													with get_context("fork" ).Pool() as pool:
																			lowerCamelCase__	:       List[Any]					=						decoder.decode_beams_batch(
																			    __lowerCamelCase      ,   __lowerCamelCase      ,   beam_width=__lowerCamelCase      ,   beam_prune_logp=__lowerCamelCase      ,   token_min_logp=__lowerCamelCase      ,   )

													lowerCamelCase__	:       str					=						[d[0][0] for d in decoded_decoder_out]
													lowerCamelCase__	:       Tuple					=						[d[0][2] for d in decoded_decoder_out]
													lowerCamelCase__	:       List[str]					=						[d[0][3] for d in decoded_decoder_out]

													self.assertListEqual(__lowerCamelCase      ,   __lowerCamelCase )
													self.assertListEqual(["</s> <s> <s>", "<s> <s> <s>"]      ,   __lowerCamelCase )

													self.assertTrue(np.array_equal(__lowerCamelCase      ,   decoded_processor_out.logit_score ) )
													self.assertTrue(np.allclose([-2_0.0_5_4, -1_8.4_4_7]      ,   __lowerCamelCase      ,   atol=1E-3 ) )

													self.assertTrue(np.array_equal(__lowerCamelCase      ,   decoded_processor_out.lm_score ) )
													self.assertTrue(np.allclose([-1_5.5_5_4, -1_3.9_4_7_4]      ,   __lowerCamelCase      ,   atol=1E-3 ) )





							def 		lowerCAmelCase  (     self							:    Optional[Any] ):


													'''simple docstring'''



													lowerCamelCase__	:       Any					=						self.get_feature_extractor()
													lowerCamelCase__	:       Union[str, Any]					=						self.get_tokenizer()
													lowerCamelCase__	:       Optional[int]					=						self.get_decoder()

													lowerCamelCase__	:       List[Any]					=						WavaVecaProcessorWithLM(tokenizer=__lowerCamelCase      ,   feature_extractor=__lowerCamelCase      ,   decoder=__lowerCamelCase )

													lowerCamelCase__	:       Any					=						self._get_dummy_logits()

													lowerCamelCase__	:       List[str]					=						2.0
													lowerCamelCase__	:       Any					=						5.0
													lowerCamelCase__	:       List[Any]					=						-2_0.0
													lowerCamelCase__	:       Tuple					=						True

													lowerCamelCase__	:       Tuple					=						processor.batch_decode(
													    __lowerCamelCase      ,   alpha=__lowerCamelCase      ,   beta=__lowerCamelCase      ,   unk_score_offset=__lowerCamelCase      ,   lm_score_boundary=__lowerCamelCase      ,   )
													lowerCamelCase__	:       Optional[Any]					=						decoded_processor_out.text

													lowerCamelCase__	:       Dict					=						list(__lowerCamelCase )
													decoder.reset_params(
													    alpha=__lowerCamelCase      ,   beta=__lowerCamelCase      ,   unk_score_offset=__lowerCamelCase      ,   lm_score_boundary=__lowerCamelCase      ,   )

													with get_context("fork" ).Pool() as pool:
																			lowerCamelCase__	:       str					=						decoder.decode_beams_batch(
																			    __lowerCamelCase      ,   __lowerCamelCase      ,   )

													lowerCamelCase__	:       Tuple					=						[d[0][0] for d in decoded_decoder_out]

													self.assertListEqual(__lowerCamelCase      ,   __lowerCamelCase )
													self.assertListEqual(["<s> </s> <s> </s> </s>", "</s> </s> <s> </s> </s>"]      ,   __lowerCamelCase )
													lowerCamelCase__	:       Optional[int]					=						processor.decoder.model_container[processor.decoder._model_key]
													self.assertEqual(lm_model.alpha      ,   2.0 )
													self.assertEqual(lm_model.beta      ,   5.0 )
													self.assertEqual(lm_model.unk_score_offset      ,   -2_0.0 )
													self.assertEqual(lm_model.score_boundary      ,   __lowerCamelCase )





							def 		lowerCAmelCase  (     self							:    str ):


													'''simple docstring'''



													lowerCamelCase__	:       Any					=						WavaVecaProcessorWithLM.from_pretrained("hf-internal-testing/processor_with_lm" )

													lowerCamelCase__	:       List[str]					=						processor.decoder.model_container[processor.decoder._model_key]
													lowerCamelCase__	:       int					=						Path(language_model._kenlm_model.path.decode("utf-8" ) ).parent.parent.absolute()

													lowerCamelCase__	:       int					=						os.listdir(__lowerCamelCase )
													lowerCamelCase__	:       List[Any]					=						["alphabet.json", "language_model"]

													downloaded_decoder_files.sort()
													expected_decoder_files.sort()

													# test that only decoder relevant files from
													# https://huggingface.co/hf-internal-testing/processor_with_lm/tree/main
													# are downloaded and none of the rest (e.g. README.md, ...)
													self.assertListEqual(__lowerCamelCase      ,   __lowerCamelCase )





							def 		lowerCAmelCase  (     self							:    str ):


													'''simple docstring'''



													lowerCamelCase__	:       Tuple					=						snapshot_download("hf-internal-testing/processor_with_lm" )

													lowerCamelCase__	:       Dict					=						WavaVecaProcessorWithLM.from_pretrained(__lowerCamelCase )

													lowerCamelCase__	:       Any					=						processor.decoder.model_container[processor.decoder._model_key]
													lowerCamelCase__	:       List[Any]					=						Path(language_model._kenlm_model.path.decode("utf-8" ) ).parent.parent.absolute()

													lowerCamelCase__	:       Union[str, Any]					=						os.listdir(__lowerCamelCase )
													lowerCamelCase__	:       List[str]					=						os.listdir(__lowerCamelCase )

													local_decoder_files.sort()
													expected_decoder_files.sort()

													# test that both decoder form hub and local files in cache are the same
													self.assertListEqual(__lowerCamelCase      ,   __lowerCamelCase )





							def 		lowerCAmelCase  (     self							:    Any ):


													'''simple docstring'''



													lowerCamelCase__	:       List[str]					=						WavaVecaProcessorWithLM.from_pretrained("hf-internal-testing/processor_with_lm" )
													lowerCamelCase__	:       Optional[Any]					=						AutoProcessor.from_pretrained("hf-internal-testing/processor_with_lm" )

													lowerCamelCase__	:       Dict					=						floats_list((3, 1000) )

													lowerCamelCase__	:       Union[str, Any]					=						processor_wavaveca(__lowerCamelCase      ,   return_tensors="np" )
													lowerCamelCase__	:       Union[str, Any]					=						processor_auto(__lowerCamelCase      ,   return_tensors="np" )

													for key in input_wavaveca.keys():
																			self.assertAlmostEqual(input_wavaveca[key].sum()      ,   input_auto[key].sum()      ,   delta=1E-2 )

													lowerCamelCase__	:       List[str]					=						self._get_dummy_logits()

													lowerCamelCase__	:       str					=						processor_wavaveca.batch_decode(__lowerCamelCase )
													lowerCamelCase__	:       List[Any]					=						processor_auto.batch_decode(__lowerCamelCase )

													self.assertListEqual(decoded_wavaveca.text      ,   decoded_auto.text )





							def 		lowerCAmelCase  (     self							:    Tuple ):


													'''simple docstring'''



													lowerCamelCase__	:       str					=						self.get_feature_extractor()
													lowerCamelCase__	:       Any					=						self.get_tokenizer()
													lowerCamelCase__	:       int					=						self.get_decoder()

													lowerCamelCase__	:       Union[str, Any]					=						WavaVecaProcessorWithLM(tokenizer=__lowerCamelCase      ,   feature_extractor=__lowerCamelCase      ,   decoder=__lowerCamelCase )

													self.assertListEqual(
													    processor.model_input_names      ,   feature_extractor.model_input_names      ,   msg="`processor` and `feature_extractor` model input names do not match"      ,   )





							@staticmethod
							def 		lowerCAmelCase  (     __lowerCamelCase							:    Optional[Any]      ,   __lowerCamelCase							:    Any ):


													'''simple docstring'''



													lowerCamelCase__	:       str					=						[d[key] for d in offsets]
													return retrieved_list





							def 		lowerCAmelCase  (     self							:    Union[str, Any] ):


													'''simple docstring'''



													lowerCamelCase__	:       List[Any]					=						WavaVecaProcessorWithLM.from_pretrained("hf-internal-testing/processor_with_lm" )
													lowerCamelCase__	:       List[str]					=						self._get_dummy_logits()[0]

													lowerCamelCase__	:       List[str]					=						processor.decode(__lowerCamelCase      ,   output_word_offsets=__lowerCamelCase )
													# check Wav2Vec2CTCTokenizerOutput keys for word
													self.assertEqual(len(outputs.keys() )      ,   4 )
													self.assertTrue("text" in outputs )
													self.assertTrue("word_offsets" in outputs )
													self.assertTrue(isinstance(__lowerCamelCase      ,   __lowerCamelCase ) )

													self.assertEqual(" ".join(self.get_from_offsets(outputs["word_offsets"]      ,   "word" ) )      ,   outputs.text )
													self.assertListEqual(self.get_from_offsets(outputs["word_offsets"]      ,   "word" )      ,   ["<s>", "<s>", "</s>"] )
													self.assertListEqual(self.get_from_offsets(outputs["word_offsets"]      ,   "start_offset" )      ,   [0, 2, 4] )
													self.assertListEqual(self.get_from_offsets(outputs["word_offsets"]      ,   "end_offset" )      ,   [1, 3, 5] )





							def 		lowerCAmelCase  (     self							:    List[Any] ):


													'''simple docstring'''



													lowerCamelCase__	:       Optional[Any]					=						WavaVecaProcessorWithLM.from_pretrained("hf-internal-testing/processor_with_lm" )
													lowerCamelCase__	:       Any					=						self._get_dummy_logits()

													lowerCamelCase__	:       Union[str, Any]					=						processor.batch_decode(__lowerCamelCase      ,   output_word_offsets=__lowerCamelCase )

													# check Wav2Vec2CTCTokenizerOutput keys for word
													self.assertEqual(len(outputs.keys() )      ,   4 )
													self.assertTrue("text" in outputs )
													self.assertTrue("word_offsets" in outputs )
													self.assertTrue(isinstance(__lowerCamelCase      ,   __lowerCamelCase ) )

													self.assertListEqual(
													    [" ".join(self.get_from_offsets(__lowerCamelCase      ,   "word" ) ) for o in outputs["word_offsets"]]      ,   outputs.text )
													self.assertListEqual(self.get_from_offsets(outputs["word_offsets"][0]      ,   "word" )      ,   ["<s>", "<s>", "</s>"] )
													self.assertListEqual(self.get_from_offsets(outputs["word_offsets"][0]      ,   "start_offset" )      ,   [0, 2, 4] )
													self.assertListEqual(self.get_from_offsets(outputs["word_offsets"][0]      ,   "end_offset" )      ,   [1, 3, 5] )





							@slow
							@require_torch
							@require_torchaudio
							def 		lowerCAmelCase  (     self							:    Optional[Any] ):


													'''simple docstring'''



													import torch

													lowerCamelCase__	:       str					=						load_dataset("common_voice"      ,   "en"      ,   split="train"      ,   streaming=__lowerCamelCase )
													lowerCamelCase__	:       str					=						ds.cast_column("audio"      ,   datasets.Audio(sampling_rate=16000 ) )
													lowerCamelCase__	:       Dict					=						iter(__lowerCamelCase )
													lowerCamelCase__	:       int					=						next(__lowerCamelCase )

													lowerCamelCase__	:       Optional[Any]					=						AutoProcessor.from_pretrained("patrickvonplaten/wav2vec2-base-100h-with-lm" )
													lowerCamelCase__	:       str					=						WavaVecaForCTC.from_pretrained("patrickvonplaten/wav2vec2-base-100h-with-lm" )

													# compare to filename `common_voice_en_100038.mp3` of dataset viewer on https://huggingface.co/datasets/common_voice/viewer/en/train
													lowerCamelCase__	:       Dict					=						processor(sample["audio"]["array"]      ,   return_tensors="pt" ).input_values

													with torch.no_grad():
																			lowerCamelCase__	:       Dict					=						model(__lowerCamelCase ).logits.cpu().numpy()

													lowerCamelCase__	:       Optional[int]					=						processor.decode(logits[0]      ,   output_word_offsets=__lowerCamelCase )

													lowerCamelCase__	:       Union[str, Any]					=						model.config.inputs_to_logits_ratio / processor.feature_extractor.sampling_rate
													lowerCamelCase__	:       List[Any]					=						[
													    {
													        "start_time": d["start_offset"] * time_offset,
													        "end_time": d["end_offset"] * time_offset,
													        "word": d["word"],
													    }
													    for d in output["word_offsets"]
													]

													lowerCamelCase__	:       Tuple					=						"WHY DOES MILISANDRA LOOK LIKE SHE WANTS TO CONSUME JOHN SNOW ON THE RIVER AT THE WALL"

													# output words
													self.assertEqual(" ".join(self.get_from_offsets(__lowerCamelCase      ,   "word" ) )      ,   __lowerCamelCase )
													self.assertEqual(" ".join(self.get_from_offsets(__lowerCamelCase      ,   "word" ) )      ,   output.text )

													# output times
													lowerCamelCase__	:       Tuple					=						torch.tensor(self.get_from_offsets(__lowerCamelCase      ,   "start_time" ) )
													lowerCamelCase__	:       List[str]					=						torch.tensor(self.get_from_offsets(__lowerCamelCase      ,   "end_time" ) )

													# fmt: off
													lowerCamelCase__	:       Any					=						torch.tensor([1.4_1_9_9, 1.6_5_9_9, 2.2_5_9_9, 3.0, 3.2_4, 3.5_9_9_9, 3.7_9_9_9, 4.0_9_9_9, 4.2_6, 4.9_4, 5.2_8, 5.6_5_9_9, 5.7_8, 5.9_4, 6.3_2, 6.5_3_9_9, 6.6_5_9_9] )
													lowerCamelCase__	:       Optional[int]					=						torch.tensor([1.5_3_9_9, 1.8_9_9_9, 2.9, 3.1_6, 3.5_3_9_9, 3.7_2, 4.0_1_9_9, 4.1_7_9_9, 4.7_6, 5.1_5_9_9, 5.5_5_9_9, 5.6_9_9_9, 5.8_6, 6.1_9_9_9, 6.3_8, 6.6_1_9_9, 6.9_4] )
													# fmt: on

													self.assertTrue(torch.allclose(__lowerCamelCase      ,   __lowerCamelCase      ,   atol=0.0_1 ) )
													self.assertTrue(torch.allclose(__lowerCamelCase      ,   __lowerCamelCase      ,   atol=0.0_1 ) )






from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


A   :					int	=      logging.get_logger(__name__)

A   :					Optional[int]	=      {
    "facebook/xmod-base": "https://huggingface.co/facebook/xmod-base/resolve/main/config.json",
    "facebook/xmod-large-prenorm": "https://huggingface.co/facebook/xmod-large-prenorm/resolve/main/config.json",
    "facebook/xmod-base-13-125k": "https://huggingface.co/facebook/xmod-base-13-125k/resolve/main/config.json",
    "facebook/xmod-base-30-125k": "https://huggingface.co/facebook/xmod-base-30-125k/resolve/main/config.json",
    "facebook/xmod-base-30-195k": "https://huggingface.co/facebook/xmod-base-30-195k/resolve/main/config.json",
    "facebook/xmod-base-60-125k": "https://huggingface.co/facebook/xmod-base-60-125k/resolve/main/config.json",
    "facebook/xmod-base-60-265k": "https://huggingface.co/facebook/xmod-base-60-265k/resolve/main/config.json",
    "facebook/xmod-base-75-125k": "https://huggingface.co/facebook/xmod-base-75-125k/resolve/main/config.json",
    "facebook/xmod-base-75-269k": "https://huggingface.co/facebook/xmod-base-75-269k/resolve/main/config.json",
}







class     _lowercase				(							lowercase__):


       """simple docstring"""




       A__      		=    "xmod"





       def __init__(     self							:    int      ,   __lowerCamelCase							:    Any=30522      ,   __lowerCamelCase							:    Any=768      ,   __lowerCamelCase							:    str=12      ,   __lowerCamelCase							:    Any=12      ,   __lowerCamelCase							:    List[str]=3072      ,   __lowerCamelCase							:    List[Any]="gelu"      ,   __lowerCamelCase							:    Union[str, Any]=0.1      ,   __lowerCamelCase							:    int=0.1      ,   __lowerCamelCase							:    Tuple=512      ,   __lowerCamelCase							:    str=2      ,   __lowerCamelCase							:    List[str]=0.0_2      ,   __lowerCamelCase							:    List[str]=1E-1_2      ,   __lowerCamelCase							:    str=1      ,   __lowerCamelCase							:    Optional[int]=0      ,   __lowerCamelCase							:    Optional[Any]=2      ,   __lowerCamelCase							:    str="absolute"      ,   __lowerCamelCase							:    List[str]=True      ,   __lowerCamelCase							:    Dict=None      ,   __lowerCamelCase							:    Optional[Any]=False      ,   __lowerCamelCase							:    Optional[Any]=2      ,   __lowerCamelCase							:    Tuple=False      ,   __lowerCamelCase							:    Tuple=True      ,   __lowerCamelCase							:    Union[str, Any]=True      ,   __lowerCamelCase							:    str=("en_XX",)      ,   __lowerCamelCase							:    Union[str, Any]=None      ,   **__lowerCamelCase							:    Optional[int]      ,   ):


             '''simple docstring'''



             super().__init__(pad_token_id=__lowerCamelCase      ,   bos_token_id=__lowerCamelCase      ,   eos_token_id=__lowerCamelCase      ,   **__lowerCamelCase )

             lowerCamelCase__	:       Union[str, Any]					=						vocab_size
             lowerCamelCase__	:       Union[str, Any]					=						hidden_size
             lowerCamelCase__	:       Optional[int]					=						num_hidden_layers
             lowerCamelCase__	:       List[Any]					=						num_attention_heads
             lowerCamelCase__	:       Union[str, Any]					=						hidden_act
             lowerCamelCase__	:       Optional[int]					=						intermediate_size
             lowerCamelCase__	:       Optional[int]					=						hidden_dropout_prob
             lowerCamelCase__	:       List[Any]					=						attention_probs_dropout_prob
             lowerCamelCase__	:       Any					=						max_position_embeddings
             lowerCamelCase__	:       List[Any]					=						type_vocab_size
             lowerCamelCase__	:       int					=						initializer_range
             lowerCamelCase__	:       Tuple					=						layer_norm_eps
             lowerCamelCase__	:       Union[str, Any]					=						position_embedding_type
             lowerCamelCase__	:       str					=						use_cache
             lowerCamelCase__	:       Union[str, Any]					=						classifier_dropout
             lowerCamelCase__	:       Any					=						pre_norm
             lowerCamelCase__	:       Tuple					=						adapter_reduction_factor
             lowerCamelCase__	:       Tuple					=						adapter_layer_norm
             lowerCamelCase__	:       List[Any]					=						adapter_reuse_layer_norm
             lowerCamelCase__	:       Dict					=						ln_before_adapter
             lowerCamelCase__	:       List[Any]					=						list(__lowerCamelCase )
             lowerCamelCase__	:       Optional[Any]					=						default_language







class     _lowercase				(							lowercase__):


       """simple docstring"""




       @property
       def 		lowerCAmelCase  (     self							:    Tuple ):


             '''simple docstring'''



             if self.task == "multiple-choice":
                   lowerCamelCase__	:       Dict					=						{0: "batch", 1: "choice", 2: "sequence"}
             else:
                   lowerCamelCase__	:       List[str]					=						{0: "batch", 1: "sequence"}
             return OrderedDict(
                 [
                     ("input_ids", dynamic_axis),
                     ("attention_mask", dynamic_axis),
                 ] )


