





"""simple docstring"""


import json
import os
import shutil
import tempfile
import unittest
from multiprocessing import get_context
from pathlib import Path

import datasets
import numpy as np
from datasets import load_dataset
from parameterized import parameterized

from transformers import AutoProcessor
from transformers.models.wavaveca import WavaVecaCTCTokenizer, WavaVecaFeatureExtractor
from transformers.models.wavaveca.tokenization_wavaveca import VOCAB_FILES_NAMES
from transformers.testing_utils import require_pyctcdecode, require_torch, require_torchaudio, slow
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_pyctcdecode_available, is_torch_available

from ..wavaveca.test_feature_extraction_wavaveca import floats_list


if is_pyctcdecode_available():
		from huggingface_hub import snapshot_download
		from pyctcdecode import BeamSearchDecoderCTC

		from transformers.models.wavaveca_with_lm import WavaVecaProcessorWithLM
		from transformers.models.wavaveca_with_lm.processing_wavaveca_with_lm import WavaVecaDecoderWithLMOutput

if is_torch_available():
		from transformers import WavaVecaForCTC



@require_pyctcdecode
class      __SCREAMING_SNAKE_CASE    (						unittest.TestCase   ):







		'''simple docstring'''


		def    __SCREAMING_SNAKE_CASE				(		self   :       Tuple	)		->				Optional[Any]:
								_UpperCamelCase      :	Any						=       "| <pad> <unk> <s> </s> a b c d e f g h i j k".split()
								_UpperCamelCase      :	Any						=       dict(zip(__a					,   range(len(__a	)	)	)	)

								_UpperCamelCase      :	Union[str, Any]						=       {
								    "unk_token": "<unk>",
								    "bos_token": "<s>",
								    "eos_token": "</s>",
								}
								_UpperCamelCase      :	List[str]						=       {
								    "feature_size": 1,
								    "padding_value": 0.0,
								    "sampling_rate": 1_6000,
								    "return_attention_mask": False,
								    "do_normalize": True,
								}

								_UpperCamelCase      :	Optional[Any]						=       tempfile.mkdtemp()
								_UpperCamelCase      :	Tuple						=       os.path.join(self.tmpdirname					,   VOCAB_FILES_NAMES["vocab_file"]	)
								_UpperCamelCase      :	List[Any]						=       os.path.join(self.tmpdirname					,   __a	)
								with open(self.vocab_file					,   "w"					,   encoding="utf-8"	) as fp:
														fp.write(json.dumps(__a	) + "\n"	)

								with open(self.feature_extraction_file					,   "w"					,   encoding="utf-8"	) as fp:
														fp.write(json.dumps(__a	) + "\n"	)

								# load decoder from hub
								_UpperCamelCase      :	List[Any]						=       "hf-internal-testing/ngram-beam-search-decoder"

		def    __SCREAMING_SNAKE_CASE				(		self   :       Tuple					,   **__a   :       Union[str, Any]	)		->				int:
								_UpperCamelCase      :	Tuple						=       self.add_kwargs_tokens_map.copy()
								kwargs.update(__a	)
								return WavaVecaCTCTokenizer.from_pretrained(self.tmpdirname					,   **__a	)

		def    __SCREAMING_SNAKE_CASE				(		self   :       Tuple					,   **__a   :       List[str]	)		->				int:
								return WavaVecaFeatureExtractor.from_pretrained(self.tmpdirname					,   **__a	)

		def    __SCREAMING_SNAKE_CASE				(		self   :       str					,   **__a   :       Optional[int]	)		->				Tuple:
								return BeamSearchDecoderCTC.load_from_hf_hub(self.decoder_name					,   **__a	)

		def    __SCREAMING_SNAKE_CASE				(		self   :       Union[str, Any]	)		->				List[str]:
								shutil.rmtree(self.tmpdirname	)

		def    __SCREAMING_SNAKE_CASE				(		self   :       Tuple	)		->				Union[str, Any]:
								_UpperCamelCase      :	List[Any]						=       self.get_tokenizer()
								_UpperCamelCase      :	Dict						=       self.get_feature_extractor()
								_UpperCamelCase      :	Dict						=       self.get_decoder()

								_UpperCamelCase      :	int						=       WavaVecaProcessorWithLM(tokenizer=__a					,   feature_extractor=__a					,   decoder=__a	)

								processor.save_pretrained(self.tmpdirname	)
								_UpperCamelCase      :	Optional[Any]						=       WavaVecaProcessorWithLM.from_pretrained(self.tmpdirname	)

								# tokenizer
								self.assertEqual(processor.tokenizer.get_vocab()					,   tokenizer.get_vocab()	)
								self.assertIsInstance(processor.tokenizer					,   __a	)

								# feature extractor
								self.assertEqual(processor.feature_extractor.to_json_string()					,   feature_extractor.to_json_string()	)
								self.assertIsInstance(processor.feature_extractor					,   __a	)

								# decoder
								self.assertEqual(processor.decoder._alphabet.labels					,   decoder._alphabet.labels	)
								self.assertEqual(
								    processor.decoder.model_container[decoder._model_key]._unigram_set					,   decoder.model_container[decoder._model_key]._unigram_set					,   )
								self.assertIsInstance(processor.decoder					,   __a	)

		def    __SCREAMING_SNAKE_CASE				(		self   :       Dict	)		->				Optional[int]:
								_UpperCamelCase      :	List[Any]						=       WavaVecaProcessorWithLM(
								    tokenizer=self.get_tokenizer()					,   feature_extractor=self.get_feature_extractor()					,   decoder=self.get_decoder()	)
								processor.save_pretrained(self.tmpdirname	)

								# make sure that error is thrown when decoder alphabet doesn't match
								_UpperCamelCase      :	Union[str, Any]						=       WavaVecaProcessorWithLM.from_pretrained(
								    self.tmpdirname					,   alpha=5.0					,   beta=3.0					,   score_boundary=-7.0					,   unk_score_offset=3	)

								# decoder
								self.assertEqual(processor.language_model.alpha					,   5.0	)
								self.assertEqual(processor.language_model.beta					,   3.0	)
								self.assertEqual(processor.language_model.score_boundary					,   -7.0	)
								self.assertEqual(processor.language_model.unk_score_offset					,   3	)

		def    __SCREAMING_SNAKE_CASE				(		self   :       Dict	)		->				Union[str, Any]:
								_UpperCamelCase      :	int						=       self.get_tokenizer()
								# add token to trigger raise
								tokenizer.add_tokens(["xx"]	)
								with self.assertRaisesRegex(__a					,   "include"	):
														WavaVecaProcessorWithLM(
														    tokenizer=__a					,   feature_extractor=self.get_feature_extractor()					,   decoder=self.get_decoder()	)

		def    __SCREAMING_SNAKE_CASE				(		self   :       Any	)		->				List[str]:
								_UpperCamelCase      :	Any						=       self.get_feature_extractor()
								_UpperCamelCase      :	Dict						=       self.get_tokenizer()
								_UpperCamelCase      :	str						=       self.get_decoder()

								_UpperCamelCase      :	Tuple						=       WavaVecaProcessorWithLM(tokenizer=__a					,   feature_extractor=__a					,   decoder=__a	)

								_UpperCamelCase      :	Union[str, Any]						=       floats_list((3, 1000)	)

								_UpperCamelCase      :	List[Any]						=       feature_extractor(__a					,   return_tensors="np"	)
								_UpperCamelCase      :	Optional[Any]						=       processor(__a					,   return_tensors="np"	)

								for key in input_feat_extract.keys():
														self.assertAlmostEqual(input_feat_extract[key].sum()					,   input_processor[key].sum()					,   delta=1e-2	)

		def    __SCREAMING_SNAKE_CASE				(		self   :       Any	)		->				int:
								_UpperCamelCase      :	str						=       self.get_feature_extractor()
								_UpperCamelCase      :	List[str]						=       self.get_tokenizer()
								_UpperCamelCase      :	Union[str, Any]						=       self.get_decoder()

								_UpperCamelCase      :	Dict						=       WavaVecaProcessorWithLM(tokenizer=__a					,   feature_extractor=__a					,   decoder=__a	)

								_UpperCamelCase      :	Optional[int]						=       "This is a test string"

								_UpperCamelCase      :	Optional[int]						=       processor(text=__a	)

								_UpperCamelCase      :	Any						=       tokenizer(__a	)

								for key in encoded_tok.keys():
														self.assertListEqual(encoded_tok[key]					,   encoded_processor[key]	)

		def    __SCREAMING_SNAKE_CASE				(		self   :       List[Any]					,   __a   :       Tuple=(2, 10, 16)					,   __a   :       int=77	)		->				List[Any]:
								np.random.seed(__a	)
								return np.random.rand(*__a	)

		def    __SCREAMING_SNAKE_CASE				(		self   :       List[Any]	)		->				Union[str, Any]:
								_UpperCamelCase      :	Optional[Any]						=       self.get_feature_extractor()
								_UpperCamelCase      :	List[Any]						=       self.get_tokenizer()
								_UpperCamelCase      :	Optional[Any]						=       self.get_decoder()

								_UpperCamelCase      :	Optional[int]						=       WavaVecaProcessorWithLM(tokenizer=__a					,   feature_extractor=__a					,   decoder=__a	)

								_UpperCamelCase      :	Union[str, Any]						=       self._get_dummy_logits(shape=(10, 16)					,   seed=13	)

								_UpperCamelCase      :	List[Any]						=       processor.decode(__a	)

								_UpperCamelCase      :	str						=       decoder.decode_beams(__a	)[0]

								self.assertEqual(decoded_decoder[0]					,   decoded_processor.text	)
								self.assertEqual("</s> <s> </s>"					,   decoded_processor.text	)
								self.assertEqual(decoded_decoder[-2]					,   decoded_processor.logit_score	)
								self.assertEqual(decoded_decoder[-1]					,   decoded_processor.lm_score	)

		@parameterized.expand([[None], ["fork"], ["spawn"]]	)
		def    __SCREAMING_SNAKE_CASE				(		self   :       List[str]					,   __a   :       Tuple	)		->				Any:
								_UpperCamelCase      :	str						=       self.get_feature_extractor()
								_UpperCamelCase      :	int						=       self.get_tokenizer()
								_UpperCamelCase      :	str						=       self.get_decoder()

								_UpperCamelCase      :	Dict						=       WavaVecaProcessorWithLM(tokenizer=__a					,   feature_extractor=__a					,   decoder=__a	)

								_UpperCamelCase      :	Any						=       self._get_dummy_logits()

								# note: pool should be instantiated *after* Wav2Vec2ProcessorWithLM.
								#       otherwise, the LM won't be available to the pool's sub-processes.
								# manual logic used to allow parameterized test for both pool=None and pool=Pool(...)
								if pool_context is None:
														_UpperCamelCase      :	List[str]						=       processor.batch_decode(__a	)
								else:
														with get_context(__a	).Pool() as pool:
																				_UpperCamelCase      :	List[str]						=       processor.batch_decode(__a					,   __a	)

								_UpperCamelCase      :	Any						=       list(__a	)

								with get_context("fork"	).Pool() as p:
														_UpperCamelCase      :	Optional[int]						=       decoder.decode_beams_batch(__a					,   __a	)

								_UpperCamelCase      :	Dict						=       [], [], []
								for beams in decoded_beams:
														texts_decoder.append(beams[0][0]	)
														logit_scores_decoder.append(beams[0][-2]	)
														lm_scores_decoder.append(beams[0][-1]	)

								self.assertListEqual(__a					,   decoded_processor.text	)
								self.assertListEqual(["<s> <s> </s>", "<s> <s> <s>"]					,   decoded_processor.text	)
								self.assertListEqual(__a					,   decoded_processor.logit_score	)
								self.assertListEqual(__a					,   decoded_processor.lm_score	)

		def    __SCREAMING_SNAKE_CASE				(		self   :       Union[str, Any]	)		->				int:
								_UpperCamelCase      :	Dict						=       self.get_feature_extractor()
								_UpperCamelCase      :	Any						=       self.get_tokenizer()
								_UpperCamelCase      :	Optional[int]						=       self.get_decoder()

								_UpperCamelCase      :	Optional[int]						=       WavaVecaProcessorWithLM(tokenizer=__a					,   feature_extractor=__a					,   decoder=__a	)

								_UpperCamelCase      :	int						=       self._get_dummy_logits()

								_UpperCamelCase      :	Tuple						=       15
								_UpperCamelCase      :	List[Any]						=       -20.0
								_UpperCamelCase      :	int						=       -4.0

								_UpperCamelCase      :	Union[str, Any]						=       processor.batch_decode(
								    __a					,   beam_width=__a					,   beam_prune_logp=__a					,   token_min_logp=__a					,   )
								_UpperCamelCase      :	List[str]						=       decoded_processor_out.text

								_UpperCamelCase      :	Optional[int]						=       list(__a	)

								with get_context("fork"	).Pool() as pool:
														_UpperCamelCase      :	List[str]						=       decoder.decode_beams_batch(
														    __a					,   __a					,   beam_width=__a					,   beam_prune_logp=__a					,   token_min_logp=__a					,   )

								_UpperCamelCase      :	int						=       [d[0][0] for d in decoded_decoder_out]
								_UpperCamelCase      :	List[Any]						=       [d[0][2] for d in decoded_decoder_out]
								_UpperCamelCase      :	Optional[Any]						=       [d[0][3] for d in decoded_decoder_out]

								self.assertListEqual(__a					,   __a	)
								self.assertListEqual(["</s> <s> <s>", "<s> <s> <s>"]					,   __a	)

								self.assertTrue(np.array_equal(__a					,   decoded_processor_out.logit_score	)	)
								self.assertTrue(np.allclose([-20.0_54, -18.4_47]					,   __a					,   atol=1e-3	)	)

								self.assertTrue(np.array_equal(__a					,   decoded_processor_out.lm_score	)	)
								self.assertTrue(np.allclose([-15.5_54, -13.94_74]					,   __a					,   atol=1e-3	)	)

		def    __SCREAMING_SNAKE_CASE				(		self   :       Any	)		->				Any:
								_UpperCamelCase      :	Optional[int]						=       self.get_feature_extractor()
								_UpperCamelCase      :	Union[str, Any]						=       self.get_tokenizer()
								_UpperCamelCase      :	Any						=       self.get_decoder()

								_UpperCamelCase      :	Dict						=       WavaVecaProcessorWithLM(tokenizer=__a					,   feature_extractor=__a					,   decoder=__a	)

								_UpperCamelCase      :	Union[str, Any]						=       self._get_dummy_logits()

								_UpperCamelCase      :	Optional[Any]						=       2.0
								_UpperCamelCase      :	Union[str, Any]						=       5.0
								_UpperCamelCase      :	Tuple						=       -20.0
								_UpperCamelCase      :	str						=       True

								_UpperCamelCase      :	Any						=       processor.batch_decode(
								    __a					,   alpha=__a					,   beta=__a					,   unk_score_offset=__a					,   lm_score_boundary=__a					,   )
								_UpperCamelCase      :	str						=       decoded_processor_out.text

								_UpperCamelCase      :	Union[str, Any]						=       list(__a	)
								decoder.reset_params(
								    alpha=__a					,   beta=__a					,   unk_score_offset=__a					,   lm_score_boundary=__a					,   )

								with get_context("fork"	).Pool() as pool:
														_UpperCamelCase      :	List[str]						=       decoder.decode_beams_batch(
														    __a					,   __a					,   )

								_UpperCamelCase      :	str						=       [d[0][0] for d in decoded_decoder_out]

								self.assertListEqual(__a					,   __a	)
								self.assertListEqual(["<s> </s> <s> </s> </s>", "</s> </s> <s> </s> </s>"]					,   __a	)
								_UpperCamelCase      :	Tuple						=       processor.decoder.model_container[processor.decoder._model_key]
								self.assertEqual(lm_model.alpha					,   2.0	)
								self.assertEqual(lm_model.beta					,   5.0	)
								self.assertEqual(lm_model.unk_score_offset					,   -20.0	)
								self.assertEqual(lm_model.score_boundary					,   __a	)

		def    __SCREAMING_SNAKE_CASE				(		self   :       List[Any]	)		->				Dict:
								_UpperCamelCase      :	List[Any]						=       WavaVecaProcessorWithLM.from_pretrained("hf-internal-testing/processor_with_lm"	)

								_UpperCamelCase      :	str						=       processor.decoder.model_container[processor.decoder._model_key]
								_UpperCamelCase      :	Optional[Any]						=       Path(language_model._kenlm_model.path.decode("utf-8"	)	).parent.parent.absolute()

								_UpperCamelCase      :	Dict						=       os.listdir(__a	)
								_UpperCamelCase      :	Dict						=       ["alphabet.json", "language_model"]

								downloaded_decoder_files.sort()
								expected_decoder_files.sort()

								# test that only decoder relevant files from
								# https://huggingface.co/hf-internal-testing/processor_with_lm/tree/main
								# are downloaded and none of the rest (e.g. README.md, ...)
								self.assertListEqual(__a					,   __a	)

		def    __SCREAMING_SNAKE_CASE				(		self   :       List[str]	)		->				Optional[Any]:
								_UpperCamelCase      :	List[str]						=       snapshot_download("hf-internal-testing/processor_with_lm"	)

								_UpperCamelCase      :	int						=       WavaVecaProcessorWithLM.from_pretrained(__a	)

								_UpperCamelCase      :	Optional[Any]						=       processor.decoder.model_container[processor.decoder._model_key]
								_UpperCamelCase      :	Dict						=       Path(language_model._kenlm_model.path.decode("utf-8"	)	).parent.parent.absolute()

								_UpperCamelCase      :	Dict						=       os.listdir(__a	)
								_UpperCamelCase      :	Optional[Any]						=       os.listdir(__a	)

								local_decoder_files.sort()
								expected_decoder_files.sort()

								# test that both decoder form hub and local files in cache are the same
								self.assertListEqual(__a					,   __a	)

		def    __SCREAMING_SNAKE_CASE				(		self   :       Dict	)		->				List[str]:
								_UpperCamelCase      :	List[Any]						=       WavaVecaProcessorWithLM.from_pretrained("hf-internal-testing/processor_with_lm"	)
								_UpperCamelCase      :	Dict						=       AutoProcessor.from_pretrained("hf-internal-testing/processor_with_lm"	)

								_UpperCamelCase      :	Dict						=       floats_list((3, 1000)	)

								_UpperCamelCase      :	Optional[Any]						=       processor_wavaveca(__a					,   return_tensors="np"	)
								_UpperCamelCase      :	Dict						=       processor_auto(__a					,   return_tensors="np"	)

								for key in input_wavaveca.keys():
														self.assertAlmostEqual(input_wavaveca[key].sum()					,   input_auto[key].sum()					,   delta=1e-2	)

								_UpperCamelCase      :	List[str]						=       self._get_dummy_logits()

								_UpperCamelCase      :	str						=       processor_wavaveca.batch_decode(__a	)
								_UpperCamelCase      :	str						=       processor_auto.batch_decode(__a	)

								self.assertListEqual(decoded_wavaveca.text					,   decoded_auto.text	)

		def    __SCREAMING_SNAKE_CASE				(		self   :       Tuple	)		->				Optional[Any]:
								_UpperCamelCase      :	Any						=       self.get_feature_extractor()
								_UpperCamelCase      :	Optional[int]						=       self.get_tokenizer()
								_UpperCamelCase      :	Any						=       self.get_decoder()

								_UpperCamelCase      :	Tuple						=       WavaVecaProcessorWithLM(tokenizer=__a					,   feature_extractor=__a					,   decoder=__a	)

								self.assertListEqual(
								    processor.model_input_names					,   feature_extractor.model_input_names					,   msg="`processor` and `feature_extractor` model input names do not match"					,   )

		@staticmethod
		def    __SCREAMING_SNAKE_CASE				(		__a   :       Optional[Any]					,   __a   :       Optional[int]	)		->				int:
								_UpperCamelCase      :	Dict						=       [d[key] for d in offsets]
								return retrieved_list

		def    __SCREAMING_SNAKE_CASE				(		self   :       Optional[int]	)		->				Optional[Any]:
								_UpperCamelCase      :	Any						=       WavaVecaProcessorWithLM.from_pretrained("hf-internal-testing/processor_with_lm"	)
								_UpperCamelCase      :	int						=       self._get_dummy_logits()[0]

								_UpperCamelCase      :	List[str]						=       processor.decode(__a					,   output_word_offsets=__a	)
								# check Wav2Vec2CTCTokenizerOutput keys for word
								self.assertEqual(len(outputs.keys()	)					,   4	)
								self.assertTrue("text" in outputs	)
								self.assertTrue("word_offsets" in outputs	)
								self.assertTrue(isinstance(__a					,   __a	)	)

								self.assertEqual(" ".join(self.get_from_offsets(outputs["word_offsets"]					,   "word"	)	)					,   outputs.text	)
								self.assertListEqual(self.get_from_offsets(outputs["word_offsets"]					,   "word"	)					,   ["<s>", "<s>", "</s>"]	)
								self.assertListEqual(self.get_from_offsets(outputs["word_offsets"]					,   "start_offset"	)					,   [0, 2, 4]	)
								self.assertListEqual(self.get_from_offsets(outputs["word_offsets"]					,   "end_offset"	)					,   [1, 3, 5]	)

		def    __SCREAMING_SNAKE_CASE				(		self   :       int	)		->				Optional[Any]:
								_UpperCamelCase      :	List[str]						=       WavaVecaProcessorWithLM.from_pretrained("hf-internal-testing/processor_with_lm"	)
								_UpperCamelCase      :	List[Any]						=       self._get_dummy_logits()

								_UpperCamelCase      :	str						=       processor.batch_decode(__a					,   output_word_offsets=__a	)

								# check Wav2Vec2CTCTokenizerOutput keys for word
								self.assertEqual(len(outputs.keys()	)					,   4	)
								self.assertTrue("text" in outputs	)
								self.assertTrue("word_offsets" in outputs	)
								self.assertTrue(isinstance(__a					,   __a	)	)

								self.assertListEqual(
								    [" ".join(self.get_from_offsets(__a					,   "word"	)	) for o in outputs["word_offsets"]]					,   outputs.text	)
								self.assertListEqual(self.get_from_offsets(outputs["word_offsets"][0]					,   "word"	)					,   ["<s>", "<s>", "</s>"]	)
								self.assertListEqual(self.get_from_offsets(outputs["word_offsets"][0]					,   "start_offset"	)					,   [0, 2, 4]	)
								self.assertListEqual(self.get_from_offsets(outputs["word_offsets"][0]					,   "end_offset"	)					,   [1, 3, 5]	)


		@slow
		@require_torch
		@require_torchaudio
		def    __SCREAMING_SNAKE_CASE				(		self   :       Any	)		->				Union[str, Any]:
								import torch

								_UpperCamelCase      :	Optional[int]						=       load_dataset("common_voice"					,   "en"					,   split="train"					,   streaming=__a	)
								_UpperCamelCase      :	Optional[int]						=       ds.cast_column("audio"					,   datasets.Audio(sampling_rate=1_6000	)	)
								_UpperCamelCase      :	Any						=       iter(__a	)
								_UpperCamelCase      :	Union[str, Any]						=       next(__a	)

								_UpperCamelCase      :	Optional[Any]						=       AutoProcessor.from_pretrained("patrickvonplaten/wav2vec2-base-100h-with-lm"	)
								_UpperCamelCase      :	List[Any]						=       WavaVecaForCTC.from_pretrained("patrickvonplaten/wav2vec2-base-100h-with-lm"	)

								# compare to filename `common_voice_en_100038.mp3` of dataset viewer on https://huggingface.co/datasets/common_voice/viewer/en/train
								_UpperCamelCase      :	int						=       processor(sample["audio"]["array"]					,   return_tensors="pt"	).input_values

								with torch.no_grad():
														_UpperCamelCase      :	Optional[Any]						=       model(__a	).logits.cpu().numpy()

								_UpperCamelCase      :	Dict						=       processor.decode(logits[0]					,   output_word_offsets=__a	)

								_UpperCamelCase      :	Any						=       model.config.inputs_to_logits_ratio / processor.feature_extractor.sampling_rate
								_UpperCamelCase      :	str						=       [
								    {
								        "start_time": d["start_offset"] * time_offset,
								        "end_time": d["end_offset"] * time_offset,
								        "word": d["word"],
								    }
								    for d in output["word_offsets"]
								]

								_UpperCamelCase      :	Dict						=       "WHY DOES MILISANDRA LOOK LIKE SHE WANTS TO CONSUME JOHN SNOW ON THE RIVER AT THE WALL"

								# output words
								self.assertEqual(" ".join(self.get_from_offsets(__a					,   "word"	)	)					,   __a	)
								self.assertEqual(" ".join(self.get_from_offsets(__a					,   "word"	)	)					,   output.text	)

								# output times
								_UpperCamelCase      :	List[str]						=       torch.tensor(self.get_from_offsets(__a					,   "start_time"	)	)
								_UpperCamelCase      :	List[Any]						=       torch.tensor(self.get_from_offsets(__a					,   "end_time"	)	)

								# fmt: off
								_UpperCamelCase      :	List[Any]						=       torch.tensor([1.41_99, 1.65_99, 2.25_99, 3.0, 3.24, 3.59_99, 3.79_99, 4.09_99, 4.26, 4.94, 5.28, 5.65_99, 5.78, 5.94, 6.32, 6.53_99, 6.65_99]	)
								_UpperCamelCase      :	Tuple						=       torch.tensor([1.53_99, 1.89_99, 2.9, 3.16, 3.53_99, 3.72, 4.01_99, 4.17_99, 4.76, 5.15_99, 5.55_99, 5.69_99, 5.86, 6.19_99, 6.38, 6.61_99, 6.94]	)
								# fmt: on

								self.assertTrue(torch.allclose(__a					,   __a					,   atol=0.01	)	)
								self.assertTrue(torch.allclose(__a					,   __a					,   atol=0.01	)	)







"""simple docstring"""


import argparse
import collections
import os
import re

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_table.py
lowerCamelCase__				  =   "src/transformers"
lowerCamelCase__				  =   "docs/source/en"
lowerCamelCase__				  =   "."




def 					lowercase__    (					lowercase_		,lowercase_		,lowercase_			)					->		List[str]:




      """simple docstring"""

      with open(lowercase_		,"r"		,encoding="utf-8"		,newline="\n"			) as f:
            _UpperCamelCase      :	Union[str, Any]						=       f.readlines()
      # Find the start prompt.
      _UpperCamelCase      :	Dict						=       0
      while not lines[start_index].startswith(lowercase_			):
            start_index += 1
      start_index += 1

      _UpperCamelCase      :	Optional[int]						=       start_index
      while not lines[end_index].startswith(lowercase_			):
            end_index += 1
      end_index -= 1

      while len(lines[start_index]			) <= 1:
            start_index += 1
      while len(lines[end_index]			) <= 1:
            end_index -= 1
      end_index += 1
      return "".join(lines[start_index:end_index]			), start_index, end_index, lines


# Add here suffixes that are used to identify models, separated by |
lowerCamelCase__				  =   "Model|Encoder|Decoder|ForConditionalGeneration"
# Regexes that match TF/Flax/PT model names.
lowerCamelCase__				  =   re.compile(R"TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")
lowerCamelCase__				  =   re.compile(R"Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
lowerCamelCase__				  =   re.compile(R"(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")


# This is to make sure the transformers module imported is the one in the repo.
lowerCamelCase__				  =   direct_transformers_import(TRANSFORMERS_PATH)




def 					lowercase__    (					lowercase_			)					->		Any:




      """simple docstring"""

      _UpperCamelCase      :	Tuple						=       re.finditer(".+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)"		,lowercase_			)
      return [m.group(0			) for m in matches]




def 					lowercase__    (					lowercase_		,lowercase_			)					->		Union[str, Any]:




      """simple docstring"""

      _UpperCamelCase      :	str						=       2 if text == "✅" or text == "❌" else len(lowercase_			)
      _UpperCamelCase      :	Union[str, Any]						=       (width - text_length) // 2
      _UpperCamelCase      :	Dict						=       width - text_length - left_indent
      return " " * left_indent + text + " " * right_indent




def 					lowercase__    (					)					->		str:




      """simple docstring"""

      _UpperCamelCase      :	Optional[Any]						=       transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
      _UpperCamelCase      :	str						=       {
          name: config_maping_names[code]
          for code, name in transformers_module.MODEL_NAMES_MAPPING.items()
          if code in config_maping_names
      }
      _UpperCamelCase      :	Dict						=       {name: config.replace("Config"		,""			) for name, config in model_name_to_config.items()}

      # Dictionaries flagging if each model prefix has a slow/fast tokenizer, backend in PT/TF/Flax.
      _UpperCamelCase      :	int						=       collections.defaultdict(lowercase_			)
      _UpperCamelCase      :	Dict						=       collections.defaultdict(lowercase_			)
      _UpperCamelCase      :	Dict						=       collections.defaultdict(lowercase_			)
      _UpperCamelCase      :	int						=       collections.defaultdict(lowercase_			)
      _UpperCamelCase      :	str						=       collections.defaultdict(lowercase_			)

      # Let's lookup through all transformers object (once).
      for attr_name in dir(lowercase_			):
            _UpperCamelCase      :	List[str]						=       None
            if attr_name.endswith("Tokenizer"			):
                  _UpperCamelCase      :	Tuple						=       slow_tokenizers
                  _UpperCamelCase      :	Any						=       attr_name[:-9]
            elif attr_name.endswith("TokenizerFast"			):
                  _UpperCamelCase      :	Optional[Any]						=       fast_tokenizers
                  _UpperCamelCase      :	List[str]						=       attr_name[:-13]
            elif _re_tf_models.match(lowercase_			) is not None:
                  _UpperCamelCase      :	List[Any]						=       tf_models
                  _UpperCamelCase      :	Dict						=       _re_tf_models.match(lowercase_			).groups()[0]
            elif _re_flax_models.match(lowercase_			) is not None:
                  _UpperCamelCase      :	Dict						=       flax_models
                  _UpperCamelCase      :	Union[str, Any]						=       _re_flax_models.match(lowercase_			).groups()[0]
            elif _re_pt_models.match(lowercase_			) is not None:
                  _UpperCamelCase      :	Optional[int]						=       pt_models
                  _UpperCamelCase      :	Any						=       _re_pt_models.match(lowercase_			).groups()[0]

            if lookup_dict is not None:
                  while len(lowercase_			) > 0:
                        if attr_name in model_name_to_prefix.values():
                              _UpperCamelCase      :	Dict						=       True
                              break
                        # Try again after removing the last word in the name
                        _UpperCamelCase      :	List[str]						=       "".join(camel_case_split(lowercase_			)[:-1]			)

    # Let's build that table!
      _UpperCamelCase      :	Any						=       list(model_name_to_config.keys()			)
      model_names.sort(key=str.lower			)
      _UpperCamelCase      :	List[str]						=       ["Model", "Tokenizer slow", "Tokenizer fast", "PyTorch support", "TensorFlow support", "Flax Support"]
      # We'll need widths to properly display everything in the center (+2 is to leave one extra space on each side).
      _UpperCamelCase      :	Union[str, Any]						=       [len(lowercase_			) + 2 for c in columns]
      _UpperCamelCase      :	Any						=       max([len(lowercase_			) for name in model_names]			) + 2

      # Build the table per se
      _UpperCamelCase      :	Tuple						=       "|" + "|".join([_center_text(lowercase_		,lowercase_			) for c, w in zip(lowercase_		,lowercase_			)]			) + "|\n"
      # Use ":-----:" format to center-aligned table cell texts
      table += "|" + "|".join([":" + "-" * (w - 2) + ":" for w in widths]			) + "|\n"

      _UpperCamelCase      :	Union[str, Any]						=       {True: "✅", False: "❌"}
      for name in model_names:
            _UpperCamelCase      :	Optional[int]						=       model_name_to_prefix[name]
            _UpperCamelCase      :	Tuple						=       [
                name,
                check[slow_tokenizers[prefix]],
                check[fast_tokenizers[prefix]],
                check[pt_models[prefix]],
                check[tf_models[prefix]],
                check[flax_models[prefix]],
            ]
            table += "|" + "|".join([_center_text(lowercase_		,lowercase_			) for l, w in zip(lowercase_		,lowercase_			)]			) + "|\n"
      return table




def 					lowercase__    (					lowercase_=False			)					->		List[Any]:




      """simple docstring"""

      _UpperCamelCase,  _UpperCamelCase,  _UpperCamelCase,  _UpperCamelCase      :	str						=       _find_text_in_file(
          filename=os.path.join(lowercase_		,"index.md"			)		,start_prompt="<!--This table is updated automatically from the auto modules"		,end_prompt="<!-- End table-->"		,)
      _UpperCamelCase      :	Any						=       get_model_table_from_auto_modules()

      if current_table != new_table:
            if overwrite:
                  with open(os.path.join(lowercase_		,"index.md"			)		,"w"		,encoding="utf-8"		,newline="\n"			) as f:
                        f.writelines(lines[:start_index] + [new_table] + lines[end_index:]			)
            else:
                  raise ValueError(
                      "The model table in the `index.md` has not been updated. Run `make fix-copies` to fix this."			)


if __name__ == "__main__":
  lowerCamelCase__				  =   argparse.ArgumentParser()
  parser.add_argument("--fix_and_overwrite", action="store_true", help="Whether to fix inconsistencies.")
  lowerCamelCase__				  =   parser.parse_args()

  check_model_table(args.fix_and_overwrite)
