



"""simple docstring"""





import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class    lowercase__     (					__a       ,      unittest.TestCase   ):







       '''simple docstring'''


       UpperCamelCase		=  MobileBertTokenizer
       UpperCamelCase		=  MobileBertTokenizerFast
       UpperCamelCase		=  True
       UpperCamelCase		=  True
       UpperCamelCase		=  filter_non_english
       UpperCamelCase		=  "google/mobilebert-uncased"



       def   lowercase__       (							self    :						Union[str, Any]					)				->		Tuple:


             '''simple docstring'''




             super().setUp()

             UpperCAmelCase_    				=							[
                 "[UNK]",
                 "[CLS]",
                 "[SEP]",
                 "[PAD]",
                 "[MASK]",
                 "want",
                 "##want",
                 "##ed",
                 "wa",
                 "un",
                 "runn",
                 "##ing",
                 ",",
                 "low",
                 "lowest",
             ]
             UpperCAmelCase_    				=							os.path.join(self.tmpdirname	,       VOCAB_FILES_NAMES["vocab_file"]					)
             with open(self.vocab_file	,       "w"	,       encoding="utf-8"					) as vocab_writer:
                   vocab_writer.write("".join([x + "\n" for x in vocab_tokens]					)					)

             UpperCAmelCase_    				=							[
                 (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
                 for tokenizer_def in self.tokenizers_list
             ]



       def   lowercase__       (							self    :						Tuple	,       _UpperCAmelCase    :						Union[str, Any]					)				->		Optional[Any]:


             '''simple docstring'''




             UpperCAmelCase_    				=							"UNwant\u00E9d,running"
             UpperCAmelCase_    				=							"unwanted, running"
             return input_text, output_text



       def   lowercase__       (							self    :						Optional[Any]					)				->		Tuple:


             '''simple docstring'''




             UpperCAmelCase_    				=							self.tokenizer_class(self.vocab_file					)

             UpperCAmelCase_    				=							tokenizer.tokenize("UNwant\u00E9d,running"					)
             self.assertListEqual(lowerCAmelCase_	,       ["un", "##want", "##ed", ",", "runn", "##ing"]					)
             self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_					)	,       [9, 6, 7, 12, 10, 11]					)



       def   lowercase__       (							self    :						Union[str, Any]					)				->		Dict:


             '''simple docstring'''




             if not self.test_rust_tokenizer:
                   return

             UpperCAmelCase_    				=							self.get_tokenizer()
             UpperCAmelCase_    				=							self.get_rust_tokenizer()

             UpperCAmelCase_    				=							"UNwant\u00E9d,running"

             UpperCAmelCase_    				=							tokenizer.tokenize(lowerCAmelCase_					)
             UpperCAmelCase_    				=							rust_tokenizer.tokenize(lowerCAmelCase_					)
             self.assertListEqual(lowerCAmelCase_	,       lowerCAmelCase_					)

             UpperCAmelCase_    				=							tokenizer.encode(lowerCAmelCase_	,       add_special_tokens=lowerCAmelCase_					)
             UpperCAmelCase_    				=							rust_tokenizer.encode(lowerCAmelCase_	,       add_special_tokens=lowerCAmelCase_					)
             self.assertListEqual(lowerCAmelCase_	,       lowerCAmelCase_					)

             UpperCAmelCase_    				=							self.get_rust_tokenizer()
             UpperCAmelCase_    				=							tokenizer.encode(lowerCAmelCase_					)
             UpperCAmelCase_    				=							rust_tokenizer.encode(lowerCAmelCase_					)
             self.assertListEqual(lowerCAmelCase_	,       lowerCAmelCase_					)

             # With lower casing
             UpperCAmelCase_    				=							self.get_tokenizer(do_lower_case=lowerCAmelCase_					)
             UpperCAmelCase_    				=							self.get_rust_tokenizer(do_lower_case=lowerCAmelCase_					)

             UpperCAmelCase_    				=							"UNwant\u00E9d,running"

             UpperCAmelCase_    				=							tokenizer.tokenize(lowerCAmelCase_					)
             UpperCAmelCase_    				=							rust_tokenizer.tokenize(lowerCAmelCase_					)
             self.assertListEqual(lowerCAmelCase_	,       lowerCAmelCase_					)

             UpperCAmelCase_    				=							tokenizer.encode(lowerCAmelCase_	,       add_special_tokens=lowerCAmelCase_					)
             UpperCAmelCase_    				=							rust_tokenizer.encode(lowerCAmelCase_	,       add_special_tokens=lowerCAmelCase_					)
             self.assertListEqual(lowerCAmelCase_	,       lowerCAmelCase_					)

             UpperCAmelCase_    				=							self.get_rust_tokenizer()
             UpperCAmelCase_    				=							tokenizer.encode(lowerCAmelCase_					)
             UpperCAmelCase_    				=							rust_tokenizer.encode(lowerCAmelCase_					)
             self.assertListEqual(lowerCAmelCase_	,       lowerCAmelCase_					)



       def   lowercase__       (							self    :						str					)				->		Any:


             '''simple docstring'''




             UpperCAmelCase_    				=							BasicTokenizer()

             self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"					)	,       ["ah", "\u535A", "\u63A8", "zz"]					)



       def   lowercase__       (							self    :						Optional[Any]					)				->		Optional[Any]:


             '''simple docstring'''




             UpperCAmelCase_    				=							BasicTokenizer(do_lower_case=lowerCAmelCase_					)

             self.assertListEqual(
                 tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "					)	,       ["hello", "!", "how", "are", "you", "?"]					)
             self.assertListEqual(tokenizer.tokenize("H\u00E9llo"					)	,       ["hello"]					)



       def   lowercase__       (							self    :						Optional[int]					)				->		Any:


             '''simple docstring'''




             UpperCAmelCase_    				=							BasicTokenizer(do_lower_case=lowerCAmelCase_	,       strip_accents=lowerCAmelCase_					)

             self.assertListEqual(
                 tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "					)	,       ["hällo", "!", "how", "are", "you", "?"]					)
             self.assertListEqual(tokenizer.tokenize("H\u00E9llo"					)	,       ["h\u00E9llo"]					)



       def   lowercase__       (							self    :						Optional[int]					)				->		str:


             '''simple docstring'''




             UpperCAmelCase_    				=							BasicTokenizer(do_lower_case=lowerCAmelCase_	,       strip_accents=lowerCAmelCase_					)

             self.assertListEqual(
                 tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "					)	,       ["hallo", "!", "how", "are", "you", "?"]					)
             self.assertListEqual(tokenizer.tokenize("H\u00E9llo"					)	,       ["hello"]					)



       def   lowercase__       (							self    :						List[Any]					)				->		List[str]:


             '''simple docstring'''




             UpperCAmelCase_    				=							BasicTokenizer(do_lower_case=lowerCAmelCase_					)

             self.assertListEqual(
                 tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "					)	,       ["hallo", "!", "how", "are", "you", "?"]					)
             self.assertListEqual(tokenizer.tokenize("H\u00E9llo"					)	,       ["hello"]					)



       def   lowercase__       (							self    :						int					)				->		Any:


             '''simple docstring'''




             UpperCAmelCase_    				=							BasicTokenizer(do_lower_case=lowerCAmelCase_					)

             self.assertListEqual(
                 tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "					)	,       ["HeLLo", "!", "how", "Are", "yoU", "?"]					)



       def   lowercase__       (							self    :						Dict					)				->		Tuple:


             '''simple docstring'''




             UpperCAmelCase_    				=							BasicTokenizer(do_lower_case=lowerCAmelCase_	,       strip_accents=lowerCAmelCase_					)

             self.assertListEqual(
                 tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "					)	,       ["HäLLo", "!", "how", "Are", "yoU", "?"]					)



       def   lowercase__       (							self    :						str					)				->		List[str]:


             '''simple docstring'''




             UpperCAmelCase_    				=							BasicTokenizer(do_lower_case=lowerCAmelCase_	,       strip_accents=lowerCAmelCase_					)

             self.assertListEqual(
                 tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "					)	,       ["HaLLo", "!", "how", "Are", "yoU", "?"]					)



       def   lowercase__       (							self    :						Optional[Any]					)				->		Dict:


             '''simple docstring'''




             UpperCAmelCase_    				=							BasicTokenizer(do_lower_case=lowerCAmelCase_	,       never_split=["[UNK]"]					)

             self.assertListEqual(
                 tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"					)	,       ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]					)



       def   lowercase__       (							self    :						Optional[int]					)				->		Tuple:


             '''simple docstring'''




             UpperCAmelCase_    				=							["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

             UpperCAmelCase_    				=							{}
             for i, token in enumerate(lowerCAmelCase_					):
                   UpperCAmelCase_    				=							i
             UpperCAmelCase_    				=							WordpieceTokenizer(vocab=lowerCAmelCase_	,       unk_token="[UNK]"					)

             self.assertListEqual(tokenizer.tokenize(""					)	,       []					)

             self.assertListEqual(tokenizer.tokenize("unwanted running"					)	,       ["un", "##want", "##ed", "runn", "##ing"]					)

             self.assertListEqual(tokenizer.tokenize("unwantedX running"					)	,       ["[UNK]", "runn", "##ing"]					)



       def   lowercase__       (							self    :						List[str]					)				->		Dict:


             '''simple docstring'''




             self.assertTrue(_is_whitespace(" "					)					)
             self.assertTrue(_is_whitespace("\t"					)					)
             self.assertTrue(_is_whitespace("\r"					)					)
             self.assertTrue(_is_whitespace("\n"					)					)
             self.assertTrue(_is_whitespace("\u00A0"					)					)

             self.assertFalse(_is_whitespace("A"					)					)
             self.assertFalse(_is_whitespace("-"					)					)



       def   lowercase__       (							self    :						Dict					)				->		List[Any]:


             '''simple docstring'''




             self.assertTrue(_is_control("\u0005"					)					)

             self.assertFalse(_is_control("A"					)					)
             self.assertFalse(_is_control(" "					)					)
             self.assertFalse(_is_control("\t"					)					)
             self.assertFalse(_is_control("\r"					)					)



       def   lowercase__       (							self    :						Dict					)				->		Dict:


             '''simple docstring'''




             self.assertTrue(_is_punctuation("-"					)					)
             self.assertTrue(_is_punctuation("$"					)					)
             self.assertTrue(_is_punctuation("`"					)					)
             self.assertTrue(_is_punctuation("."					)					)

             self.assertFalse(_is_punctuation("A"					)					)
             self.assertFalse(_is_punctuation(" "					)					)



       def   lowercase__       (							self    :						Optional[int]					)				->		List[Any]:


             '''simple docstring'''




             UpperCAmelCase_    				=							self.get_tokenizer()
             UpperCAmelCase_    				=							self.get_rust_tokenizer()

             # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
             self.assertListEqual([tokenizer.tokenize(lowerCAmelCase_					) for t in ["Test", "\xad", "test"]]	,       [["[UNK]"], [], ["[UNK]"]]					)

             self.assertListEqual(
                 [rust_tokenizer.tokenize(lowerCAmelCase_					) for t in ["Test", "\xad", "test"]]	,       [["[UNK]"], [], ["[UNK]"]]					)



       @slow
       def   lowercase__       (							self    :						Dict					)				->		Optional[Any]:


             '''simple docstring'''




             UpperCAmelCase_    				=							self.tokenizer_class.from_pretrained("google/mobilebert-uncased"					)

             UpperCAmelCase_    				=							tokenizer.encode("sequence builders"	,       add_special_tokens=lowerCAmelCase_					)
             UpperCAmelCase_    				=							tokenizer.encode("multi-sequence build"	,       add_special_tokens=lowerCAmelCase_					)

             UpperCAmelCase_    				=							tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_					)
             UpperCAmelCase_    				=							tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_	,       lowerCAmelCase_					)

             assert encoded_sentence == [101] + text + [102]
             assert encoded_pair == [101] + text + [102] + text_a + [102]



       def   lowercase__       (							self    :						Dict					)				->		Dict:


             '''simple docstring'''




             for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                   with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""					):
                         UpperCAmelCase_    				=							self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_	,       **lowerCAmelCase_					)

                         UpperCAmelCase_    				=							F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
                         UpperCAmelCase_    				=							tokenizer_r.encode_plus(
                             lowerCAmelCase_	,       return_attention_mask=lowerCAmelCase_	,       return_token_type_ids=lowerCAmelCase_	,       return_offsets_mapping=lowerCAmelCase_	,       add_special_tokens=lowerCAmelCase_	,       )

                         UpperCAmelCase_    				=							tokenizer_r.do_lower_case if hasattr(lowerCAmelCase_	,       "do_lower_case"					) else False
                         UpperCAmelCase_    				=							(
                             [
                                 ((0, 0), tokenizer_r.cls_token),
                                 ((0, 1), "A"),
                                 ((1, 2), ","),
                                 ((3, 5), "na"),
                                 ((5, 6), "##ï"),
                                 ((6, 8), "##ve"),
                                 ((9, 15), tokenizer_r.mask_token),
                                 ((16, 21), "Allen"),
                                 ((21, 23), "##NL"),
                                 ((23, 24), "##P"),
                                 ((25, 33), "sentence"),
                                 ((33, 34), "."),
                                 ((0, 0), tokenizer_r.sep_token),
                             ]
                             if not do_lower_case
                             else [
                                 ((0, 0), tokenizer_r.cls_token),
                                 ((0, 1), "a"),
                                 ((1, 2), ","),
                                 ((3, 8), "naive"),
                                 ((9, 15), tokenizer_r.mask_token),
                                 ((16, 21), "allen"),
                                 ((21, 23), "##nl"),
                                 ((23, 24), "##p"),
                                 ((25, 33), "sentence"),
                                 ((33, 34), "."),
                                 ((0, 0), tokenizer_r.sep_token),
                             ]
                         )

                         self.assertEqual(
                             [e[1] for e in expected_results]	,       tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]					)					)
                         self.assertEqual([e[0] for e in expected_results]	,       tokens["offset_mapping"]					)







       def   lowercase__       (							self    :						Optional[int]					)				->		int:


             '''simple docstring'''




             UpperCAmelCase_    				=							["的", "人", "有"]
             UpperCAmelCase_    				=							"".join(lowerCAmelCase_					)
             for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                   with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""					):
                         UpperCAmelCase_    				=							True
                         UpperCAmelCase_    				=							self.tokenizer_class.from_pretrained(lowerCAmelCase_	,       **lowerCAmelCase_					)
                         UpperCAmelCase_    				=							self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_	,       **lowerCAmelCase_					)

                         UpperCAmelCase_    				=							tokenizer_p.encode(lowerCAmelCase_	,       add_special_tokens=lowerCAmelCase_					)
                         UpperCAmelCase_    				=							tokenizer_r.encode(lowerCAmelCase_	,       add_special_tokens=lowerCAmelCase_					)

                         UpperCAmelCase_    				=							tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_					)
                         UpperCAmelCase_    				=							tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_					)

                         # it is expected that each Chinese character is not preceded by "##"
                         self.assertListEqual(lowerCAmelCase_	,       lowerCAmelCase_					)
                         self.assertListEqual(lowerCAmelCase_	,       lowerCAmelCase_					)

                         UpperCAmelCase_    				=							False
                         UpperCAmelCase_    				=							self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_	,       **lowerCAmelCase_					)
                         UpperCAmelCase_    				=							self.tokenizer_class.from_pretrained(lowerCAmelCase_	,       **lowerCAmelCase_					)

                         UpperCAmelCase_    				=							tokenizer_r.encode(lowerCAmelCase_	,       add_special_tokens=lowerCAmelCase_					)
                         UpperCAmelCase_    				=							tokenizer_p.encode(lowerCAmelCase_	,       add_special_tokens=lowerCAmelCase_					)

                         UpperCAmelCase_    				=							tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_					)
                         UpperCAmelCase_    				=							tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_					)

                         # it is expected that only the first Chinese character is not preceded by "##".
                         UpperCAmelCase_    				=							[
                             F"""##{token}""" if idx != 0 else token for idx, token in enumerate(lowerCAmelCase_					)
                         ]
                         self.assertListEqual(lowerCAmelCase_	,       lowerCAmelCase_					)
                         self.assertListEqual(lowerCAmelCase_	,       lowerCAmelCase_					)









"""simple docstring"""





from heapq import heappop, heappush

import numpy as np
def      a__			(   lowerCAmelCase__			,    lowerCAmelCase__			,    lowerCAmelCase__			,    lowerCAmelCase__			,    ):
      UpperCAmelCase_				,					UpperCAmelCase_    				=							grid.shape
      UpperCAmelCase_    				=							[-1, 1, 0, 0]
      UpperCAmelCase_    				=							[0, 0, -1, 1]
      if allow_diagonal:
            dx += [-1, -1, 1, 1]
            dy += [-1, 1, -1, 1]

      UpperCAmelCase_				,					UpperCAmelCase_    				=							[(0, source)], set()
      UpperCAmelCase_    				=							np.full((rows, cols)			,    np.inf   )
      UpperCAmelCase_    				=							0
      UpperCAmelCase_    				=							np.empty((rows, cols)			,    dtype=lowerCAmelCase__   )
      UpperCAmelCase_    				=							None

      while queue:
            ((UpperCAmelCase_)				,					(UpperCAmelCase_))    				=							heappop(lowerCAmelCase__   )
            if (x, y) in visited:
                  continue
            visited.add((x, y)   )

            if (x, y) == destination:
                  UpperCAmelCase_    				=							[]
                  while (x, y) != source:
                        path.append((x, y)   )
                        UpperCAmelCase_				,					UpperCAmelCase_    				=							predecessors[x, y]
                  path.append(lowerCAmelCase__   )  # add the source manually
                  path.reverse()
                  return matrix[destination], path

            for i in range(len(lowerCAmelCase__   )   ):
                  UpperCAmelCase_				,					UpperCAmelCase_    				=							x + dx[i], y + dy[i]
                  if 0 <= nx < rows and 0 <= ny < cols:
                        UpperCAmelCase_    				=							grid[nx][ny]
                        if next_node == 1 and matrix[nx, ny] > dist + 1:
                              heappush(lowerCAmelCase__			,    (dist + 1, (nx, ny))   )
                              UpperCAmelCase_    				=							dist + 1
                              UpperCAmelCase_    				=							(x, y)

      return np.inf, []


if __name__ == "__main__":
  import doctest

  doctest.testmod()




