




'''simple docstring'''


import unittest

from transformers.testing_utils import require_bsa
from transformers.utils import is_bsa_available

from ...test_feature_extraction_common import FeatureExtractionSavingTestMixin


if is_bsa_available():
  from transformers import MarkupLMFeatureExtractor





class    __lowercase						(  unittest.TestCase ):







   def __init__(	self      ,							UpperCamelCase      )				->    Dict:
     __a			=					parent







   def 		UpperCamelCase__			(	self      )				->    str:
     return {}


def 						SCREAMING_SNAKE_CASE       (      ):
  __a			=					'<HTML>\n\n    <HEAD>\n    <TITLE>sample document</TITLE>\n    </HEAD>\n\n    <BODY BGCOLOR="FFFFFF">\n    <HR>\n    <a href="http://google.com">Goog</a>\n    <H1>This is one header</H1>\n    <H2>This is a another Header</H2>\n    <P>Travel from\n        <P>\n        <B>SFO to JFK</B>\n        <BR>\n        <B><I>on May 2, 2015 at 2:00 pm. For details go to confirm.com </I></B>\n        <HR>\n        <div style="color:#0000FF">\n            <h3>Traveler <b> name </b> is\n            <p> John Doe </p>\n        </div>'

  __a			=					'\n    <!DOCTYPE html>\n    <html>\n    <body>\n\n    <h1>My First Heading</h1>\n    <p>My first paragraph.</p>\n\n    </body>\n    </html>\n    '

  return [html_string_a, html_string_a]





@require_bsa
class    __lowercase						(  __magic_name__       ,							unittest.TestCase ):
   _a							=      MarkupLMFeatureExtractor if is_bsa_available() else None







   def 		UpperCamelCase__			(	self      )				->    Dict:
     __a			=					MarkupLMFeatureExtractionTester(self      )







   @property
   def 		UpperCamelCase__			(	self      )				->    Optional[int]:
     return self.feature_extract_tester.prepare_feat_extract_dict()







   def 		UpperCamelCase__			(	self      )				->    List[str]:
     # Initialize feature_extractor
     __a			=					self.feature_extraction_class()

     # Test not batched input
     __a			=					get_html_strings()[0]
     __a			=					feature_extractor(UpperCamelCase      )

     # fmt: off
     __a			=					[['sample document', 'Goog', 'This is one header', 'This is a another Header', 'Travel from', 'SFO to JFK', 'on May 2, 2015 at 2:00 pm. For details go to confirm.com', 'Traveler', 'name', 'is', 'John Doe']]
     __a			=					[['/html/head/title', '/html/body/a', '/html/body/h1', '/html/body/h2', '/html/body/p', '/html/body/p/p/b[1]', '/html/body/p/p/b[2]/i', '/html/body/p/p/div/h3', '/html/body/p/p/div/h3/b', '/html/body/p/p/div/h3', '/html/body/p/p/div/h3/p']]
     # fmt: on

     self.assertEqual(encoding.nodes      ,							UpperCamelCase      )
     self.assertEqual(encoding.xpaths      ,							UpperCamelCase      )

     # Test batched
     __a			=					get_html_strings()
     __a			=					feature_extractor(UpperCamelCase      )

     # fmt: off
     __a			=					expected_nodes + [['My First Heading', 'My first paragraph.']]
     __a			=					expected_xpaths + [['/html/body/h1', '/html/body/p']]

     self.assertEqual(len(encoding.nodes      )      ,							2      )
     self.assertEqual(len(encoding.xpaths      )      ,							2      )

     self.assertEqual(encoding.nodes      ,							UpperCamelCase      )
     self.assertEqual(encoding.xpaths      ,							UpperCamelCase      )









'''simple docstring'''


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mobilebert import MobileBertTokenizer


UpperCAmelCase_ 		=      logging.get_logger(__name__)

UpperCAmelCase_ 		=      {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

UpperCAmelCase_ 		=      {
    "vocab_file": {"mobilebert-uncased": "https://huggingface.co/google/mobilebert-uncased/resolve/main/vocab.txt"},
    "tokenizer_file": {
        "mobilebert-uncased": "https://huggingface.co/google/mobilebert-uncased/resolve/main/tokenizer.json"
    },
}

UpperCAmelCase_ 		=      {"mobilebert-uncased": 5_12}


UpperCAmelCase_ 		=      {}





class    __lowercase						(  __magic_name__ ):
   _a							=      VOCAB_FILES_NAMES
   _a							=      PRETRAINED_VOCAB_FILES_MAP
   _a							=      PRETRAINED_INIT_CONFIGURATION
   _a							=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _a							=      MobileBertTokenizer







   def __init__(	self      ,							UpperCamelCase=None      ,							UpperCamelCase=None      ,							UpperCamelCase=True      ,							UpperCamelCase="[UNK]"      ,							UpperCamelCase="[SEP]"      ,							UpperCamelCase="[PAD]"      ,							UpperCamelCase="[CLS]"      ,							UpperCamelCase="[MASK]"      ,							UpperCamelCase=True      ,							UpperCamelCase=None      ,							**UpperCamelCase      ,							)				->    List[Any]:
     super().__init__(
         UpperCamelCase      ,							tokenizer_file=UpperCamelCase      ,							do_lower_case=UpperCamelCase      ,							unk_token=UpperCamelCase      ,							sep_token=UpperCamelCase      ,							pad_token=UpperCamelCase      ,							cls_token=UpperCamelCase      ,							mask_token=UpperCamelCase      ,							tokenize_chinese_chars=UpperCamelCase      ,							strip_accents=UpperCamelCase      ,							**UpperCamelCase      ,							)

     __a			=					json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
     if (
         normalizer_state.get('lowercase'      ,							UpperCamelCase      ) != do_lower_case
         or normalizer_state.get('strip_accents'      ,							UpperCamelCase      ) != strip_accents
         or normalizer_state.get('handle_chinese_chars'      ,							UpperCamelCase      ) != tokenize_chinese_chars
     ):
       __a			=					getattr(UpperCamelCase      ,							normalizer_state.pop('type'      )      )
       __a			=					do_lower_case
       __a			=					strip_accents
       __a			=					tokenize_chinese_chars
       __a			=					normalizer_class(**UpperCamelCase      )

     __a			=					do_lower_case







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      ,							UpperCamelCase=None      )				->    Tuple:
     __a			=					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

     if token_ids_a:
       output += token_ids_a + [self.sep_token_id]

     return output







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      ,							UpperCamelCase = None      )				->    List[int]:
     __a			=					[self.sep_token_id]
     __a			=					[self.cls_token_id]
     if token_ids_a is None:
       return len(cls + token_ids_a + sep      ) * [0]
     return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]







   def 		UpperCamelCase__			(	self      ,							UpperCamelCase      ,							UpperCamelCase = None      )				->    Tuple[str]:
     __a			=					self._tokenizer.model.save(UpperCamelCase      ,							name=UpperCamelCase      )
     return tuple(UpperCamelCase      )



