





from collections import defaultdict
from typing import Optional

from ..image_utils import load_image
from ..utils import (
    add_end_docstrings,
    is_torch_available,
    logging,
    requires_backends,
)
from .base import PIPELINE_INIT_ARGS, ChunkPipeline


if is_torch_available():
	import torch

	from ..models.auto.modeling_auto import MODEL_FOR_MASK_GENERATION_MAPPING

_lowerCamelCase   				=			logging.get_logger(__name__)




@add_end_docstrings(UpperCamelCase__    )
class   UpperCamelCase_       (     UpperCamelCase__    ):


							def __init__(							self    :List[Any]							,				**__A    :Optional[Any]		)   ->   int:



									"""simple docstring"""



									super().__init__(**__A		)
									requires_backends(self							,				"""vision"""		)
									requires_backends(self							,				"""torch"""		)

									if self.framework != "pt":
											raise ValueError(f'''The {self.__class__} is only available in PyTorch.'''		)

									self.check_model_type(__A		)


							def      _snake_case  (							self    :str							,				**__A    :Optional[Any]		)   ->   str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      {}
									SCREAMING_SNAKE_CASE__			=      {}
									SCREAMING_SNAKE_CASE__			=      {}
									# preprocess args
									if "points_per_batch" in kwargs:
											SCREAMING_SNAKE_CASE__			=      kwargs["""points_per_batch"""]
									if "points_per_crop" in kwargs:
											SCREAMING_SNAKE_CASE__			=      kwargs["""points_per_crop"""]
									if "crops_n_layers" in kwargs:
											SCREAMING_SNAKE_CASE__			=      kwargs["""crops_n_layers"""]
									if "crop_overlap_ratio" in kwargs:
											SCREAMING_SNAKE_CASE__			=      kwargs["""crop_overlap_ratio"""]
									if "crop_n_points_downscale_factor" in kwargs:
											SCREAMING_SNAKE_CASE__			=      kwargs["""crop_n_points_downscale_factor"""]
									# postprocess args
									if "pred_iou_thresh" in kwargs:
											SCREAMING_SNAKE_CASE__			=      kwargs["""pred_iou_thresh"""]
									if "stability_score_offset" in kwargs:
											SCREAMING_SNAKE_CASE__			=      kwargs["""stability_score_offset"""]
									if "mask_threshold" in kwargs:
											SCREAMING_SNAKE_CASE__			=      kwargs["""mask_threshold"""]
									if "stability_score_thresh" in kwargs:
											SCREAMING_SNAKE_CASE__			=      kwargs["""stability_score_thresh"""]
									if "crops_nms_thresh" in kwargs:
											SCREAMING_SNAKE_CASE__			=      kwargs["""crops_nms_thresh"""]
									if "output_rle_mask" in kwargs:
											SCREAMING_SNAKE_CASE__			=      kwargs["""output_rle_mask"""]
									if "output_bboxes_mask" in kwargs:
											SCREAMING_SNAKE_CASE__			=      kwargs["""output_bboxes_mask"""]
									return preprocess_kwargs, forward_params, postprocess_kwargs


							def __call__(							self    :List[Any]							,				__A    :List[str]							,				*__A    :Tuple							,				__A    :Optional[int]=None							,				__A    :str=None							,				**__A    :Union[str, Any]		)   ->   List[Any]:



									"""simple docstring"""



									return super().__call__(__A							,				*__A							,				num_workers=__A							,				batch_size=__A							,				**__A		)


							def      _snake_case  (							self    :Any							,				__A    :int							,				__A    :Optional[int]=64							,				__A    :int = 0							,				__A    :float = 512 / 1500							,				__A    :Optional[int] = 32							,				__A    :Optional[int] = 1							,				)   ->   Tuple:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      load_image(__A		)
									SCREAMING_SNAKE_CASE__			=      self.image_processor.size["""longest_edge"""]
									SCREAMING_SNAKE_CASE__	,						SCREAMING_SNAKE_CASE__	,						SCREAMING_SNAKE_CASE__	,						SCREAMING_SNAKE_CASE__			=      self.image_processor.generate_crop_boxes(
									    __A							,				__A							,				__A							,				__A							,				__A							,				__A		)
									SCREAMING_SNAKE_CASE__			=      self.image_processor(images=__A							,				return_tensors="""pt"""		)

									with self.device_placement():
											if self.framework == "pt":
													SCREAMING_SNAKE_CASE__			=      self.get_inference_context()
													with inference_context():
															SCREAMING_SNAKE_CASE__			=      self._ensure_tensor_on_device(__A							,				device=self.device		)
															SCREAMING_SNAKE_CASE__			=      self.model.get_image_embeddings(model_inputs.pop("""pixel_values"""		)		)
															SCREAMING_SNAKE_CASE__			=      image_embeddings

									SCREAMING_SNAKE_CASE__			=      grid_points.shape[1]
									SCREAMING_SNAKE_CASE__			=      points_per_batch if points_per_batch is not None else n_points

									if points_per_batch <= 0:
											raise ValueError(
											    """Cannot have points_per_batch<=0. Must be >=1 to returned batched outputs. """
											    """To return all points at once, set points_per_batch to None"""		)

									for i in range(0							,				__A							,				__A		):
											SCREAMING_SNAKE_CASE__			=      grid_points[:, i : i + points_per_batch, :, :]
											SCREAMING_SNAKE_CASE__			=      input_labels[:, i : i + points_per_batch]
											SCREAMING_SNAKE_CASE__			=      i == n_points - points_per_batch
											yield {
											    "input_points": batched_points,
											    "input_labels": labels,
											    "input_boxes": crop_boxes,
											    "is_last": is_last,
											    **model_inputs,
											}


							def      _snake_case  (							self    :Optional[Any]							,				__A    :str							,				__A    :List[Any]=0.8_8							,				__A    :Tuple=0.9_5							,				__A    :Optional[Any]=0							,				__A    :List[Any]=1							,				)   ->   int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      model_inputs.pop("""input_boxes"""		)
									SCREAMING_SNAKE_CASE__			=      model_inputs.pop("""is_last"""		)
									SCREAMING_SNAKE_CASE__			=      model_inputs.pop("""original_sizes"""		).tolist()
									SCREAMING_SNAKE_CASE__			=      model_inputs.pop("""reshaped_input_sizes"""		).tolist()

									SCREAMING_SNAKE_CASE__			=      self.model(**__A		)

									# post processing happens here in order to avoid CPU GPU copies of ALL the masks
									SCREAMING_SNAKE_CASE__			=      model_outputs["""pred_masks"""]
									SCREAMING_SNAKE_CASE__			=      self.image_processor.post_process_masks(
									    __A							,				__A							,				__A							,				__A							,				binarize=__A		)
									SCREAMING_SNAKE_CASE__			=      model_outputs["""iou_scores"""]
									SCREAMING_SNAKE_CASE__	,						SCREAMING_SNAKE_CASE__	,						SCREAMING_SNAKE_CASE__			=      self.image_processor.filter_masks(
									    masks[0]							,				iou_scores[0]							,				original_sizes[0]							,				input_boxes[0]							,				__A							,				__A							,				__A							,				__A							,				)
									return {
									    "masks": masks,
									    "is_last": is_last,
									    "boxes": boxes,
									    "iou_scores": iou_scores,
									}


							def      _snake_case  (							self    :Union[str, Any]							,				__A    :Union[str, Any]							,				__A    :Any=False							,				__A    :Any=False							,				__A    :str=0.7							,				)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      []
									SCREAMING_SNAKE_CASE__			=      []
									SCREAMING_SNAKE_CASE__			=      []
									for model_output in model_outputs:
											all_scores.append(model_output.pop("""iou_scores"""		)		)
											all_masks.extend(model_output.pop("""masks"""		)		)
											all_boxes.append(model_output.pop("""boxes"""		)		)

									SCREAMING_SNAKE_CASE__			=      torch.cat(__A		)
									SCREAMING_SNAKE_CASE__			=      torch.cat(__A		)
									SCREAMING_SNAKE_CASE__	,						SCREAMING_SNAKE_CASE__	,						SCREAMING_SNAKE_CASE__	,						SCREAMING_SNAKE_CASE__			=      self.image_processor.post_process_for_mask_generation(
									    __A							,				__A							,				__A							,				__A		)

									SCREAMING_SNAKE_CASE__			=      defaultdict(__A		)
									for output in model_outputs:
											for k, v in output.items():
													extra[k].append(__A		)

									SCREAMING_SNAKE_CASE__			=      {}
									if output_rle_mask:
											SCREAMING_SNAKE_CASE__			=      rle_mask

									if output_bboxes_mask:
											SCREAMING_SNAKE_CASE__			=      bounding_boxes

									return {"masks": output_masks, "scores": iou_scores, **optional, **extra}











import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class   UpperCamelCase_       (     UpperCamelCase__				,  unittest.TestCase    ):
							lowerCamelCase_           =					RoCBertTokenizer
							lowerCamelCase_           =					None
							lowerCamelCase_           =					False
							lowerCamelCase_           =					True
							lowerCamelCase_           =					filter_non_english


							def      _snake_case  (							self    :List[Any]		)   ->   List[Any]:



									"""simple docstring"""



									super().setUp()

									SCREAMING_SNAKE_CASE__			=      ["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """你""", """好""", """是""", """谁""", """a""", """b""", """c""", """d"""]
									SCREAMING_SNAKE_CASE__			=      {}
									SCREAMING_SNAKE_CASE__			=      {}
									for i, value in enumerate(__A		):
											SCREAMING_SNAKE_CASE__			=      i
											SCREAMING_SNAKE_CASE__			=      i
									SCREAMING_SNAKE_CASE__			=      os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""vocab_file"""]		)
									SCREAMING_SNAKE_CASE__			=      os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""word_shape_file"""]		)
									SCREAMING_SNAKE_CASE__			=      os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""word_pronunciation_file"""]		)
									with open(self.vocab_file							,				"""w"""							,				encoding="""utf-8"""		) as vocab_writer:
											vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]		)		)
									with open(self.word_shape_file							,				"""w"""							,				encoding="""utf-8"""		) as word_shape_writer:
											json.dump(__A							,				__A							,				ensure_ascii=__A		)
									with open(self.word_pronunciation_file							,				"""w"""							,				encoding="""utf-8"""		) as word_pronunciation_writer:
											json.dump(__A							,				__A							,				ensure_ascii=__A		)


							def      _snake_case  (							self    :List[Any]		)   ->   Dict:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.tokenizer_class(self.vocab_file							,				self.word_shape_file							,				self.word_pronunciation_file		)

									SCREAMING_SNAKE_CASE__			=      tokenizer.tokenize("""你好[SEP]你是谁"""		)
									self.assertListEqual(__A							,				["""你""", """好""", """[SEP]""", """你""", """是""", """谁"""]		)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(__A		)							,				[5, 6, 2, 5, 7, 8]		)
									self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(__A		)							,				[5, 6, 2, 5, 7, 8]		)
									self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(__A		)							,				[5, 6, 2, 5, 7, 8]		)


							def      _snake_case  (							self    :List[Any]		)   ->   Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer()

									self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""		)							,				["""ah""", """\u535A""", """\u63A8""", """zz"""]		)


							def      _snake_case  (							self    :List[str]		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """		)							,				["""hello""", """!""", """how""", """are""", """you""", """?"""]		)
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""		)							,				["""hello"""]		)


							def      _snake_case  (							self    :str		)   ->   str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A							,				strip_accents=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """		)							,				["""hällo""", """!""", """how""", """are""", """you""", """?"""]		)
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""		)							,				["""h\u00E9llo"""]		)


							def      _snake_case  (							self    :Any		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A							,				strip_accents=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """		)							,				["""hallo""", """!""", """how""", """are""", """you""", """?"""]		)
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""		)							,				["""hello"""]		)


							def      _snake_case  (							self    :List[str]		)   ->   int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """		)							,				["""hallo""", """!""", """how""", """are""", """you""", """?"""]		)
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""		)							,				["""hello"""]		)


							def      _snake_case  (							self    :Union[str, Any]		)   ->   Optional[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """		)							,				["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]		)


							def      _snake_case  (							self    :int		)   ->   Dict:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A							,				strip_accents=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """		)							,				["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]		)


							def      _snake_case  (							self    :Union[str, Any]		)   ->   List[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A							,				strip_accents=__A		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """		)							,				["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]		)


							def      _snake_case  (							self    :List[Any]		)   ->   str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      RoCBertBasicTokenizer(do_lower_case=__A							,				never_split=["""[UNK]"""]		)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""		)							,				["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]		)


							def      _snake_case  (							self    :Any		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

									SCREAMING_SNAKE_CASE__			=      {}
									for i, token in enumerate(__A		):
											SCREAMING_SNAKE_CASE__			=      i
									SCREAMING_SNAKE_CASE__			=      RoCBertWordpieceTokenizer(vocab=__A							,				unk_token="""[UNK]"""		)

									self.assertListEqual(tokenizer.tokenize(""""""		)							,				[]		)

									self.assertListEqual(tokenizer.tokenize("""unwanted running"""		)							,				["""un""", """##want""", """##ed""", """runn""", """##ing"""]		)

									self.assertListEqual(tokenizer.tokenize("""unwantedX running"""		)							,				["""[UNK]""", """runn""", """##ing"""]		)


							def      _snake_case  (							self    :Any		)   ->   str:



									"""simple docstring"""



									self.assertTrue(_is_whitespace(""" """		)		)
									self.assertTrue(_is_whitespace("""\t"""		)		)
									self.assertTrue(_is_whitespace("""\r"""		)		)
									self.assertTrue(_is_whitespace("""\n"""		)		)
									self.assertTrue(_is_whitespace("""\u00A0"""		)		)

									self.assertFalse(_is_whitespace("""A"""		)		)
									self.assertFalse(_is_whitespace("""-"""		)		)


							def      _snake_case  (							self    :int		)   ->   str:



									"""simple docstring"""



									self.assertTrue(_is_control("""\u0005"""		)		)

									self.assertFalse(_is_control("""A"""		)		)
									self.assertFalse(_is_control(""" """		)		)
									self.assertFalse(_is_control("""\t"""		)		)
									self.assertFalse(_is_control("""\r"""		)		)


							def      _snake_case  (							self    :List[str]		)   ->   List[str]:



									"""simple docstring"""



									self.assertTrue(_is_punctuation("""-"""		)		)
									self.assertTrue(_is_punctuation("""$"""		)		)
									self.assertTrue(_is_punctuation("""`"""		)		)
									self.assertTrue(_is_punctuation("""."""		)		)

									self.assertFalse(_is_punctuation("""A"""		)		)
									self.assertFalse(_is_punctuation(""" """		)		)


							def      _snake_case  (							self    :str		)   ->   str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.get_tokenizer()

									# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
									self.assertListEqual([tokenizer.tokenize(__A		) for t in ["""Test""", """\xad""", """test"""]]							,				[["""[UNK]"""], [], ["""[UNK]"""]]		)

									if self.test_rust_tokenizer:
											SCREAMING_SNAKE_CASE__			=      self.get_rust_tokenizer()
											self.assertListEqual(
											    [rust_tokenizer.tokenize(__A		) for t in ["""Test""", """\xad""", """test"""]]							,				[["""[UNK]"""], [], ["""[UNK]"""]]		)


							def      _snake_case  (							self    :int		)   ->   Any:



									"""simple docstring"""



									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
													SCREAMING_SNAKE_CASE__			=      self.rust_tokenizer_class.from_pretrained(__A							,				**__A		)

													SCREAMING_SNAKE_CASE__			=      f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
													SCREAMING_SNAKE_CASE__			=      tokenizer_r.encode_plus(
													    __A							,				return_attention_mask=__A							,				return_token_type_ids=__A							,				return_offsets_mapping=__A							,				add_special_tokens=__A							,				)

													SCREAMING_SNAKE_CASE__			=      tokenizer_r.do_lower_case if hasattr(__A							,				"""do_lower_case"""		) else False
													SCREAMING_SNAKE_CASE__			=      (
													    [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), """A"""),
													        ((1, 2), ""","""),
													        ((3, 5), """na"""),
													        ((5, 6), """##ï"""),
													        ((6, 8), """##ve"""),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), """Allen"""),
													        ((21, 23), """##NL"""),
													        ((23, 24), """##P"""),
													        ((25, 33), """sentence"""),
													        ((33, 34), """."""),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													    if not do_lower_case
													    else [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), """a"""),
													        ((1, 2), ""","""),
													        ((3, 8), """naive"""),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), """allen"""),
													        ((21, 23), """##nl"""),
													        ((23, 24), """##p"""),
													        ((25, 33), """sentence"""),
													        ((33, 34), """."""),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													)

													self.assertEqual(
													    [e[1] for e in expected_results]							,				tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]		)		)
													self.assertEqual([e[0] for e in expected_results]							,				tokens["""offset_mapping"""]		)


							def      _snake_case  (							self    :Optional[int]		)   ->   Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      ["""的""", """人""", """有"""]
									SCREAMING_SNAKE_CASE__			=      """""".join(__A		)
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
													SCREAMING_SNAKE_CASE__			=      True
													SCREAMING_SNAKE_CASE__			=      self.tokenizer_class.from_pretrained(__A							,				**__A		)
													SCREAMING_SNAKE_CASE__			=      self.rust_tokenizer_class.from_pretrained(__A							,				**__A		)

													SCREAMING_SNAKE_CASE__			=      tokenizer_p.encode(__A							,				add_special_tokens=__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer_r.encode(__A							,				add_special_tokens=__A		)

													SCREAMING_SNAKE_CASE__			=      tokenizer_r.convert_ids_to_tokens(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer_p.convert_ids_to_tokens(__A		)

													# it is expected that each Chinese character is not preceded by "##"
													self.assertListEqual(__A							,				__A		)
													self.assertListEqual(__A							,				__A		)

													SCREAMING_SNAKE_CASE__			=      False
													SCREAMING_SNAKE_CASE__			=      self.rust_tokenizer_class.from_pretrained(__A							,				**__A		)
													SCREAMING_SNAKE_CASE__			=      self.tokenizer_class.from_pretrained(__A							,				**__A		)

													SCREAMING_SNAKE_CASE__			=      tokenizer_r.encode(__A							,				add_special_tokens=__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer_p.encode(__A							,				add_special_tokens=__A		)

													SCREAMING_SNAKE_CASE__			=      tokenizer_r.convert_ids_to_tokens(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer_p.convert_ids_to_tokens(__A		)

													# it is expected that only the first Chinese character is not preceded by "##".
													SCREAMING_SNAKE_CASE__			=      [
													    f'''##{token}''' if idx != 0 else token for idx, token in enumerate(__A		)
													]
													self.assertListEqual(__A							,				__A		)
													self.assertListEqual(__A							,				__A		)


							@slow
							def      _snake_case  (							self    :Union[str, Any]		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.tokenizer_class(self.vocab_file							,				self.word_shape_file							,				self.word_pronunciation_file		)

									SCREAMING_SNAKE_CASE__			=      tokenizer.encode("""你好"""							,				add_special_tokens=__A		)
									SCREAMING_SNAKE_CASE__			=      tokenizer.encode("""你是谁"""							,				add_special_tokens=__A		)

									SCREAMING_SNAKE_CASE__			=      tokenizer.build_inputs_with_special_tokens(__A		)
									SCREAMING_SNAKE_CASE__			=      tokenizer.build_inputs_with_special_tokens(__A							,				__A		)

									assert encoded_sentence == [1] + text + [2]
									assert encoded_pair == [1] + text + [2] + text_a + [2]


							def      _snake_case  (							self    :List[str]		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.get_tokenizers(do_lower_case=__A		)
									for tokenizer in tokenizers:
											with self.subTest(f'''{tokenizer.__class__.__name__}'''		):
													SCREAMING_SNAKE_CASE__			=      """你好，你是谁"""
													SCREAMING_SNAKE_CASE__			=      tokenizer.tokenize(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer.convert_tokens_to_ids(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer.convert_tokens_to_shape_ids(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer.convert_tokens_to_pronunciation_ids(__A		)
													SCREAMING_SNAKE_CASE__			=      tokenizer.prepare_for_model(
													    __A							,				__A							,				__A							,				add_special_tokens=__A		)

													SCREAMING_SNAKE_CASE__			=      tokenizer.encode_plus(__A							,				add_special_tokens=__A		)

													self.assertEqual(__A							,				__A		)