

import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class        a__       (   UpperCAmelCase	,			unittest.TestCase       ):







  """simple docstring"""




  UpperCAmelCase__      : Dict			       =CodeGenTokenizer
  UpperCAmelCase__      : Union[str, Any]			       =CodeGenTokenizerFast
  UpperCAmelCase__      : Any			       =True
  UpperCAmelCase__      : Optional[int]			       ={"""add_prefix_space""": True}
  UpperCAmelCase__      : Optional[int]			       =False


  def  _lowercase      (			self    : List[Any]		)					->Any:
        """simple docstring"""







        super().setUp()

        # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
        SCREAMING_SNAKE_CASE :      Union[str, Any]       =   [
            """l""",
            """o""",
            """w""",
            """e""",
            """r""",
            """s""",
            """t""",
            """i""",
            """d""",
            """n""",
            """\u0120""",
            """\u0120l""",
            """\u0120n""",
            """\u0120lo""",
            """\u0120low""",
            """er""",
            """\u0120lowest""",
            """\u0120newer""",
            """\u0120wider""",
            """<unk>""",
            """<|endoftext|>""",
        ]
        SCREAMING_SNAKE_CASE :      Any       =   dict(zip(UpperCAmelCase__				,				range(len(UpperCAmelCase__		)		)		)		)
        SCREAMING_SNAKE_CASE :      List[Any]       =   ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
        SCREAMING_SNAKE_CASE :      str       =   {"""unk_token""": """<unk>"""}

        SCREAMING_SNAKE_CASE :      Union[str, Any]       =   os.path.join(self.tmpdirname				,				VOCAB_FILES_NAMES["""vocab_file"""]		)
        SCREAMING_SNAKE_CASE :      Any       =   os.path.join(self.tmpdirname				,				VOCAB_FILES_NAMES["""merges_file"""]		)
        with open(self.vocab_file				,				"""w"""				,				encoding="""utf-8"""		) as fp:
              fp.write(json.dumps(UpperCAmelCase__		) + """\n"""		)
        with open(self.merges_file				,				"""w"""				,				encoding="""utf-8"""		) as fp:
              fp.write("""\n""".join(UpperCAmelCase__		)		)


  def  _lowercase      (			self    : Optional[int]				,				**UpperCAmelCase__    : int		)					->Dict:
        """simple docstring"""







        kwargs.update(self.special_tokens_map		)
        return CodeGenTokenizer.from_pretrained(self.tmpdirname				,				**UpperCAmelCase__		)


  def  _lowercase      (			self    : List[str]				,				**UpperCAmelCase__    : List[str]		)					->Optional[Any]:
        """simple docstring"""







        kwargs.update(self.special_tokens_map		)
        return CodeGenTokenizerFast.from_pretrained(self.tmpdirname				,				**UpperCAmelCase__		)


  def  _lowercase      (			self    : Optional[int]				,				UpperCAmelCase__    : int		)					->str:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      Dict       =   """lower newer"""
        SCREAMING_SNAKE_CASE :      Optional[Any]       =   """lower newer"""
        return input_text, output_text


  def  _lowercase      (			self    : List[str]		)					->Union[str, Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      List[str]       =   CodeGenTokenizer(self.vocab_file				,				self.merges_file				,				**self.special_tokens_map		)
        SCREAMING_SNAKE_CASE :      List[Any]       =   """lower newer"""
        SCREAMING_SNAKE_CASE :      Optional[int]       =   ["""\u0120low""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
        SCREAMING_SNAKE_CASE :      List[Any]       =   tokenizer.tokenize(UpperCAmelCase__				,				add_prefix_space=UpperCAmelCase__		)
        self.assertListEqual(UpperCAmelCase__				,				UpperCAmelCase__		)

        SCREAMING_SNAKE_CASE :      Union[str, Any]       =   tokens + [tokenizer.unk_token]
        SCREAMING_SNAKE_CASE :      Tuple       =   [1_4, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase__		)				,				UpperCAmelCase__		)


  def  _lowercase      (			self    : int		)					->Optional[Any]:
        """simple docstring"""







        if not self.test_rust_tokenizer:
              return

        SCREAMING_SNAKE_CASE :      Optional[Any]       =   self.get_tokenizer()
        SCREAMING_SNAKE_CASE :      Optional[int]       =   self.get_rust_tokenizer(add_prefix_space=UpperCAmelCase__		)

        SCREAMING_SNAKE_CASE :      str       =   """lower newer"""

        # Testing tokenization
        SCREAMING_SNAKE_CASE :      str       =   tokenizer.tokenize(UpperCAmelCase__				,				add_prefix_space=UpperCAmelCase__		)
        SCREAMING_SNAKE_CASE :      Optional[Any]       =   rust_tokenizer.tokenize(UpperCAmelCase__		)
        self.assertListEqual(UpperCAmelCase__				,				UpperCAmelCase__		)

        # Testing conversion to ids without special tokens
        SCREAMING_SNAKE_CASE :      List[str]       =   tokenizer.encode(UpperCAmelCase__				,				add_special_tokens=UpperCAmelCase__				,				add_prefix_space=UpperCAmelCase__		)
        SCREAMING_SNAKE_CASE :      List[Any]       =   rust_tokenizer.encode(UpperCAmelCase__				,				add_special_tokens=UpperCAmelCase__		)
        self.assertListEqual(UpperCAmelCase__				,				UpperCAmelCase__		)

        # Testing conversion to ids with special tokens
        SCREAMING_SNAKE_CASE :      Any       =   self.get_rust_tokenizer(add_prefix_space=UpperCAmelCase__		)
        SCREAMING_SNAKE_CASE :      int       =   tokenizer.encode(UpperCAmelCase__				,				add_prefix_space=UpperCAmelCase__		)
        SCREAMING_SNAKE_CASE :      Any       =   rust_tokenizer.encode(UpperCAmelCase__		)
        self.assertListEqual(UpperCAmelCase__				,				UpperCAmelCase__		)

        # Testing the unknown token
        SCREAMING_SNAKE_CASE :      str       =   tokens + [rust_tokenizer.unk_token]
        SCREAMING_SNAKE_CASE :      List[Any]       =   [1_4, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
        self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(UpperCAmelCase__		)				,				UpperCAmelCase__		)


  def  _lowercase      (			self    : str				,				*UpperCAmelCase__    : Optional[Any]				,				**UpperCAmelCase__    : Optional[int]		)					->Tuple:
        """simple docstring"""







        pass


  def  _lowercase      (			self    : Union[str, Any]				,				UpperCAmelCase__    : List[Any]=1_5		)					->Any:
        """simple docstring"""







        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"		):
                    SCREAMING_SNAKE_CASE :      Union[str, Any]       =   self.rust_tokenizer_class.from_pretrained(UpperCAmelCase__				,				**UpperCAmelCase__		)

                    # Simple input
                    SCREAMING_SNAKE_CASE :      List[str]       =   """This is a simple input"""
                    SCREAMING_SNAKE_CASE :      int       =   ["""This is a simple input 1""", """This is a simple input 2"""]
                    SCREAMING_SNAKE_CASE :      Dict       =   ("""This is a simple input""", """This is a pair""")
                    SCREAMING_SNAKE_CASE :      Union[str, Any]       =   [
                        ("""This is a simple input 1""", """This is a simple input 2"""),
                        ("""This is a simple pair 1""", """This is a simple pair 2"""),
                    ]

                    # Simple input tests
                    self.assertRaises(UpperCAmelCase__				,				tokenizer_r.encode				,				UpperCAmelCase__				,				max_length=UpperCAmelCase__				,				padding="""max_length"""		)

                    # Simple input
                    self.assertRaises(UpperCAmelCase__				,				tokenizer_r.encode_plus				,				UpperCAmelCase__				,				max_length=UpperCAmelCase__				,				padding="""max_length"""		)

                    # Simple input
                    self.assertRaises(
                        UpperCAmelCase__				,				tokenizer_r.batch_encode_plus				,				UpperCAmelCase__				,				max_length=UpperCAmelCase__				,				padding="""max_length"""				,				)

                    # Pair input
                    self.assertRaises(UpperCAmelCase__				,				tokenizer_r.encode				,				UpperCAmelCase__				,				max_length=UpperCAmelCase__				,				padding="""max_length"""		)

                    # Pair input
                    self.assertRaises(UpperCAmelCase__				,				tokenizer_r.encode_plus				,				UpperCAmelCase__				,				max_length=UpperCAmelCase__				,				padding="""max_length"""		)

                    # Pair input
                    self.assertRaises(
                        UpperCAmelCase__				,				tokenizer_r.batch_encode_plus				,				UpperCAmelCase__				,				max_length=UpperCAmelCase__				,				padding="""max_length"""				,				)


  def  _lowercase      (			self    : Optional[int]		)					->Optional[int]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      Optional[int]       =   CodeGenTokenizer.from_pretrained(self.tmpdirname				,				pad_token="""<pad>"""		)

        # Simple input
        SCREAMING_SNAKE_CASE :      Tuple       =   """This is a simple input"""
        SCREAMING_SNAKE_CASE :      Optional[Any]       =   ["""This is a simple input looooooooong""", """This is a simple input"""]
        SCREAMING_SNAKE_CASE :      List[Any]       =   ("""This is a simple input""", """This is a pair""")
        SCREAMING_SNAKE_CASE :      Dict       =   [
            ("""This is a simple input loooooong""", """This is a simple input"""),
            ("""This is a simple pair loooooong""", """This is a simple pair"""),
        ]

        SCREAMING_SNAKE_CASE :      List[str]       =   tokenizer.pad_token_id

        SCREAMING_SNAKE_CASE :      str       =   tokenizer(UpperCAmelCase__				,				padding="""max_length"""				,				max_length=3_0				,				return_tensors="""np"""		)
        SCREAMING_SNAKE_CASE :      List[str]       =   tokenizer(UpperCAmelCase__				,				padding=UpperCAmelCase__				,				truncate=UpperCAmelCase__				,				return_tensors="""np"""		)
        SCREAMING_SNAKE_CASE :      Optional[int]       =   tokenizer(*UpperCAmelCase__				,				padding="""max_length"""				,				max_length=6_0				,				return_tensors="""np"""		)
        SCREAMING_SNAKE_CASE :      Optional[int]       =   tokenizer(UpperCAmelCase__				,				padding=UpperCAmelCase__				,				truncate=UpperCAmelCase__				,				return_tensors="""np"""		)

        # s
        # test single string max_length padding
        self.assertEqual(out_s["""input_ids"""].shape[-1]				,				3_0		)
        self.assertTrue(pad_token_id in out_s["""input_ids"""]		)
        self.assertTrue(0 in out_s["""attention_mask"""]		)

        # s2
        # test automatic padding
        self.assertEqual(out_sa["""input_ids"""].shape[-1]				,				3_3		)
        # long slice doesn't have padding
        self.assertFalse(pad_token_id in out_sa["""input_ids"""][0]		)
        self.assertFalse(0 in out_sa["""attention_mask"""][0]		)
        # short slice does have padding
        self.assertTrue(pad_token_id in out_sa["""input_ids"""][1]		)
        self.assertTrue(0 in out_sa["""attention_mask"""][1]		)

        # p
        # test single pair max_length padding
        self.assertEqual(out_p["""input_ids"""].shape[-1]				,				6_0		)
        self.assertTrue(pad_token_id in out_p["""input_ids"""]		)
        self.assertTrue(0 in out_p["""attention_mask"""]		)

        # p2
        # test automatic padding pair
        self.assertEqual(out_pa["""input_ids"""].shape[-1]				,				5_2		)
        # long slice pair doesn't have padding
        self.assertFalse(pad_token_id in out_pa["""input_ids"""][0]		)
        self.assertFalse(0 in out_pa["""attention_mask"""][0]		)
        # short slice pair does have padding
        self.assertTrue(pad_token_id in out_pa["""input_ids"""][1]		)
        self.assertTrue(0 in out_pa["""attention_mask"""][1]		)


  def  _lowercase      (			self    : Any		)					->Union[str, Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      Optional[Any]       =   """$$$"""
        SCREAMING_SNAKE_CASE :      Tuple       =   CodeGenTokenizer.from_pretrained(self.tmpdirname				,				bos_token=UpperCAmelCase__				,				add_bos_token=UpperCAmelCase__		)

        SCREAMING_SNAKE_CASE :      int       =   """This is a simple input"""
        SCREAMING_SNAKE_CASE :      Dict       =   ["""This is a simple input 1""", """This is a simple input 2"""]

        SCREAMING_SNAKE_CASE :      Optional[int]       =   tokenizer.bos_token_id

        SCREAMING_SNAKE_CASE :      Dict       =   tokenizer(UpperCAmelCase__		)
        SCREAMING_SNAKE_CASE :      Union[str, Any]       =   tokenizer(UpperCAmelCase__		)

        self.assertEqual(out_s.input_ids[0]				,				UpperCAmelCase__		)
        self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids		)		)

        SCREAMING_SNAKE_CASE :      Any       =   tokenizer.decode(out_s.input_ids		)
        SCREAMING_SNAKE_CASE :      Tuple       =   tokenizer.batch_decode(out_sa.input_ids		)

        self.assertEqual(decode_s.split()[0]				,				UpperCAmelCase__		)
        self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa		)		)


  @slow
  def  _lowercase      (			self    : Optional[Any]		)					->Optional[int]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE :      Union[str, Any]       =   CodeGenTokenizer.from_pretrained("""Salesforce/codegen-350M-mono"""		)

        SCREAMING_SNAKE_CASE :      Optional[int]       =   """\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"""
        SCREAMING_SNAKE_CASE :      Optional[int]       =   """\nif len_a > len_b:      result = a\nelse:      result = b"""

        SCREAMING_SNAKE_CASE :      Dict       =   tokenizer.encode(UpperCAmelCase__		)
        SCREAMING_SNAKE_CASE :      Union[str, Any]       =   ["""^#""", re.escape("""<|endoftext|>"""		), """^'''""", """^\"\"\"""", """\n\n\n"""]
        SCREAMING_SNAKE_CASE :      Optional[int]       =   tokenizer.decode(UpperCAmelCase__				,				truncate_before_pattern=UpperCAmelCase__		)
        self.assertEqual(UpperCAmelCase__				,				UpperCAmelCase__		)




  def  _lowercase      (			self    : List[str]		)					->Dict:
        """simple docstring"""







        pass







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
    is_vision_available,
)


UpperCAmelCase__ :     List[Any]				    =    {"""configuration_vit""": ["""VIT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """ViTConfig""", """ViTOnnxConfig"""]}

try:
  if not is_vision_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCAmelCase__ :     Union[str, Any]				    =    ["""ViTFeatureExtractor"""]
  UpperCAmelCase__ :     List[Any]				    =    ["""ViTImageProcessor"""]

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCAmelCase__ :     List[Any]				    =    [
      """VIT_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """ViTForImageClassification""",
      """ViTForMaskedImageModeling""",
      """ViTModel""",
      """ViTPreTrainedModel""",
  ]

try:
  if not is_tf_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCAmelCase__ :     Tuple				    =    [
      """TFViTForImageClassification""",
      """TFViTModel""",
      """TFViTPreTrainedModel""",
  ]

try:
  if not is_flax_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCAmelCase__ :     str				    =    [
      """FlaxViTForImageClassification""",
      """FlaxViTModel""",
      """FlaxViTPreTrainedModel""",
  ]

if TYPE_CHECKING:
  from .configuration_vit import VIT_PRETRAINED_CONFIG_ARCHIVE_MAP, ViTConfig, ViTOnnxConfig

  try:
    if not is_vision_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .feature_extraction_vit import ViTFeatureExtractor
    from .image_processing_vit import ViTImageProcessor

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_vit import (
        VIT_PRETRAINED_MODEL_ARCHIVE_LIST,
        ViTForImageClassification,
        ViTForMaskedImageModeling,
        ViTModel,
        ViTPreTrainedModel,
    )

  try:
    if not is_tf_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_tf_vit import TFViTForImageClassification, TFViTModel, TFViTPreTrainedModel

  try:
    if not is_flax_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_flax_vit import FlaxViTForImageClassification, FlaxViTModel, FlaxViTPreTrainedModel


else:
  import sys

  UpperCAmelCase__ :     Optional[Any]				    =    _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)




