



"""simple docstring"""

import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class  _UpperCAmelCase (    lowerCAmelCase__					,  unittest.TestCase):
 _lowerCAmelCase     :    Optional[Any]	      =	CodeGenTokenizer
 _lowerCAmelCase     :    List[str]	      =	CodeGenTokenizerFast
 _lowerCAmelCase     :    List[str]	      =	True
 _lowerCAmelCase     :    List[str]	      =	{"""add_prefix_space""": True}
 _lowerCAmelCase     :    Tuple	      =	False

 def 		_snake_case      (  self	: List[Any]       ):
     super().setUp()

     # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
     snake_case_				:  Any       =			[
         '''l''',
         '''o''',
         '''w''',
         '''e''',
         '''r''',
         '''s''',
         '''t''',
         '''i''',
         '''d''',
         '''n''',
         '''\u0120''',
         '''\u0120l''',
         '''\u0120n''',
         '''\u0120lo''',
         '''\u0120low''',
         '''er''',
         '''\u0120lowest''',
         '''\u0120newer''',
         '''\u0120wider''',
         '''<unk>''',
         '''<|endoftext|>''',
     ]
     snake_case_				:  Union[str, Any]       =			dict(zip(lowercase_						,  range(len(lowercase_       )       )       )       )
     snake_case_				:  Optional[int]       =			['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
     snake_case_				:  int       =			{'''unk_token''': '''<unk>'''}

     snake_case_				:  List[str]       =			os.path.join(self.tmpdirname						,  VOCAB_FILES_NAMES['''vocab_file''']       )
     snake_case_				:  Optional[int]       =			os.path.join(self.tmpdirname						,  VOCAB_FILES_NAMES['''merges_file''']       )
     with open(self.vocab_file						,  '''w'''						,  encoding='''utf-8'''       ) as fp:
         fp.write(json.dumps(lowercase_       ) + '''\n'''       )
     with open(self.merges_file						,  '''w'''						,  encoding='''utf-8'''       ) as fp:
         fp.write('''\n'''.join(lowercase_       )       )

 def 		_snake_case      (  self	: Optional[Any]						,  **lowercase_	: str       ):
     kwargs.update(self.special_tokens_map       )
     return CodeGenTokenizer.from_pretrained(self.tmpdirname						,  **lowercase_       )

 def 		_snake_case      (  self	: int						,  **lowercase_	: str       ):
     kwargs.update(self.special_tokens_map       )
     return CodeGenTokenizerFast.from_pretrained(self.tmpdirname						,  **lowercase_       )

 def 		_snake_case      (  self	: int						,  lowercase_	: Any       ):
     snake_case_				:  str       =			'''lower newer'''
     snake_case_				:  str       =			'''lower newer'''
     return input_text, output_text

 def 		_snake_case      (  self	: Dict       ):
     snake_case_				:  List[str]       =			CodeGenTokenizer(self.vocab_file						,  self.merges_file						,  **self.special_tokens_map       )
     snake_case_				:  List[Any]       =			'''lower newer'''
     snake_case_				:  int       =			['''\u0120low''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
     snake_case_				:  Optional[Any]       =			tokenizer.tokenize(lowercase_						,  add_prefix_space=lowercase_       )
     self.assertListEqual(lowercase_						,  lowercase_       )

     snake_case_				:  Optional[int]       =			tokens + [tokenizer.unk_token]
     snake_case_				:  Tuple       =			[14, 15, 10, 9, 3, 2, 15, 19]
     self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase_       )						,  lowercase_       )

 def 		_snake_case      (  self	: Optional[int]       ):
     if not self.test_rust_tokenizer:
         return

     snake_case_				:  Dict       =			self.get_tokenizer()
     snake_case_				:  str       =			self.get_rust_tokenizer(add_prefix_space=lowercase_       )

     snake_case_				:  Union[str, Any]       =			'''lower newer'''

     # Testing tokenization
     snake_case_				:  Tuple       =			tokenizer.tokenize(lowercase_						,  add_prefix_space=lowercase_       )
     snake_case_				:  Union[str, Any]       =			rust_tokenizer.tokenize(lowercase_       )
     self.assertListEqual(lowercase_						,  lowercase_       )

     # Testing conversion to ids without special tokens
     snake_case_				:  Optional[Any]       =			tokenizer.encode(lowercase_						,  add_special_tokens=lowercase_						,  add_prefix_space=lowercase_       )
     snake_case_				:  str       =			rust_tokenizer.encode(lowercase_						,  add_special_tokens=lowercase_       )
     self.assertListEqual(lowercase_						,  lowercase_       )

     # Testing conversion to ids with special tokens
     snake_case_				:  Dict       =			self.get_rust_tokenizer(add_prefix_space=lowercase_       )
     snake_case_				:  Union[str, Any]       =			tokenizer.encode(lowercase_						,  add_prefix_space=lowercase_       )
     snake_case_				:  List[Any]       =			rust_tokenizer.encode(lowercase_       )
     self.assertListEqual(lowercase_						,  lowercase_       )

     # Testing the unknown token
     snake_case_				:  Dict       =			tokens + [rust_tokenizer.unk_token]
     snake_case_				:  int       =			[14, 15, 10, 9, 3, 2, 15, 19]
     self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(lowercase_       )						,  lowercase_       )

 def 		_snake_case      (  self	: Any						,  *lowercase_	: Any						,  **lowercase_	: Optional[Any]       ):
     # It's very difficult to mix/test pretokenization with byte-level
     # And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
     pass

 def 		_snake_case      (  self	: Tuple						,  lowercase_	: Dict=15       ):
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"       ):
             snake_case_				:  List[str]       =			self.rust_tokenizer_class.from_pretrained(lowercase_						,  **lowercase_       )

             # Simple input
             snake_case_				:  Any       =			'''This is a simple input'''
             snake_case_				:  Tuple       =			['''This is a simple input 1''', '''This is a simple input 2''']
             snake_case_				:  Optional[Any]       =			('''This is a simple input''', '''This is a pair''')
             snake_case_				:  Optional[Any]       =			[
                 ('''This is a simple input 1''', '''This is a simple input 2'''),
                 ('''This is a simple pair 1''', '''This is a simple pair 2'''),
             ]

             # Simple input tests
             self.assertRaises(lowercase_						,  tokenizer_r.encode						,  lowercase_						,  max_length=lowercase_						,  padding='''max_length'''       )

             # Simple input
             self.assertRaises(lowercase_						,  tokenizer_r.encode_plus						,  lowercase_						,  max_length=lowercase_						,  padding='''max_length'''       )

             # Simple input
             self.assertRaises(
                 lowercase_						,  tokenizer_r.batch_encode_plus						,  lowercase_						,  max_length=lowercase_						,  padding='''max_length'''						,  )

             # Pair input
             self.assertRaises(lowercase_						,  tokenizer_r.encode						,  lowercase_						,  max_length=lowercase_						,  padding='''max_length'''       )

             # Pair input
             self.assertRaises(lowercase_						,  tokenizer_r.encode_plus						,  lowercase_						,  max_length=lowercase_						,  padding='''max_length'''       )

             # Pair input
             self.assertRaises(
                 lowercase_						,  tokenizer_r.batch_encode_plus						,  lowercase_						,  max_length=lowercase_						,  padding='''max_length'''						,  )

 def 		_snake_case      (  self	: Union[str, Any]       ):
     snake_case_				:  int       =			CodeGenTokenizer.from_pretrained(self.tmpdirname						,  pad_token='''<pad>'''       )

     # Simple input
     snake_case_				:  str       =			'''This is a simple input'''
     snake_case_				:  List[Any]       =			['''This is a simple input looooooooong''', '''This is a simple input''']
     snake_case_				:  Dict       =			('''This is a simple input''', '''This is a pair''')
     snake_case_				:  Union[str, Any]       =			[
         ('''This is a simple input loooooong''', '''This is a simple input'''),
         ('''This is a simple pair loooooong''', '''This is a simple pair'''),
     ]

     snake_case_				:  List[Any]       =			tokenizer.pad_token_id

     snake_case_				:  List[Any]       =			tokenizer(lowercase_						,  padding='''max_length'''						,  max_length=30						,  return_tensors='''np'''       )
     snake_case_				:  Tuple       =			tokenizer(lowercase_						,  padding=lowercase_						,  truncate=lowercase_						,  return_tensors='''np'''       )
     snake_case_				:  Union[str, Any]       =			tokenizer(*lowercase_						,  padding='''max_length'''						,  max_length=60						,  return_tensors='''np'''       )
     snake_case_				:  Optional[Any]       =			tokenizer(lowercase_						,  padding=lowercase_						,  truncate=lowercase_						,  return_tensors='''np'''       )

     # s
     # test single string max_length padding
     self.assertEqual(out_s['''input_ids'''].shape[-1]						,  30       )
     self.assertTrue(pad_token_id in out_s['''input_ids''']       )
     self.assertTrue(0 in out_s['''attention_mask''']       )

     # s2
     # test automatic padding
     self.assertEqual(out_sa['''input_ids'''].shape[-1]						,  33       )
     # long slice doesn't have padding
     self.assertFalse(pad_token_id in out_sa['''input_ids'''][0]       )
     self.assertFalse(0 in out_sa['''attention_mask'''][0]       )
     # short slice does have padding
     self.assertTrue(pad_token_id in out_sa['''input_ids'''][1]       )
     self.assertTrue(0 in out_sa['''attention_mask'''][1]       )

     # p
     # test single pair max_length padding
     self.assertEqual(out_p['''input_ids'''].shape[-1]						,  60       )
     self.assertTrue(pad_token_id in out_p['''input_ids''']       )
     self.assertTrue(0 in out_p['''attention_mask''']       )

     # p2
     # test automatic padding pair
     self.assertEqual(out_pa['''input_ids'''].shape[-1]						,  52       )
     # long slice pair doesn't have padding
     self.assertFalse(pad_token_id in out_pa['''input_ids'''][0]       )
     self.assertFalse(0 in out_pa['''attention_mask'''][0]       )
     # short slice pair does have padding
     self.assertTrue(pad_token_id in out_pa['''input_ids'''][1]       )
     self.assertTrue(0 in out_pa['''attention_mask'''][1]       )

 def 		_snake_case      (  self	: Dict       ):
     snake_case_				:  Any       =			'''$$$'''
     snake_case_				:  str       =			CodeGenTokenizer.from_pretrained(self.tmpdirname						,  bos_token=lowercase_						,  add_bos_token=lowercase_       )

     snake_case_				:  Optional[int]       =			'''This is a simple input'''
     snake_case_				:  str       =			['''This is a simple input 1''', '''This is a simple input 2''']

     snake_case_				:  Any       =			tokenizer.bos_token_id

     snake_case_				:  List[Any]       =			tokenizer(lowercase_       )
     snake_case_				:  Optional[Any]       =			tokenizer(lowercase_       )

     self.assertEqual(out_s.input_ids[0]						,  lowercase_       )
     self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids       )       )

     snake_case_				:  List[str]       =			tokenizer.decode(out_s.input_ids       )
     snake_case_				:  Optional[Any]       =			tokenizer.batch_decode(out_sa.input_ids       )

     self.assertEqual(decode_s.split()[0]						,  lowercase_       )
     self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa       )       )

 @slow
 def 		_snake_case      (  self	: Optional[Any]       ):
     snake_case_				:  List[Any]       =			CodeGenTokenizer.from_pretrained('''Salesforce/codegen-350M-mono'''       )

     snake_case_				:  List[str]       =			'''\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'''
     snake_case_				:  Dict       =			'''\nif len_a > len_b:      result = a\nelse:      result = b'''

     snake_case_				:  Tuple       =			tokenizer.encode(lowercase_       )
     snake_case_				:  List[str]       =			['''^#''', re.escape('''<|endoftext|>'''       ), '''^\'\'\'''', '''^"""''', '''\n\n\n''']
     snake_case_				:  Any       =			tokenizer.decode(lowercase_						,  truncate_before_pattern=lowercase_       )
     self.assertEqual(lowercase_						,  lowercase_       )






 def 		_snake_case      (  self	: str       ):
     pass










"""simple docstring"""

import numpy as np
from numpy import ndarray
from scipy.optimize import Bounds, LinearConstraint, minimize




def __lowercase    (							_a					):
    return np.dot(_a	,    _a					)




class  _UpperCAmelCase :

 def __init__(  self	: int						,  *,
     lowercase_	: float = np.inf						,  lowercase_	: str = "linear"						,  lowercase_	: float = 0.0						,  ):
     snake_case_				:  Optional[Any]       =			regularization
     snake_case_				:  Tuple       =			gamma
     if kernel == "linear":
         snake_case_				:  int       =			self.__linear
     elif kernel == "rbf":
         if self.gamma == 0:
             raise ValueError('''rbf kernel requires gamma'''       )
         if not isinstance(self.gamma						,  (float, int)       ):
             raise ValueError('''gamma must be float or int'''       )
         if not self.gamma > 0:
             raise ValueError('''gamma must be > 0'''       )
         snake_case_				:  int       =			self.__rbf
         # in the future, there could be a default value like in sklearn
         # sklear: def_gamma = 1/(n_features * X.var()) (wiki)
         # previously it was 1/(n_features)
     else:
         snake_case_				:  List[Any]       =			f"Unknown kernel: {kernel}"
         raise ValueError(lowercase_       )

 def 		_snake_case      (  self	: int						,  lowercase_	: ndarray						,  lowercase_	: ndarray       ):
     return np.dot(lowercase_						,  lowercase_       )

 def 		_snake_case      (  self	: int						,  lowercase_	: ndarray						,  lowercase_	: ndarray       ):
     return np.exp(-(self.gamma * norm_squared(vectora - vectora       ))       )

 def 		_snake_case      (  self	: Any						,  lowercase_	: list[ndarray]						,  lowercase_	: ndarray       ):
     snake_case_				:  Union[str, Any]       =			observations
     snake_case_				:  int       =			classes

     # using Wolfe's Dual to calculate w.
     # Primal problem: minimize 1/2*norm_squared(w)
     #   constraint: yn(w . xn + b) >= 1
     #
     # With l a vector
     # Dual problem: maximize sum_n(ln) -
     #       1/2 * sum_n(sum_m(ln*lm*yn*ym*xn . xm))
     #   constraint: self.C >= ln >= 0
     #           and sum_n(ln*yn) = 0
     # Then we get w using w = sum_n(ln*yn*xn)
     # At the end we can get b ~= mean(yn - w . xn)
     #
     # Since we use kernels, we only need l_star to calculate b
     # and to classify observations

     ((snake_case_),			)				:  List[str]       =			np.shape(lowercase_       )

     def to_minimize(lowercase_	: ndarray       ) -> float:
         snake_case_				:  Tuple       =			0
         ((snake_case_),			)				:  Optional[Any]       =			np.shape(lowercase_       )
         for i in range(lowercase_       ):
             for j in range(lowercase_       ):
                 s += (
                     candidate[i]
                     * candidate[j]
                     * classes[i]
                     * classes[j]
                     * self.kernel(observations[i]						,  observations[j]       )
                 )
         return 1 / 2 * s - sum(lowercase_       )

     snake_case_				:  Optional[Any]       =			LinearConstraint(lowercase_						,  0						,  0       )
     snake_case_				:  str       =			Bounds(0						,  self.regularization       )

     snake_case_				:  int       =			minimize(
         lowercase_						,  np.ones(lowercase_       )						,  bounds=lowercase_						,  constraints=[ly_contraint]       ).x
     snake_case_				:  Optional[Any]       =			l_star

     # calculating mean offset of separation plane to points
     snake_case_				:  List[Any]       =			0
     for i in range(lowercase_       ):
         for j in range(lowercase_       ):
             s += classes[i] - classes[i] * self.optimum[i] * self.kernel(
                 observations[i]						,  observations[j]       )
     snake_case_				:  Union[str, Any]       =			s / n






 def 		_snake_case      (  self	: List[str]						,  lowercase_	: ndarray       ):
     snake_case_				:  int       =			sum(
         self.optimum[n]
         * self.classes[n]
         * self.kernel(self.observations[n]						,  lowercase_       )
         for n in range(len(self.classes       )       )       )
     return 1 if s + self.offset >= 0 else -1


if __name__ == "__main__":
     import doctest

     doctest.testmod()





