






import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
				if is_torch_available():
								import torch
				if is_tf_available():
								import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


a_		:						str             =    logging.get_logger(__name__)

a_		:						str             =    {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

a_		:						int             =    {
    'vocab_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
    },
    'merges_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'Salesforce/codegen-350M-mono': (
            'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json'
        ),
    },
}

a_		:						Any             =    {
    'Salesforce/codegen-350M-mono': 20_48,
}

class 		_snake_case  (  A__					):
							_lowercase		:    Optional[Any]			     =							VOCAB_FILES_NAMES
							_lowercase		:    str			     =							PRETRAINED_VOCAB_FILES_MAP
							_lowercase		:    Dict			     =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							_lowercase		:    int			     =							['''input_ids''', '''attention_mask''']
							_lowercase		:    List[str]			     =							CodeGenTokenizer






							def __init__(       self		,					a=None		,					a=None		,					a=None		,					a="<|endoftext|>"		,					a="<|endoftext|>"		,					a="<|endoftext|>"		,					a=False		,					**a		,					)      ->       int:
										super().__init__(
										    a		,					a		,					tokenizer_file=a		,					unk_token=a		,					bos_token=a		,					eos_token=a		,					add_prefix_space=a		,					**a		,					)

										if kwargs.pop('add_bos_token'		,					a):
													SCREAMING_SNAKE_CASE						= kwargs.pop('name_or_path'		,					'')
													raise ValueError(
													    'Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token.'
													    'Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n'
													    f'''`CodeGenTokenizer.from_pretrained(\'{model_id}\')`\nor\n'''
													    f'''`AutoTokenizer.from_pretrained(\'{model_id}\', use_fast=False)`\n'''
													    'This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005.'
													    ' so that the fast tokenizer works correctly.')

										SCREAMING_SNAKE_CASE						= json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
										if pre_tok_state.get('add_prefix_space'		,					a) != add_prefix_space:
													SCREAMING_SNAKE_CASE						= getattr(a		,					pre_tok_state.pop('type'))
													SCREAMING_SNAKE_CASE						= add_prefix_space
													SCREAMING_SNAKE_CASE						= pre_tok_class(**a)

										SCREAMING_SNAKE_CASE						= add_prefix_space






							def    SCREAMING_SNAKE_CASE__    (       self		,					*a		,					**a)      ->       BatchEncoding:
										SCREAMING_SNAKE_CASE						= kwargs.get('is_split_into_words'		,					a)
										assert self.add_prefix_space or not is_split_into_words, (
										    f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
										    "to use it with pretokenized inputs."
										)

										return super()._batch_encode_plus(*a		,					**a)






							def    SCREAMING_SNAKE_CASE__    (       self		,					*a		,					**a)      ->       BatchEncoding:
										SCREAMING_SNAKE_CASE						= kwargs.get('is_split_into_words'		,					a)

										assert self.add_prefix_space or not is_split_into_words, (
										    f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
										    "to use it with pretokenized inputs."
										)

										return super()._encode_plus(*a		,					**a)






							def    SCREAMING_SNAKE_CASE__    (       self		,					a		,					a = None)      ->       Tuple[str]:
										SCREAMING_SNAKE_CASE						= self._tokenizer.model.save(a		,					name=a)
										return tuple(a)






							def    SCREAMING_SNAKE_CASE__    (       self		,					a		,					a = False		,					a = None		,					a = None		,					**a		,					)      ->       str:
										SCREAMING_SNAKE_CASE						= super().decode(
										    token_ids=a		,					skip_special_tokens=a		,					clean_up_tokenization_spaces=a		,					**a		,					)

										if truncate_before_pattern is not None and len(a) > 0:
													SCREAMING_SNAKE_CASE						= self.truncate(a		,					a)

										return decoded_text







							def    SCREAMING_SNAKE_CASE__    (       self		,					a		,					a)      ->       Optional[Any]:
										def find_re(a		,					a		,					a):
													SCREAMING_SNAKE_CASE						= pattern.search(a		,					a)
													return m.start() if m else -1

										SCREAMING_SNAKE_CASE						= [re.compile(a		,					re.MULTILINE) for pattern in truncate_before_pattern]

										SCREAMING_SNAKE_CASE						= list(re.finditer('^print'		,					a		,					re.MULTILINE))

										if len(a) > 1:
													SCREAMING_SNAKE_CASE						= completion[: prints[1].start()]

										SCREAMING_SNAKE_CASE						= list(re.finditer('^def'		,					a		,					re.MULTILINE))

										if len(a) > 1:
													SCREAMING_SNAKE_CASE						= completion[: defs[1].start()]

										SCREAMING_SNAKE_CASE						= 0

										SCREAMING_SNAKE_CASE						= [
										    pos for pos in [find_re(a		,					a		,					a) for terminal in terminals] if pos != -1
										]

										if len(a) > 0:
													return completion[: min(a)]
										else:
													return completion












import numpy as np
from scipy.spatial.distance import cdist
from sklearn.metrics import fa_score

import datasets


a_		:						Optional[int]             =    '\\n    @inproceedings{kakwani2020indicnlpsuite,\n    title={{IndicNLPSuite: Monolingual Corpora, Evaluation Benchmarks and Pre-trained Multilingual Language Models for Indian Languages}},\n    author={Divyanshu Kakwani and Anoop Kunchukuttan and Satish Golla and Gokul N.C. and Avik Bhattacharyya and Mitesh M. Khapra and Pratyush Kumar},\n    year={2020},\n    booktitle={Findings of EMNLP},\n}\n'

a_		:						Tuple             =    '\\n    IndicGLUE is a natural language understanding benchmark for Indian languages. It contains a wide\n    variety of tasks and covers 11 major Indian languages - as, bn, gu, hi, kn, ml, mr, or, pa, ta, te.\n'

a_		:						Optional[Any]             =    '\nCompute IndicGLUE evaluation metric associated to each IndicGLUE dataset.\nArgs:\n    predictions: list of predictions to score (as int64),\n        except for \'cvit-mkb-clsr\' where each prediction is a vector (of float32).\n    references: list of ground truth labels corresponding to the predictions (as int64),\n        except for \'cvit-mkb-clsr\' where each reference is a vector (of float32).\nReturns: depending on the IndicGLUE subset, one or several of:\n    "accuracy": Accuracy\n    "f1": F1 score\n    "precision": Precision@10\nExamples:\n\n    >>> indic_glue_metric = datasets.load_metric(\'indic_glue\', \'wnli\')  # \'wnli\' or any of ["copa", "sna", "csqa", "wstp", "inltkh", "bbca", "iitp-mr", "iitp-pr", "actsa-sc", "md"]\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0}\n\n    >>> indic_glue_metric = datasets.load_metric(\'indic_glue\', \'wiki-ner\')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0, \'f1\': 1.0}\n\n    >>> indic_glue_metric = datasets.load_metric(\'indic_glue\', \'cvit-mkb-clsr\')\n    >>> references = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]]\n    >>> predictions = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]]\n    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'precision@10\': 1.0}\n\n'






def 				lowerCamelCase__				(_UpperCAmelCase							,       _UpperCAmelCase):
			return float((preds == labels).mean())






def 				lowerCamelCase__				(_UpperCAmelCase							,       _UpperCAmelCase):
			SCREAMING_SNAKE_CASE						= simple_accuracy(_UpperCAmelCase							,       _UpperCAmelCase)
			SCREAMING_SNAKE_CASE						= float(fa_score(y_true=_UpperCAmelCase							,       y_pred=_UpperCAmelCase))
			return {
			    "accuracy": acc,
			    "f1": fa,
			}






def 				lowerCamelCase__				(_UpperCAmelCase							,       _UpperCAmelCase):
			SCREAMING_SNAKE_CASE						= np.array(_UpperCAmelCase)
			SCREAMING_SNAKE_CASE						= np.array(_UpperCAmelCase)
			SCREAMING_SNAKE_CASE						= en_sentvecs.shape[0]

			# mean centering
			SCREAMING_SNAKE_CASE						= en_sentvecs - np.mean(_UpperCAmelCase							,       axis=0)
			SCREAMING_SNAKE_CASE						= in_sentvecs - np.mean(_UpperCAmelCase							,       axis=0)

			SCREAMING_SNAKE_CASE						= cdist(_UpperCAmelCase							,       _UpperCAmelCase							,       'cosine')
			SCREAMING_SNAKE_CASE						= np.array(range(_UpperCAmelCase))
			SCREAMING_SNAKE_CASE						= sim.argsort(axis=1)[:, :10]
			SCREAMING_SNAKE_CASE						= np.any(preds == actual[:, None]							,       axis=1)
			return float(matches.mean())







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION   ,    _KWARGS_DESCRIPTION					)
class 		_snake_case  (  datasets.Metric					):






			def    SCREAMING_SNAKE_CASE__    (       self)      ->       Dict:
						if self.config_name not in [
						    "wnli",
						    "copa",
						    "sna",
						    "csqa",
						    "wstp",
						    "inltkh",
						    "bbca",
						    "cvit-mkb-clsr",
						    "iitp-mr",
						    "iitp-pr",
						    "actsa-sc",
						    "md",
						    "wiki-ner",
						]:
									raise KeyError(
									    'You should supply a configuration name selected in '
									    '["wnli", "copa", "sna", "csqa", "wstp", "inltkh", "bbca", '
									    '"cvit-mkb-clsr", "iitp-mr", "iitp-pr", "actsa-sc", "md", '
									    '"wiki-ner"]')
						return datasets.MetricInfo(
						    description=_DESCRIPTION		,					citation=_CITATION		,					inputs_description=_KWARGS_DESCRIPTION		,					features=datasets.Features(
						        {
						            'predictions': datasets.Value('int64')
						            if self.config_name != 'cvit-mkb-clsr'
						            else datasets.Sequence(datasets.Value('float32')),
						            'references': datasets.Value('int64')
						            if self.config_name != 'cvit-mkb-clsr'
						            else datasets.Sequence(datasets.Value('float32')),
						        })		,					codebase_urls=[]		,					reference_urls=[]		,					format='numpy' if self.config_name != 'cvit-mkb-clsr' else None		,					)







			def    SCREAMING_SNAKE_CASE__    (       self		,					a		,					a)      ->       List[Any]:
						if self.config_name == "cvit-mkb-clsr":
									return {"precision@10": precision_at_aa(a		,					a)}
						elif self.config_name in ["wiki-ner"]:
									return acc_and_fa(a		,					a)
						elif self.config_name in [
						    "wnli",
						    "copa",
						    "sna",
						    "csqa",
						    "wstp",
						    "inltkh",
						    "bbca",
						    "iitp-mr",
						    "iitp-pr",
						    "actsa-sc",
						    "md",
						]:
									return {"accuracy": simple_accuracy(a		,					a)}
						else:
									raise KeyError(
									    'You should supply a configuration name selected in '
									    '["wnli", "copa", "sna", "csqa", "wstp", "inltkh", "bbca", '
									    '"cvit-mkb-clsr", "iitp-mr", "iitp-pr", "actsa-sc", "md", '
									    '"wiki-ner"]')




