



'''simple docstring'''


import doctest
import glob
import importlib
import inspect
import os
import re
from contextlib import contextmanager
from functools import wraps
from unittest.mock import patch

import numpy as np
import pytest
from absl.testing import parameterized

import datasets
from datasets import load_metric

from .utils import for_all_test_methods, local, slow


# mark all tests as integration
lowercase							       =			pytest.mark.integration


lowercase							       =			{'''comet'''}
lowercase							       =			importlib.util.find_spec('''fairseq''') is not None

lowercase							       =			{'''code_eval'''}
lowercase							       =			os.name == '''nt'''

lowercase							       =			{'''bertscore''', '''frugalscore''', '''perplexity'''}
lowercase							       =			importlib.util.find_spec('''transformers''') is not None






def  UpperCAmelCase_       (		lowercase__       ):


     '''simple docstring'''







     @wraps(lowercase__       )
     def wrapper(self       ,	lowercase__       ):
          if not _has_fairseq and metric_name in REQUIRE_FAIRSEQ:
               self.skipTest("\"test requires Fairseq\""       )
          else:
               test_case(self       ,	lowercase__       )

     return wrapper






def  UpperCAmelCase_       (		lowercase__       ):


     '''simple docstring'''







     @wraps(lowercase__       )
     def wrapper(self       ,	lowercase__       ):
          if not _has_transformers and metric_name in REQUIRE_TRANSFORMERS:
               self.skipTest("\"test requires transformers\""       )
          else:
               test_case(self       ,	lowercase__       )

     return wrapper






def  UpperCAmelCase_       (		lowercase__       ):


     '''simple docstring'''







     @wraps(lowercase__       )
     def wrapper(self       ,	lowercase__       ):
          if _on_windows and metric_name in UNSUPPORTED_ON_WINDOWS:
               self.skipTest("\"test not supported on Windows\""       )
          else:
               test_case(self       ,	lowercase__       )

     return wrapper






def  UpperCAmelCase_       (		):


     '''simple docstring'''







     a_       		=[metric_dir.split(os.sep       )[-2] for metric_dir in glob.glob("./metrics/*/"       )]
     return [{"testcase_name": x, "metric_name": x} for x in metrics if x != "gleu"]  # gleu is unfinished







@parameterized.named_parameters(get_local_metric_names())
@for_all_test_methods(
    __a					,	__a					,	__a)
@local
class 	UpperCAmelCase	(		parameterized.TestCase):


  '''simple docstring'''




  __magic_name__					: Any  =      {}
  __magic_name__					: Optional[int]  =      None




  @pytest.mark.filterwarnings("ignore:metric_module_factory is deprecated:FutureWarning")
  @pytest.mark.filterwarnings("ignore:load_metric is deprecated:FutureWarning")
  def       lowercase_  (   self							,  lowerCAmelCase_) ->      List[str]:



       """simple docstring"""







       a_       		="[...]"
       a_       		=importlib.import_module(
           datasets.load.metric_module_factory(os.path.join("metrics"							,  lowerCAmelCase_)).module_path)
       a_       		=datasets.load.import_main_class(metric_module.__name__							,  dataset=lowerCAmelCase_)
       # check parameters
       a_       		=inspect.signature(metric._compute).parameters
       self.assertTrue(all(p.kind != p.VAR_KEYWORD for p in parameters.values()))  # no **kwargs
       # run doctest
       with self.patch_intensive_calls(lowerCAmelCase_							,  metric_module.__name__):
            with self.use_local_metrics():
                 try:
                      a_       		=doctest.testmod(lowerCAmelCase_							,  verbose=lowerCAmelCase_							,  raise_on_error=lowerCAmelCase_)
                 except doctest.UnexpectedException as e:
                      raise e.exc_info[1]  # raise the exception that doctest caught
       self.assertEqual(results.failed							,  0)
       self.assertGreater(results.attempted							,  1)




  @slow
  def       lowercase_  (   self							,  lowerCAmelCase_) ->      Tuple:



       """simple docstring"""







       a_       		="[...]"
       a_       		=importlib.import_module(
           datasets.load.metric_module_factory(os.path.join("metrics"							,  lowerCAmelCase_)).module_path)
       # run doctest
       with self.use_local_metrics():
            a_       		=doctest.testmod(lowerCAmelCase_							,  verbose=lowerCAmelCase_							,  raise_on_error=lowerCAmelCase_)
       self.assertEqual(results.failed							,  0)
       self.assertGreater(results.attempted							,  1)




  @contextmanager
  def       lowercase_  (   self							,  lowerCAmelCase_							,  lowerCAmelCase_) ->      Tuple:



       """simple docstring"""







       if metric_name in self.INTENSIVE_CALLS_PATCHER:
            with self.INTENSIVE_CALLS_PATCHER[metric_name](lowerCAmelCase_):
                 yield
       else:
            yield




  @contextmanager
  def       lowercase_  (   self) ->      Optional[Any]:



       """simple docstring"""







       def load_local_metric(lowerCAmelCase_							,  *lowerCAmelCase_							,  **lowerCAmelCase_):
            return load_metric(os.path.join("metrics"							,  lowerCAmelCase_)							,  *lowerCAmelCase_							,  **lowerCAmelCase_)

       with patch("datasets.load_metric") as mock_load_metric:
            a_       		=load_local_metric
            yield




  @classmethod
  def       lowercase_  (   cls							,  lowerCAmelCase_) ->      List[Any]:



       """simple docstring"""







       def wrapper(lowerCAmelCase_):
            a_       		=contextmanager(lowerCAmelCase_)
            a_       		=patcher
            return patcher

       return wrapper






@LocalMetricTest.register_intensive_calls_patcher("bleurt"       )
def  UpperCAmelCase_       (		lowercase__       ):


     '''simple docstring'''







     import tensorflow.compat.va as tf
     from bleurt.score import Predictor

     tf.flags.DEFINE_string("sv"       ,	""       ,	""       )  # handle pytest cli flags







     class 	UpperCAmelCase	(		__a):


       '''simple docstring'''




       def       lowercase_  (   self							,  lowerCAmelCase_) ->      Optional[Any]:



            """simple docstring"""







            assert len(input_dict["input_ids"]) == 2
            return np.array([1.0_3, 1.0_4])

    # mock predict_fn which is supposed to do a forward pass with a bleurt model
     with patch("bleurt.score._create_predictor"       ) as mock_create_predictor:
          a_       		=MockedPredictor()
          yield






@LocalMetricTest.register_intensive_calls_patcher("bertscore"       )
def  UpperCAmelCase_       (		lowercase__       ):


     '''simple docstring'''







     import torch

     def bert_cos_score_idf(lowercase__       ,	lowercase__       ,	*lowercase__       ,	**lowercase__       ):
          return torch.tensor([[1.0, 1.0, 1.0]] * len(lowercase__       )       )

     # mock get_model which is supposed to do download a bert model
     # mock bert_cos_score_idf which is supposed to do a forward pass with a bert model
     with patch("bert_score.scorer.get_model"       ), patch(
         "bert_score.scorer.bert_cos_score_idf"       ) as mock_bert_cos_score_idf:
          a_       		=bert_cos_score_idf
          yield






@LocalMetricTest.register_intensive_calls_patcher("comet"       )
def  UpperCAmelCase_       (		lowercase__       ):


     '''simple docstring'''







     def load_from_checkpoint(lowercase__       ):







          class 	UpperCAmelCase	:


            '''simple docstring'''




            def       lowercase_  (   self							,  lowerCAmelCase_							,  *lowerCAmelCase_							,  **lowerCAmelCase_) ->      Dict:



                 """simple docstring"""







                 assert len(lowerCAmelCase_) == 2
                 a_       		=[0.1_9, 0.9_2]
                 return scores, sum(lowerCAmelCase_) / len(lowerCAmelCase_)



          return Model()

     # mock load_from_checkpoint which is supposed to do download a bert model
     # mock load_from_checkpoint which is supposed to do download a bert model
     with patch("comet.download_model"       ) as mock_download_model:
          a_       		=None
          with patch("comet.load_from_checkpoint"       ) as mock_load_from_checkpoint:
               a_       		=load_from_checkpoint
               yield






def  UpperCAmelCase_       (		):


     '''simple docstring'''







     a_       		=load_metric(os.path.join("metrics"       ,	"seqeval"       )       )
     a_       		="ERROR"
     a_       		=F"""Scheme should be one of [IOB1, IOB2, IOE1, IOE2, IOBES, BILOU], got {wrong_scheme}"""
     with pytest.raises(lowercase__       ,	match=re.escape(lowercase__       )       ):
          metric.compute(predictions=[]       ,	references=[]       ,	scheme=lowercase__       )







'''simple docstring'''


import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
  from transformers.pipelines.conversational import Conversation


lowercase							       =			logging.get_logger(__name__)

lowercase							       =			{'''vocab_file''': '''vocab.txt''', '''emoji_file''': '''emoji.json'''}

lowercase							       =			{
    '''vocab_file''': {
        '''abeja/gpt-neox-japanese-2.7b''': '''https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt''',
    },
    '''emoji_file''': {
        '''abeja/gpt-neox-japanese-2.7b''': '''https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json''',
    },
}

lowercase							       =			{
    '''abeja/gpt-neox-japanese-2.7b''': 2_048,
}






def  UpperCAmelCase_       (		lowercase__       ,	lowercase__       ):


     '''simple docstring'''







     with open(lowercase__       ,	"r"       ,	encoding="utf-8"       ) as f:
          a_       		=json.loads(f.read()       )

     a_       		=collections.OrderedDict()
     a_       		=collections.OrderedDict()
     a_       		=collections.OrderedDict()
     with open(lowercase__       ,	"r"       ,	encoding="utf-8"       ) as f:
          a_       		=f.readlines()
     a_       		=[[t.rstrip("\n"       )] if (t == "," or "," not in t) else t.rstrip("\n"       ).split(","       ) for t in token]
     for idx, b in enumerate(lowercase__       ):
          a_       		=b
          a_       		=idx
          for wd in b:
               a_       		=idx

     return vocab, raw_vocab, ids_to_tokens, emoji







class 	UpperCAmelCase	(		__a):


 '''simple docstring'''




 __magic_name__					: Optional[int]  =      VOCAB_FILES_NAMES
 __magic_name__					: str  =      PRETRAINED_VOCAB_FILES_MAP
 __magic_name__					: Union[str, Any]  =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __magic_name__					: str  =      ["input_ids", "attention_mask"]




 def __init__(   self							,  lowerCAmelCase_							,  lowerCAmelCase_							,  lowerCAmelCase_="<|endoftext|>"							,  lowerCAmelCase_="<|endoftext|>"							,  lowerCAmelCase_="<|startoftext|>"							,  lowerCAmelCase_="<|endoftext|>"							,  lowerCAmelCase_=False							,  **lowerCAmelCase_							,  ) ->      List[Any]:



      """simple docstring"""







      super().__init__(
          unk_token=lowerCAmelCase_							,  pad_token=lowerCAmelCase_							,  bos_token=lowerCAmelCase_							,  eos_token=lowerCAmelCase_							,  do_clean_text=lowerCAmelCase_							,  **lowerCAmelCase_							,  )
      if not os.path.isfile(lowerCAmelCase_):
           raise ValueError(
               f"""Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"""
               " model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`")
      if not os.path.isfile(lowerCAmelCase_):
           raise ValueError(
               f"""Can't find a emoji file at path '{emoji_file}'. To load the emoji information from a Google"""
               " pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`")
      a_       		=do_clean_text
      a_		,						a_		,						a_		,						a_       		=load_vocab_and_emoji(lowerCAmelCase_							,  lowerCAmelCase_)
      a_       		=SubWordJapaneseTokenizer(
          vocab=self.vocab							,  ids_to_tokens=self.ids_to_tokens							,  emoji=self.emoji)




 @property
 def       lowercase_  (   self) ->      int:



      """simple docstring"""







      return len(self.raw_vocab)




 def       lowercase_  (   self) ->      Optional[Any]:



      """simple docstring"""







      return dict(self.raw_vocab							,  **self.added_tokens_encoder)




 def       lowercase_  (   self							,  lowerCAmelCase_) ->      List[str]:



      """simple docstring"""







      return self.subword_tokenizer.tokenize(lowerCAmelCase_							,  clean=self.do_clean_text)




 def       lowercase_  (   self							,  lowerCAmelCase_) ->      Optional[int]:



      """simple docstring"""







      return self.vocab.get(lowerCAmelCase_							,  self.vocab.get(self.unk_token))




 def       lowercase_  (   self							,  lowerCAmelCase_) ->      List[str]:



      """simple docstring"""







      return self.subword_tokenizer.convert_id_to_token(lowerCAmelCase_)




 def       lowercase_  (   self							,  lowerCAmelCase_) ->      Optional[Any]:



      """simple docstring"""







      a_       		="".join(lowerCAmelCase_).strip()
      return out_string




 def       lowercase_  (   self							,  lowerCAmelCase_) ->      List[int]:



      """simple docstring"""







      a_       		=[]
      for is_user, text in conversation.iter_texts():
           input_ids.extend(self.encode(lowerCAmelCase_							,  add_special_tokens=lowerCAmelCase_) + [self.eos_token_id])

      if len(lowerCAmelCase_) > self.model_max_length:
           a_       		=input_ids[-self.model_max_length :]
      return input_ids




 def       lowercase_  (   self							,  lowerCAmelCase_							,  lowerCAmelCase_ = None) ->      Tuple[str]:



      """simple docstring"""







      a_       		=0
      if os.path.isdir(lowerCAmelCase_):
           a_       		=os.path.join(
               lowerCAmelCase_							,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])
           a_       		=os.path.join(
               lowerCAmelCase_							,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["emoji_file"])
      else:
           a_       		=(
               (filename_prefix + "-" if filename_prefix else "") + save_directory + VOCAB_FILES_NAMES["vocab_file"]
           )
           a_       		=(
               (filename_prefix + "-" if filename_prefix else "") + save_directory + VOCAB_FILES_NAMES["emoji_file"]
           )
      with open(lowerCAmelCase_							,  "w"							,  encoding="utf-8") as writer:
           for token_index, token in self.ids_to_tokens.items():
                if index != token_index:
                     logger.warning(
                         f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
                         " Please check that the vocabulary is not corrupted!")
                     a_       		=token_index
                writer.write(",".join(lowerCAmelCase_) + "\n")
                index += 1
      with open(lowerCAmelCase_							,  "w"							,  encoding="utf-8") as writer:
           json.dump(self.emoji							,  lowerCAmelCase_)
      return vocab_file, emoji_file







class 	UpperCAmelCase	(		__a):


 '''simple docstring'''




 def __init__(   self							,  lowerCAmelCase_							,  lowerCAmelCase_							,  lowerCAmelCase_) ->      str:



      """simple docstring"""







      a_       		=vocab  # same as swe
      a_       		=ids_to_tokens  # same as bpe
      a_       		=emoji
      a_       		=np.max([len(lowerCAmelCase_) for w in self.vocab.keys()])
      a_       		=re.compile(r"(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)")
      a_       		=re.compile(r"[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*")
      a_       		=re.compile(r"[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}")
      a_       		=re.compile(
          r"([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*")
      a_       		=re.compile(
          r"(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*")
      a_       		=re.compile(
          r"((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*")
      a_       		="─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿"
      a_       		="▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟"
      a_       		=str.maketrans({k: "<BLOCK>" for k in keisen + blocks})




 def __len__(   self) ->      Tuple:



      """simple docstring"""







      return len(self.ids_to_tokens)




 def       lowercase_  (   self							,  lowerCAmelCase_) ->      Any:



      """simple docstring"""







      a_       		=self.content_repattera.sub("<URL>"							,  lowerCAmelCase_)
      a_       		=self.content_repattera.sub("<EMAIL>"							,  lowerCAmelCase_)
      a_       		=self.content_repattera.sub("<TEL>"							,  lowerCAmelCase_)
      a_       		=self.content_repattera.sub("<DATE>"							,  lowerCAmelCase_)
      a_       		=self.content_repattera.sub("<DATE>"							,  lowerCAmelCase_)
      a_       		=self.content_repattera.sub("<PRICE>"							,  lowerCAmelCase_)
      a_       		=content.translate(self.content_transa)
      while "<BLOCK><BLOCK>" in content:
           a_       		=content.replace("<BLOCK><BLOCK>"							,  "<BLOCK>")
      return content




 def       lowercase_  (   self							,  lowerCAmelCase_							,  lowerCAmelCase_=False) ->      Union[str, Any]:



      """simple docstring"""







      a_       		=text.replace(" "							,  "<SP>")
      a_       		=text.replace("　"							,  "<SP>")
      a_       		=text.replace("\r\n"							,  "<BR>")
      a_       		=text.replace("\n"							,  "<BR>")
      a_       		=text.replace("\r"							,  "<BR>")
      a_       		=text.replace("\t"							,  "<TAB>")
      a_       		=text.replace("—"							,  "ー")
      a_       		=text.replace("−"							,  "ー")
      for k, v in self.emoji["emoji"].items():
           if k in text:
                a_       		=text.replace(lowerCAmelCase_							,  lowerCAmelCase_)
      if clean:
           a_       		=self.clean_text(lowerCAmelCase_)

      def check_simbol(lowerCAmelCase_):
           a_       		=x.encode()
           if len(lowerCAmelCase_) == 1 and len(lowerCAmelCase_) == 2:
                a_       		=(int(e[0]) << 8) + int(e[1])
                if (
                    (c >= 0xc2a1 and c <= 0xc2bf)
                    or (c >= 0xc780 and c <= 0xc783)
                    or (c >= 0xcab9 and c <= 0xcbbf)
                    or (c >= 0xcc80 and c <= 0xcda2)
                ):
                     return True
           return False

      def checkuae(lowerCAmelCase_):
           a_       		=x.encode()
           if len(lowerCAmelCase_) == 1 and len(lowerCAmelCase_) == 3:
                a_       		=(int(e[0]) << 1_6) + (int(e[1]) << 8) + int(e[2])
                if c >= 0xe2_8080 and c <= 0xe2_b07f:
                     return True
           return False

      a_       		=0
      a_       		=[]
      while pos < len(lowerCAmelCase_):
           a_       		=min(len(lowerCAmelCase_)							,  pos + self.maxlen + 1) if text[pos] == "<" else pos + 3
           a_       		=[]  # (token_id, token, pos)
           for e in range(lowerCAmelCase_							,  lowerCAmelCase_							,  -1):
                a_       		=text[pos:e]
                if wd in self.vocab:
                     if wd[0] == "<" and len(lowerCAmelCase_) > 2:
                          a_       		=[(self.vocab[wd], wd, e)]
                          break
                     else:
                          candidates.append((self.vocab[wd], wd, e))
           if len(lowerCAmelCase_) > 0:
                # the smallest token_id is adopted
                a_		,						a_		,						a_       		=sorted(lowerCAmelCase_							,  key=lambda lowerCAmelCase_: x[0])[0]
                result.append(lowerCAmelCase_)
                a_       		=e
           else:
                a_       		=pos + 1
                a_       		=text[pos:end]
                if check_simbol(lowerCAmelCase_):
                     result.append("<KIGOU>")
                elif checkuae(lowerCAmelCase_):
                     result.append("<U2000U2BFF>")
                else:
                     for i in wd.encode("utf-8"):
                          result.append("<|byte%d|>" % i)
                a_       		=end
      return result




 def       lowercase_  (   self							,  lowerCAmelCase_							,  lowerCAmelCase_="\n") ->      List[Any]:



      """simple docstring"""







      a_       		=[]
      a_       		=[]
      a_       		=self.ids_to_tokens[index][0]
      if word[:6] == "<|byte" and word[-2:] == "|>":
           byte_tokens.append(int(word[6:-2]))
      else:
           if len(lowerCAmelCase_) > 0:
                words.append(bytearray(lowerCAmelCase_).decode("utf-8"							,  errors="replace"))
                a_       		=[]
           if word[:7] == "<|emoji" and word[-2:] == "|>":
                words.append(self.emoji["emoji_inv"][word])
           elif word == "<SP>":
                words.append(" ")
           elif word == "<BR>":
                words.append(lowerCAmelCase_)
           elif word == "<TAB>":
                words.append("\t")
           elif word == "<BLOCK>":
                words.append("▀")
           elif word == "<KIGOU>":
                words.append("ǀ")
           elif word == "<U2000U2BFF>":
                words.append("‖")
           else:
                words.append(lowerCAmelCase_)
      if len(lowerCAmelCase_) > 0:
           words.append(bytearray(lowerCAmelCase_).decode("utf-8"							,  errors="replace"))
      a_       		="".join(lowerCAmelCase_)
      return text


