



'''simple docstring'''


import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


lowercase							       =			re.compile(R'''\b(a|an|the)\b''', re.UNICODE)

lowercase							       =			None






def  UpperCAmelCase_       (		):


     '''simple docstring'''







     a_       		=argparse.ArgumentParser("Official evaluation script for SQuAD version 2.0."       )
     parser.add_argument("data_file"       ,	metavar="data.json"       ,	help="Input data JSON file."       )
     parser.add_argument("pred_file"       ,	metavar="pred.json"       ,	help="Model predictions."       )
     parser.add_argument(
         "--out-file"       ,	"-o"       ,	metavar="eval.json"       ,	help="Write accuracy metrics to file (default is stdout)."       )
     parser.add_argument(
         "--na-prob-file"       ,	"-n"       ,	metavar="na_prob.json"       ,	help="Model estimates of probability of no answer."       )
     parser.add_argument(
         "--na-prob-thresh"       ,	"-t"       ,	type=lowercase__       ,	default=1.0       ,	help="Predict \"\" if no-answer probability exceeds this (default = 1.0)."       ,	)
     parser.add_argument(
         "--out-image-dir"       ,	"-p"       ,	metavar="out_images"       ,	default=lowercase__       ,	help="Save precision-recall curves to directory."       )
     parser.add_argument("--verbose"       ,	"-v"       ,	action="store_true"       )
     if len(sys.argv       ) == 1:
          parser.print_help()
          sys.exit(1       )
     return parser.parse_args()






def  UpperCAmelCase_       (		lowercase__       ):


     '''simple docstring'''







     a_       		={}
     for article in dataset:
          for p in article["paragraphs"]:
               for qa in p["qas"]:
                    a_       		=bool(qa["answers"]["text"]       )
     return qid_to_has_ans






def  UpperCAmelCase_       (		lowercase__       ):


     '''simple docstring'''







     def remove_articles(lowercase__       ):
          return ARTICLES_REGEX.sub(" "       ,	lowercase__       )

     def white_space_fix(lowercase__       ):
          return " ".join(text.split()       )

     def remove_punc(lowercase__       ):
          a_       		=set(string.punctuation       )
          return "".join(ch for ch in text if ch not in exclude       )

     def lower(lowercase__       ):
          return text.lower()

     return white_space_fix(remove_articles(remove_punc(lower(lowercase__       )       )       )       )






def  UpperCAmelCase_       (		lowercase__       ):


     '''simple docstring'''







     if not s:
          return []
     return normalize_answer(lowercase__       ).split()






def  UpperCAmelCase_       (		lowercase__       ,	lowercase__       ):


     '''simple docstring'''







     return int(normalize_answer(lowercase__       ) == normalize_answer(lowercase__       )       )






def  UpperCAmelCase_       (		lowercase__       ,	lowercase__       ):


     '''simple docstring'''







     a_       		=get_tokens(lowercase__       )
     a_       		=get_tokens(lowercase__       )
     a_       		=collections.Counter(lowercase__       ) & collections.Counter(lowercase__       )
     a_       		=sum(common.values()       )
     if len(lowercase__       ) == 0 or len(lowercase__       ) == 0:
          # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
          return int(gold_toks == pred_toks       )
     if num_same == 0:
          return 0
     a_       		=1.0 * num_same / len(lowercase__       )
     a_       		=1.0 * num_same / len(lowercase__       )
     a_       		=(2 * precision * recall) / (precision + recall)
     return fa






def  UpperCAmelCase_       (		lowercase__       ,	lowercase__       ):


     '''simple docstring'''







     a_       		={}
     a_       		={}
     for article in dataset:
          for p in article["paragraphs"]:
               for qa in p["qas"]:
                    a_       		=qa["id"]
                    a_       		=[t for t in qa["answers"]["text"] if normalize_answer(lowercase__       )]
                    if not gold_answers:
                         # For unanswerable questions, only correct answer is empty string
                         a_       		=[""]
                    if qid not in preds:
                         print(F"""Missing prediction for {qid}"""       )
                         continue
                    a_       		=preds[qid]
                    # Take max over all gold answers
                    a_       		=max(compute_exact(lowercase__       ,	lowercase__       ) for a in gold_answers       )
                    a_       		=max(compute_fa(lowercase__       ,	lowercase__       ) for a in gold_answers       )
     return exact_scores, fa_scores






def  UpperCAmelCase_       (		lowercase__       ,	lowercase__       ,	lowercase__       ,	lowercase__       ):


     '''simple docstring'''







     a_       		={}
     for qid, s in scores.items():
          a_       		=na_probs[qid] > na_prob_thresh
          if pred_na:
               a_       		=float(not qid_to_has_ans[qid]       )
          else:
               a_       		=s
     return new_scores






def  UpperCAmelCase_       (		lowercase__       ,	lowercase__       ,	lowercase__=None       ):


     '''simple docstring'''







     if not qid_list:
          a_       		=len(lowercase__       )
          return collections.OrderedDict(
              [
                  ("exact", 100.0 * sum(exact_scores.values()       ) / total),
                  ("f1", 100.0 * sum(fa_scores.values()       ) / total),
                  ("total", total),
              ]       )
     else:
          a_       		=len(lowercase__       )
          return collections.OrderedDict(
              [
                  ("exact", 100.0 * sum(exact_scores[k] for k in qid_list       ) / total),
                  ("f1", 100.0 * sum(fa_scores[k] for k in qid_list       ) / total),
                  ("total", total),
              ]       )






def  UpperCAmelCase_       (		lowercase__       ,	lowercase__       ,	lowercase__       ):


     '''simple docstring'''







     for k in new_eval:
          a_       		=new_eval[k]






def  UpperCAmelCase_       (		lowercase__       ,	lowercase__       ,	lowercase__       ,	lowercase__       ):


     '''simple docstring'''







     plt.step(lowercase__       ,	lowercase__       ,	color="b"       ,	alpha=0.2       ,	where="post"       )
     plt.fill_between(lowercase__       ,	lowercase__       ,	step="post"       ,	alpha=0.2       ,	color="b"       )
     plt.xlabel("Recall"       )
     plt.ylabel("Precision"       )
     plt.xlim([0.0, 1.05]       )
     plt.ylim([0.0, 1.05]       )
     plt.title(lowercase__       )
     plt.savefig(lowercase__       )
     plt.clf()






def  UpperCAmelCase_       (		lowercase__       ,	lowercase__       ,	lowercase__       ,	lowercase__       ,	lowercase__=None       ,	lowercase__=None       ):


     '''simple docstring'''







     a_       		=sorted(lowercase__       ,	key=lambda lowercase__       : na_probs[k]       )
     a_       		=0.0
     a_       		=1.0
     a_       		=0.0
     a_       		=[1.0]
     a_       		=[0.0]
     a_       		=0.0
     for i, qid in enumerate(lowercase__       ):
          if qid_to_has_ans[qid]:
               true_pos += scores[qid]
          a_       		=true_pos / float(i + 1       )
          a_       		=true_pos / float(lowercase__       )
          if i == len(lowercase__       ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
               # i.e., if we can put a threshold after this point
               avg_prec += cur_p * (cur_r - recalls[-1])
               precisions.append(lowercase__       )
               recalls.append(lowercase__       )
     if out_image:
          plot_pr_curve(lowercase__       ,	lowercase__       ,	lowercase__       ,	lowercase__       )
     return {"ap": 100.0 * avg_prec}






def  UpperCAmelCase_       (		lowercase__       ,	lowercase__       ,	lowercase__       ,	lowercase__       ,	lowercase__       ,	lowercase__       ):


     '''simple docstring'''







     if out_image_dir and not os.path.exists(lowercase__       ):
          os.makedirs(lowercase__       )
     a_       		=sum(1 for v in qid_to_has_ans.values() if v       )
     if num_true_pos == 0:
          return
     a_       		=make_precision_recall_eval(
         lowercase__       ,	lowercase__       ,	lowercase__       ,	lowercase__       ,	out_image=os.path.join(lowercase__       ,	"pr_exact.png"       )       ,	title="Precision-Recall curve for Exact Match score"       ,	)
     a_       		=make_precision_recall_eval(
         lowercase__       ,	lowercase__       ,	lowercase__       ,	lowercase__       ,	out_image=os.path.join(lowercase__       ,	"pr_f1.png"       )       ,	title="Precision-Recall curve for F1 score"       ,	)
     a_       		={k: float(lowercase__       ) for k, v in qid_to_has_ans.items()}
     a_       		=make_precision_recall_eval(
         lowercase__       ,	lowercase__       ,	lowercase__       ,	lowercase__       ,	out_image=os.path.join(lowercase__       ,	"pr_oracle.png"       )       ,	title="Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"       ,	)
     merge_eval(lowercase__       ,	lowercase__       ,	"pr_exact"       )
     merge_eval(lowercase__       ,	lowercase__       ,	"pr_f1"       )
     merge_eval(lowercase__       ,	lowercase__       ,	"pr_oracle"       )






def  UpperCAmelCase_       (		lowercase__       ,	lowercase__       ,	lowercase__       ,	lowercase__       ):


     '''simple docstring'''







     if not qid_list:
          return
     a_       		=[na_probs[k] for k in qid_list]
     a_       		=np.ones_like(lowercase__       ) / float(len(lowercase__       )       )
     plt.hist(lowercase__       ,	weights=lowercase__       ,	bins=2_0       ,	range=(0.0, 1.0)       )
     plt.xlabel("Model probability of no-answer"       )
     plt.ylabel("Proportion of dataset"       )
     plt.title(F"""Histogram of no-answer probability: {name}"""       )
     plt.savefig(os.path.join(lowercase__       ,	F"""na_prob_hist_{name}.png"""       )       )
     plt.clf()






def  UpperCAmelCase_       (		lowercase__       ,	lowercase__       ,	lowercase__       ,	lowercase__       ):


     '''simple docstring'''







     a_       		=sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]       )
     a_       		=num_no_ans
     a_       		=cur_score
     a_       		=0.0
     a_       		=sorted(lowercase__       ,	key=lambda lowercase__       : na_probs[k]       )
     for i, qid in enumerate(lowercase__       ):
          if qid not in scores:
               continue
          if qid_to_has_ans[qid]:
               a_       		=scores[qid]
          else:
               if preds[qid]:
                    a_       		=-1
               else:
                    a_       		=0
          cur_score += diff
          if cur_score > best_score:
               a_       		=cur_score
               a_       		=na_probs[qid]
     return 100.0 * best_score / len(lowercase__       ), best_thresh






def  UpperCAmelCase_       (		lowercase__       ,	lowercase__       ,	lowercase__       ,	lowercase__       ,	lowercase__       ,	lowercase__       ):


     '''simple docstring'''







     a_		,						a_       		=find_best_thresh(lowercase__       ,	lowercase__       ,	lowercase__       ,	lowercase__       )
     a_		,						a_       		=find_best_thresh(lowercase__       ,	lowercase__       ,	lowercase__       ,	lowercase__       )
     a_       		=best_exact
     a_       		=exact_thresh
     a_       		=best_fa
     a_       		=fa_thresh






def  UpperCAmelCase_       (		):


     '''simple docstring'''







     with open(OPTS.data_file       ) as f:
          a_       		=json.load(lowercase__       )
          a_       		=dataset_json["data"]
     with open(OPTS.pred_file       ) as f:
          a_       		=json.load(lowercase__       )
     if OPTS.na_prob_file:
          with open(OPTS.na_prob_file       ) as f:
               a_       		=json.load(lowercase__       )
     else:
          a_       		={k: 0.0 for k in preds}
     a_       		=make_qid_to_has_ans(lowercase__       )  # maps qid to True/False
     a_       		=[k for k, v in qid_to_has_ans.items() if v]
     a_       		=[k for k, v in qid_to_has_ans.items() if not v]
     a_		,						a_       		=get_raw_scores(lowercase__       ,	lowercase__       )
     a_       		=apply_no_ans_threshold(lowercase__       ,	lowercase__       ,	lowercase__       ,	OPTS.na_prob_thresh       )
     a_       		=apply_no_ans_threshold(lowercase__       ,	lowercase__       ,	lowercase__       ,	OPTS.na_prob_thresh       )
     a_       		=make_eval_dict(lowercase__       ,	lowercase__       )
     if has_ans_qids:
          a_       		=make_eval_dict(lowercase__       ,	lowercase__       ,	qid_list=lowercase__       )
          merge_eval(lowercase__       ,	lowercase__       ,	"HasAns"       )
     if no_ans_qids:
          a_       		=make_eval_dict(lowercase__       ,	lowercase__       ,	qid_list=lowercase__       )
          merge_eval(lowercase__       ,	lowercase__       ,	"NoAns"       )
     if OPTS.na_prob_file:
          find_all_best_thresh(lowercase__       ,	lowercase__       ,	lowercase__       ,	lowercase__       ,	lowercase__       ,	lowercase__       )
     if OPTS.na_prob_file and OPTS.out_image_dir:
          run_precision_recall_analysis(lowercase__       ,	lowercase__       ,	lowercase__       ,	lowercase__       ,	lowercase__       ,	OPTS.out_image_dir       )
          histogram_na_prob(lowercase__       ,	lowercase__       ,	OPTS.out_image_dir       ,	"hasAns"       )
          histogram_na_prob(lowercase__       ,	lowercase__       ,	OPTS.out_image_dir       ,	"noAns"       )
     if OPTS.out_file:
          with open(OPTS.out_file       ,	"w"       ) as f:
               json.dump(lowercase__       ,	lowercase__       )
     else:
          print(json.dumps(lowercase__       ,	indent=2       )       )


if __name__ == "__main__":
  lowercase							       =			parse_args()
  if OPTS.out_image_dir:
    import matplotlib

    matplotlib.use('''Agg''')
    import matplotlib.pyplot as plt
  main()







'''simple docstring'''


import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
  from transformers.pipelines.conversational import Conversation


lowercase							       =			logging.get_logger(__name__)

lowercase							       =			{'''vocab_file''': '''vocab.txt''', '''emoji_file''': '''emoji.json'''}

lowercase							       =			{
    '''vocab_file''': {
        '''abeja/gpt-neox-japanese-2.7b''': '''https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt''',
    },
    '''emoji_file''': {
        '''abeja/gpt-neox-japanese-2.7b''': '''https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json''',
    },
}

lowercase							       =			{
    '''abeja/gpt-neox-japanese-2.7b''': 2_048,
}






def  UpperCAmelCase_       (		lowercase__       ,	lowercase__       ):


     '''simple docstring'''







     with open(lowercase__       ,	"r"       ,	encoding="utf-8"       ) as f:
          a_       		=json.loads(f.read()       )

     a_       		=collections.OrderedDict()
     a_       		=collections.OrderedDict()
     a_       		=collections.OrderedDict()
     with open(lowercase__       ,	"r"       ,	encoding="utf-8"       ) as f:
          a_       		=f.readlines()
     a_       		=[[t.rstrip("\n"       )] if (t == "," or "," not in t) else t.rstrip("\n"       ).split(","       ) for t in token]
     for idx, b in enumerate(lowercase__       ):
          a_       		=b
          a_       		=idx
          for wd in b:
               a_       		=idx

     return vocab, raw_vocab, ids_to_tokens, emoji







class 	UpperCAmelCase	(		__a):


 '''simple docstring'''




 __magic_name__					: Optional[int]  =      VOCAB_FILES_NAMES
 __magic_name__					: str  =      PRETRAINED_VOCAB_FILES_MAP
 __magic_name__					: Union[str, Any]  =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __magic_name__					: str  =      ["input_ids", "attention_mask"]




 def __init__(   self							,  lowerCAmelCase_							,  lowerCAmelCase_							,  lowerCAmelCase_="<|endoftext|>"							,  lowerCAmelCase_="<|endoftext|>"							,  lowerCAmelCase_="<|startoftext|>"							,  lowerCAmelCase_="<|endoftext|>"							,  lowerCAmelCase_=False							,  **lowerCAmelCase_							,  ) ->      List[Any]:



      """simple docstring"""







      super().__init__(
          unk_token=lowerCAmelCase_							,  pad_token=lowerCAmelCase_							,  bos_token=lowerCAmelCase_							,  eos_token=lowerCAmelCase_							,  do_clean_text=lowerCAmelCase_							,  **lowerCAmelCase_							,  )
      if not os.path.isfile(lowerCAmelCase_):
           raise ValueError(
               f"""Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"""
               " model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`")
      if not os.path.isfile(lowerCAmelCase_):
           raise ValueError(
               f"""Can't find a emoji file at path '{emoji_file}'. To load the emoji information from a Google"""
               " pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`")
      a_       		=do_clean_text
      a_		,						a_		,						a_		,						a_       		=load_vocab_and_emoji(lowerCAmelCase_							,  lowerCAmelCase_)
      a_       		=SubWordJapaneseTokenizer(
          vocab=self.vocab							,  ids_to_tokens=self.ids_to_tokens							,  emoji=self.emoji)




 @property
 def       lowercase_  (   self) ->      int:



      """simple docstring"""







      return len(self.raw_vocab)




 def       lowercase_  (   self) ->      Optional[Any]:



      """simple docstring"""







      return dict(self.raw_vocab							,  **self.added_tokens_encoder)




 def       lowercase_  (   self							,  lowerCAmelCase_) ->      List[str]:



      """simple docstring"""







      return self.subword_tokenizer.tokenize(lowerCAmelCase_							,  clean=self.do_clean_text)




 def       lowercase_  (   self							,  lowerCAmelCase_) ->      Optional[int]:



      """simple docstring"""







      return self.vocab.get(lowerCAmelCase_							,  self.vocab.get(self.unk_token))




 def       lowercase_  (   self							,  lowerCAmelCase_) ->      List[str]:



      """simple docstring"""







      return self.subword_tokenizer.convert_id_to_token(lowerCAmelCase_)




 def       lowercase_  (   self							,  lowerCAmelCase_) ->      Optional[Any]:



      """simple docstring"""







      a_       		="".join(lowerCAmelCase_).strip()
      return out_string




 def       lowercase_  (   self							,  lowerCAmelCase_) ->      List[int]:



      """simple docstring"""







      a_       		=[]
      for is_user, text in conversation.iter_texts():
           input_ids.extend(self.encode(lowerCAmelCase_							,  add_special_tokens=lowerCAmelCase_) + [self.eos_token_id])

      if len(lowerCAmelCase_) > self.model_max_length:
           a_       		=input_ids[-self.model_max_length :]
      return input_ids




 def       lowercase_  (   self							,  lowerCAmelCase_							,  lowerCAmelCase_ = None) ->      Tuple[str]:



      """simple docstring"""







      a_       		=0
      if os.path.isdir(lowerCAmelCase_):
           a_       		=os.path.join(
               lowerCAmelCase_							,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])
           a_       		=os.path.join(
               lowerCAmelCase_							,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["emoji_file"])
      else:
           a_       		=(
               (filename_prefix + "-" if filename_prefix else "") + save_directory + VOCAB_FILES_NAMES["vocab_file"]
           )
           a_       		=(
               (filename_prefix + "-" if filename_prefix else "") + save_directory + VOCAB_FILES_NAMES["emoji_file"]
           )
      with open(lowerCAmelCase_							,  "w"							,  encoding="utf-8") as writer:
           for token_index, token in self.ids_to_tokens.items():
                if index != token_index:
                     logger.warning(
                         f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
                         " Please check that the vocabulary is not corrupted!")
                     a_       		=token_index
                writer.write(",".join(lowerCAmelCase_) + "\n")
                index += 1
      with open(lowerCAmelCase_							,  "w"							,  encoding="utf-8") as writer:
           json.dump(self.emoji							,  lowerCAmelCase_)
      return vocab_file, emoji_file







class 	UpperCAmelCase	(		__a):


 '''simple docstring'''




 def __init__(   self							,  lowerCAmelCase_							,  lowerCAmelCase_							,  lowerCAmelCase_) ->      str:



      """simple docstring"""







      a_       		=vocab  # same as swe
      a_       		=ids_to_tokens  # same as bpe
      a_       		=emoji
      a_       		=np.max([len(lowerCAmelCase_) for w in self.vocab.keys()])
      a_       		=re.compile(r"(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)")
      a_       		=re.compile(r"[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*")
      a_       		=re.compile(r"[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}")
      a_       		=re.compile(
          r"([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*")
      a_       		=re.compile(
          r"(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*")
      a_       		=re.compile(
          r"((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*")
      a_       		="─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿"
      a_       		="▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟"
      a_       		=str.maketrans({k: "<BLOCK>" for k in keisen + blocks})




 def __len__(   self) ->      Tuple:



      """simple docstring"""







      return len(self.ids_to_tokens)




 def       lowercase_  (   self							,  lowerCAmelCase_) ->      Any:



      """simple docstring"""







      a_       		=self.content_repattera.sub("<URL>"							,  lowerCAmelCase_)
      a_       		=self.content_repattera.sub("<EMAIL>"							,  lowerCAmelCase_)
      a_       		=self.content_repattera.sub("<TEL>"							,  lowerCAmelCase_)
      a_       		=self.content_repattera.sub("<DATE>"							,  lowerCAmelCase_)
      a_       		=self.content_repattera.sub("<DATE>"							,  lowerCAmelCase_)
      a_       		=self.content_repattera.sub("<PRICE>"							,  lowerCAmelCase_)
      a_       		=content.translate(self.content_transa)
      while "<BLOCK><BLOCK>" in content:
           a_       		=content.replace("<BLOCK><BLOCK>"							,  "<BLOCK>")
      return content




 def       lowercase_  (   self							,  lowerCAmelCase_							,  lowerCAmelCase_=False) ->      Union[str, Any]:



      """simple docstring"""







      a_       		=text.replace(" "							,  "<SP>")
      a_       		=text.replace("　"							,  "<SP>")
      a_       		=text.replace("\r\n"							,  "<BR>")
      a_       		=text.replace("\n"							,  "<BR>")
      a_       		=text.replace("\r"							,  "<BR>")
      a_       		=text.replace("\t"							,  "<TAB>")
      a_       		=text.replace("—"							,  "ー")
      a_       		=text.replace("−"							,  "ー")
      for k, v in self.emoji["emoji"].items():
           if k in text:
                a_       		=text.replace(lowerCAmelCase_							,  lowerCAmelCase_)
      if clean:
           a_       		=self.clean_text(lowerCAmelCase_)

      def check_simbol(lowerCAmelCase_):
           a_       		=x.encode()
           if len(lowerCAmelCase_) == 1 and len(lowerCAmelCase_) == 2:
                a_       		=(int(e[0]) << 8) + int(e[1])
                if (
                    (c >= 0xc2a1 and c <= 0xc2bf)
                    or (c >= 0xc780 and c <= 0xc783)
                    or (c >= 0xcab9 and c <= 0xcbbf)
                    or (c >= 0xcc80 and c <= 0xcda2)
                ):
                     return True
           return False

      def checkuae(lowerCAmelCase_):
           a_       		=x.encode()
           if len(lowerCAmelCase_) == 1 and len(lowerCAmelCase_) == 3:
                a_       		=(int(e[0]) << 1_6) + (int(e[1]) << 8) + int(e[2])
                if c >= 0xe2_8080 and c <= 0xe2_b07f:
                     return True
           return False

      a_       		=0
      a_       		=[]
      while pos < len(lowerCAmelCase_):
           a_       		=min(len(lowerCAmelCase_)							,  pos + self.maxlen + 1) if text[pos] == "<" else pos + 3
           a_       		=[]  # (token_id, token, pos)
           for e in range(lowerCAmelCase_							,  lowerCAmelCase_							,  -1):
                a_       		=text[pos:e]
                if wd in self.vocab:
                     if wd[0] == "<" and len(lowerCAmelCase_) > 2:
                          a_       		=[(self.vocab[wd], wd, e)]
                          break
                     else:
                          candidates.append((self.vocab[wd], wd, e))
           if len(lowerCAmelCase_) > 0:
                # the smallest token_id is adopted
                a_		,						a_		,						a_       		=sorted(lowerCAmelCase_							,  key=lambda lowerCAmelCase_: x[0])[0]
                result.append(lowerCAmelCase_)
                a_       		=e
           else:
                a_       		=pos + 1
                a_       		=text[pos:end]
                if check_simbol(lowerCAmelCase_):
                     result.append("<KIGOU>")
                elif checkuae(lowerCAmelCase_):
                     result.append("<U2000U2BFF>")
                else:
                     for i in wd.encode("utf-8"):
                          result.append("<|byte%d|>" % i)
                a_       		=end
      return result




 def       lowercase_  (   self							,  lowerCAmelCase_							,  lowerCAmelCase_="\n") ->      List[Any]:



      """simple docstring"""







      a_       		=[]
      a_       		=[]
      a_       		=self.ids_to_tokens[index][0]
      if word[:6] == "<|byte" and word[-2:] == "|>":
           byte_tokens.append(int(word[6:-2]))
      else:
           if len(lowerCAmelCase_) > 0:
                words.append(bytearray(lowerCAmelCase_).decode("utf-8"							,  errors="replace"))
                a_       		=[]
           if word[:7] == "<|emoji" and word[-2:] == "|>":
                words.append(self.emoji["emoji_inv"][word])
           elif word == "<SP>":
                words.append(" ")
           elif word == "<BR>":
                words.append(lowerCAmelCase_)
           elif word == "<TAB>":
                words.append("\t")
           elif word == "<BLOCK>":
                words.append("▀")
           elif word == "<KIGOU>":
                words.append("ǀ")
           elif word == "<U2000U2BFF>":
                words.append("‖")
           else:
                words.append(lowerCAmelCase_)
      if len(lowerCAmelCase_) > 0:
           words.append(bytearray(lowerCAmelCase_).decode("utf-8"							,  errors="replace"))
      a_       		="".join(lowerCAmelCase_)
      return text


