




import argparse
import json
import logging
import os
import sys
from unittest.mock import patch

from transformers.testing_utils import TestCasePlus, get_gpu_count, slow


_UpperCAmelCase	:						str			=     [
    os.path.join(os.path.dirname(__file__), dirname)
    for dirname in [
        "text-classification",
        "language-modeling",
        "summarization",
        "token-classification",
        "question-answering",
    ]
]
sys.path.extend(SRC_DIRS)


if SRC_DIRS is not None:
     import run_clm_flax
     import run_flax_glue
     import run_flax_ner
     import run_mlm_flax
     import run_qa
     import run_summarization_flax
     import run_ta_mlm_flax


logging.basicConfig(level=logging.DEBUG)

_UpperCAmelCase	:						Tuple			=     logging.getLogger()



def       UpperCAmelCase__  (     ):
  lowercase    :Dict      	=						argparse.ArgumentParser()
  parser.add_argument("-f"   )
  lowercase    :Union[str, Any]      	=						parser.parse_args()
  return args.f



def       UpperCAmelCase__  (     lowerCamelCase,			lowerCamelCase="eval"   ):
  lowercase    :Any      	=						os.path.join(lowerCamelCase,			F"{split}_results.json"   )
  if os.path.exists(lowerCamelCase   ):
    with open(lowerCamelCase,			"r"   ) as f:
      return json.load(lowerCamelCase   )
  raise ValueError(F"can't find {path}"   )


_UpperCAmelCase	:						Tuple			=     logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)







class    __lowerCAmelCase       (							lowerCAmelCase):






       def 					SCREAMING_SNAKE_CASE    (   self:					List[Any]				):
         lowercase    :Optional[Any]      	=						self.get_auto_remove_tmp_dir()
         lowercase    :Tuple      	=						F"\n            run_glue.py\n            --model_name_or_path distilbert-base-uncased\n            --output_dir {tmp_dir}\n            --train_file ./tests/fixtures/tests_samples/MRPC/train.csv\n            --validation_file ./tests/fixtures/tests_samples/MRPC/dev.csv\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n            --learning_rate=1e-4\n            --eval_steps=2\n            --warmup_steps=2\n            --seed=42\n            --max_seq_length=128\n            ".split()

         with patch.object(_lowerCAmelCase       ,					"argv"       ,					_lowerCAmelCase				):
           run_flax_glue.main()
           lowercase    :Dict      	=						get_results(_lowerCAmelCase				)
           self.assertGreaterEqual(result["eval_accuracy"]       ,					0.75				)






       @slow
       def 					SCREAMING_SNAKE_CASE    (   self:					List[str]				):
         lowercase    :Optional[int]      	=						self.get_auto_remove_tmp_dir()
         lowercase    :List[str]      	=						F"\n            run_clm_flax.py\n            --model_name_or_path distilgpt2\n            --train_file ./tests/fixtures/sample_text.txt\n            --validation_file ./tests/fixtures/sample_text.txt\n            --do_train\n            --do_eval\n            --block_size 128\n            --per_device_train_batch_size 4\n            --per_device_eval_batch_size 4\n            --num_train_epochs 2\n            --logging_steps 2 --eval_steps 2\n            --output_dir {tmp_dir}\n            --overwrite_output_dir\n            ".split()

         with patch.object(_lowerCAmelCase       ,					"argv"       ,					_lowerCAmelCase				):
           run_clm_flax.main()
           lowercase    :Dict      	=						get_results(_lowerCAmelCase				)
           self.assertLess(result["eval_perplexity"]       ,					1_00				)






       @slow
       def 					SCREAMING_SNAKE_CASE    (   self:					List[str]				):
         lowercase    :Dict      	=						self.get_auto_remove_tmp_dir()
         lowercase    :List[str]      	=						F"\n            run_summarization.py\n            --model_name_or_path t5-small\n            --train_file tests/fixtures/tests_samples/xsum/sample.json\n            --validation_file tests/fixtures/tests_samples/xsum/sample.json\n            --test_file tests/fixtures/tests_samples/xsum/sample.json\n            --output_dir {tmp_dir}\n            --overwrite_output_dir\n            --num_train_epochs=3\n            --warmup_steps=8\n            --do_train\n            --do_eval\n            --do_predict\n            --learning_rate=2e-4\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n            --predict_with_generate\n        ".split()

         with patch.object(_lowerCAmelCase       ,					"argv"       ,					_lowerCAmelCase				):
           run_summarization_flax.main()
           lowercase    :str      	=						get_results(_lowerCAmelCase       ,					split="test"				)
           self.assertGreaterEqual(result["test_rouge1"]       ,					10				)
           self.assertGreaterEqual(result["test_rouge2"]       ,					2				)
           self.assertGreaterEqual(result["test_rougeL"]       ,					7				)
           self.assertGreaterEqual(result["test_rougeLsum"]       ,					7				)






       @slow
       def 					SCREAMING_SNAKE_CASE    (   self:					List[Any]				):
         lowercase    :Dict      	=						self.get_auto_remove_tmp_dir()
         lowercase    :Tuple      	=						F"\n            run_mlm.py\n            --model_name_or_path distilroberta-base\n            --train_file ./tests/fixtures/sample_text.txt\n            --validation_file ./tests/fixtures/sample_text.txt\n            --output_dir {tmp_dir}\n            --overwrite_output_dir\n            --max_seq_length 128\n            --per_device_train_batch_size 4\n            --per_device_eval_batch_size 4\n            --logging_steps 2 --eval_steps 2\n            --do_train\n            --do_eval\n            --num_train_epochs=1\n        ".split()

         with patch.object(_lowerCAmelCase       ,					"argv"       ,					_lowerCAmelCase				):
           run_mlm_flax.main()
           lowercase    :Optional[int]      	=						get_results(_lowerCAmelCase				)
           self.assertLess(result["eval_perplexity"]       ,					42				)






       @slow
       def 					SCREAMING_SNAKE_CASE    (   self:					Dict				):
         lowercase    :Any      	=						self.get_auto_remove_tmp_dir()
         lowercase    :Dict      	=						F"\n            run_t5_mlm_flax.py\n            --model_name_or_path t5-small\n            --train_file ./tests/fixtures/sample_text.txt\n            --validation_file ./tests/fixtures/sample_text.txt\n            --do_train\n            --do_eval\n            --max_seq_length 128\n            --per_device_train_batch_size 4\n            --per_device_eval_batch_size 4\n            --num_train_epochs 2\n            --logging_steps 2 --eval_steps 2\n            --output_dir {tmp_dir}\n            --overwrite_output_dir\n            ".split()

         with patch.object(_lowerCAmelCase       ,					"argv"       ,					_lowerCAmelCase				):
           run_ta_mlm_flax.main()
           lowercase    :int      	=						get_results(_lowerCAmelCase				)
           self.assertGreaterEqual(result["eval_accuracy"]       ,					0.42				)






       @slow
       def 					SCREAMING_SNAKE_CASE    (   self:					Optional[int]				):
         # with so little data distributed training needs more epochs to get the score on par with 0/1 gpu
         lowercase    :Optional[Any]      	=						7 if get_gpu_count() > 1 else 2

         lowercase    :int      	=						self.get_auto_remove_tmp_dir()
         lowercase    :List[str]      	=						F"\n            run_flax_ner.py\n            --model_name_or_path bert-base-uncased\n            --train_file tests/fixtures/tests_samples/conll/sample.json\n            --validation_file tests/fixtures/tests_samples/conll/sample.json\n            --output_dir {tmp_dir}\n            --overwrite_output_dir\n            --do_train\n            --do_eval\n            --warmup_steps=2\n            --learning_rate=2e-4\n            --logging_steps 2 --eval_steps 2\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=2\n            --num_train_epochs={epochs}\n            --seed 7\n        ".split()

         with patch.object(_lowerCAmelCase       ,					"argv"       ,					_lowerCAmelCase				):
           run_flax_ner.main()
           lowercase    :Union[str, Any]      	=						get_results(_lowerCAmelCase				)
           self.assertGreaterEqual(result["eval_accuracy"]       ,					0.75				)
           self.assertGreaterEqual(result["eval_f1"]       ,					0.3				)






       @slow
       def 					SCREAMING_SNAKE_CASE    (   self:					str				):
         lowercase    :List[str]      	=						self.get_auto_remove_tmp_dir()
         lowercase    :Optional[Any]      	=						F"\n            run_qa.py\n            --model_name_or_path bert-base-uncased\n            --version_2_with_negative\n            --train_file tests/fixtures/tests_samples/SQUAD/sample.json\n            --validation_file tests/fixtures/tests_samples/SQUAD/sample.json\n            --output_dir {tmp_dir}\n            --overwrite_output_dir\n            --num_train_epochs=3\n            --warmup_steps=2\n            --do_train\n            --do_eval\n            --logging_steps 2 --eval_steps 2\n            --learning_rate=2e-4\n            --per_device_train_batch_size=2\n            --per_device_eval_batch_size=1\n        ".split()

         with patch.object(_lowerCAmelCase       ,					"argv"       ,					_lowerCAmelCase				):
           run_qa.main()
           lowercase    :List[Any]      	=						get_results(_lowerCAmelCase				)
           self.assertGreaterEqual(result["eval_f1"]       ,					30				)
           self.assertGreaterEqual(result["eval_exact"]       ,					30				)









import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


_UpperCAmelCase	:						Optional[int]			=     logging.get_logger(__name__)

_UpperCAmelCase	:						Union[str, Any]			=     {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_UpperCAmelCase	:						Optional[Any]			=     {
    "vocab_file": {
        "google/realm-cc-news-pretrained-embedder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-encoder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-scorer": (
            "https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-openqa": (
            "https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"
        ),
        "google/realm-orqa-nq-openqa": "https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt",
        "google/realm-orqa-nq-reader": "https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt",
        "google/realm-orqa-wq-openqa": "https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt",
        "google/realm-orqa-wq-reader": "https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt",
    },
    "tokenizer_file": {
        "google/realm-cc-news-pretrained-embedder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"
        ),
        "google/realm-cc-news-pretrained-encoder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"
        ),
        "google/realm-cc-news-pretrained-scorer": (
            "https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"
        ),
        "google/realm-cc-news-pretrained-openqa": (
            "https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"
        ),
        "google/realm-orqa-nq-openqa": (
            "https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-nq-reader": (
            "https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-wq-openqa": (
            "https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-wq-reader": (
            "https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"
        ),
    },
}

_UpperCAmelCase	:						Tuple			=     {
    "google/realm-cc-news-pretrained-embedder": 512,
    "google/realm-cc-news-pretrained-encoder": 512,
    "google/realm-cc-news-pretrained-scorer": 512,
    "google/realm-cc-news-pretrained-openqa": 512,
    "google/realm-orqa-nq-openqa": 512,
    "google/realm-orqa-nq-reader": 512,
    "google/realm-orqa-wq-openqa": 512,
    "google/realm-orqa-wq-reader": 512,
}

_UpperCAmelCase	:						Dict			=     {
    "google/realm-cc-news-pretrained-embedder": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-encoder": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-scorer": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-openqa": {"do_lower_case": True},
    "google/realm-orqa-nq-openqa": {"do_lower_case": True},
    "google/realm-orqa-nq-reader": {"do_lower_case": True},
    "google/realm-orqa-wq-openqa": {"do_lower_case": True},
    "google/realm-orqa-wq-reader": {"do_lower_case": True},
}




class    __lowerCAmelCase       (							lowerCAmelCase):
       _a 					=						VOCAB_FILES_NAMES
       _a 					=						PRETRAINED_VOCAB_FILES_MAP
       _a 					=						PRETRAINED_INIT_CONFIGURATION
       _a 					=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       _a 					=						RealmTokenizer






       def __init__(   self:					Tuple       ,					_lowerCAmelCase:					int=None       ,					_lowerCAmelCase:					List[Any]=None       ,					_lowerCAmelCase:					Optional[int]=True       ,					_lowerCAmelCase:					Optional[Any]="[UNK]"       ,					_lowerCAmelCase:					Dict="[SEP]"       ,					_lowerCAmelCase:					List[str]="[PAD]"       ,					_lowerCAmelCase:					Optional[int]="[CLS]"       ,					_lowerCAmelCase:					List[Any]="[MASK]"       ,					_lowerCAmelCase:					Tuple=True       ,					_lowerCAmelCase:					Tuple=None       ,					**_lowerCAmelCase:					Union[str, Any]       ,					):
         super().__init__(
             _lowerCAmelCase       ,					tokenizer_file=_lowerCAmelCase       ,					do_lower_case=_lowerCAmelCase       ,					unk_token=_lowerCAmelCase       ,					sep_token=_lowerCAmelCase       ,					pad_token=_lowerCAmelCase       ,					cls_token=_lowerCAmelCase       ,					mask_token=_lowerCAmelCase       ,					tokenize_chinese_chars=_lowerCAmelCase       ,					strip_accents=_lowerCAmelCase       ,					**_lowerCAmelCase       ,					)

         lowercase    :List[str]      	=						json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
         if (
             normalizer_state.get("lowercase"       ,					_lowerCAmelCase				) != do_lower_case
             or normalizer_state.get("strip_accents"       ,					_lowerCAmelCase				) != strip_accents
             or normalizer_state.get("handle_chinese_chars"       ,					_lowerCAmelCase				) != tokenize_chinese_chars
         ):
           lowercase    :List[str]      	=						getattr(_lowerCAmelCase       ,					normalizer_state.pop("type"				)				)
           lowercase    :List[str]      	=						do_lower_case
           lowercase    :Any      	=						strip_accents
           lowercase    :Tuple      	=						tokenize_chinese_chars
           lowercase    :Any      	=						normalizer_class(**_lowerCAmelCase				)

         lowercase    :Any      	=						do_lower_case






       def 					SCREAMING_SNAKE_CASE    (   self:					Any       ,					_lowerCAmelCase:					Optional[Any]       ,					**_lowerCAmelCase:					Optional[int]				):
         lowercase    :List[str]      	=						PaddingStrategy.MAX_LENGTH

         lowercase    :Optional[Any]      	=						text
         lowercase    :str      	=						kwargs.pop("text_pair"       ,					_lowerCAmelCase				)
         lowercase    :Any      	=						kwargs.pop("return_tensors"       ,					_lowerCAmelCase				)

         lowercase    :List[str]      	=						{
             "input_ids": [],
             "attention_mask": [],
             "token_type_ids": [],
         }

         for idx, candidate_text in enumerate(_lowerCAmelCase				):
           if batch_text_pair is not None:
             lowercase    :Any      	=						batch_text_pair[idx]
           else:
             lowercase    :Optional[Any]      	=						None

           lowercase    :Optional[Any]      	=						super().__call__(_lowerCAmelCase       ,					_lowerCAmelCase       ,					return_tensors=_lowerCAmelCase       ,					**_lowerCAmelCase				)

           lowercase    :Tuple      	=						encoded_candidates.get("input_ids"				)
           lowercase    :Optional[Any]      	=						encoded_candidates.get("attention_mask"				)
           lowercase    :int      	=						encoded_candidates.get("token_type_ids"				)

           if encoded_input_ids is not None:
             output_data["input_ids"].append(_lowerCAmelCase				)
           if encoded_attention_mask is not None:
             output_data["attention_mask"].append(_lowerCAmelCase				)
           if encoded_token_type_ids is not None:
             output_data["token_type_ids"].append(_lowerCAmelCase				)

         lowercase    :str      	=						{key: item for key, item in output_data.items() if len(_lowerCAmelCase				) != 0}

         return BatchEncoding(_lowerCAmelCase       ,					tensor_type=_lowerCAmelCase				)






       def 					SCREAMING_SNAKE_CASE    (   self:					Any       ,					_lowerCAmelCase:					Any       ,					_lowerCAmelCase:					Tuple=None				):
         lowercase    :Dict      	=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
           output += token_ids_a + [self.sep_token_id]

         return output






       def 					SCREAMING_SNAKE_CASE    (   self:					Tuple       ,					_lowerCAmelCase:					List[int]       ,					_lowerCAmelCase:					Optional[List[int]] = None				):
         lowercase    :List[str]      	=						[self.sep_token_id]
         lowercase    :List[Any]      	=						[self.cls_token_id]
         if token_ids_a is None:
           return len(cls + token_ids_a + sep				) * [0]
         return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]






       def 					SCREAMING_SNAKE_CASE    (   self:					Optional[Any]       ,					_lowerCAmelCase:					str       ,					_lowerCAmelCase:					Optional[str] = None				):
         lowercase    :Tuple      	=						self._tokenizer.model.save(_lowerCAmelCase       ,					name=_lowerCAmelCase				)
         return tuple(_lowerCAmelCase				)



