

import argparse
import datetime
import io
import itertools
import json
import math
import os
import platform
import re
import shlex
import subprocess
import sys
from pathlib import Path
from statistics import fmean

import pandas as pd
import torch
from tqdm import tqdm

import transformers


A_       						=							float("nan")





class 							_snake_case :
   def __init__( self			:      Any     ,SCREAMING_SNAKE_CASE__			:      Optional[int]							):
     SCREAMING_SNAKE_CASE:Union[str, Any]						      =					sys.stdout
     SCREAMING_SNAKE_CASE:List[Any]						      =					open(SCREAMING_SNAKE_CASE__     ,"a"							)
   def __getattr__( self			:      Union[str, Any]     ,SCREAMING_SNAKE_CASE__			:      Optional[Any]							):
     return getattr(self.stdout     ,SCREAMING_SNAKE_CASE__							)







   def     __UpperCamelCase						( self			:      Optional[int]     ,SCREAMING_SNAKE_CASE__			:      str							):
     self.stdout.write(SCREAMING_SNAKE_CASE__							)
     # strip tqdm codes
     self.file.write(re.sub(R"^.*\r"     ,""     ,SCREAMING_SNAKE_CASE__     ,0     ,re.M							)							)





def    A_  ( snake_case=80      ,       snake_case=False ):
  SCREAMING_SNAKE_CASE:Tuple						      =					[]

  # deal with critical env vars
  SCREAMING_SNAKE_CASE:Optional[int]						      =					["CUDA_VISIBLE_DEVICES"]
  for key in env_keys:
    SCREAMING_SNAKE_CASE:Any						      =					os.environ.get(snake_case      ,       snake_case )
    if val is not None:
      cmd.append(F'''{key}={val}''' )

    # python executable (not always needed if the script is executable)
  SCREAMING_SNAKE_CASE:Optional[int]						      =					sys.executable if full_python_path else sys.executable.split("/" )[-1]
  cmd.append(snake_case )

  # now the normal args
  cmd += list(map(shlex.quote      ,       sys.argv ) )

  # split up into up to MAX_WIDTH lines with shell multi-line escapes
  SCREAMING_SNAKE_CASE:str						      =					[]
  SCREAMING_SNAKE_CASE:str						      =					""
  while len(snake_case ) > 0:
    current_line += F'''{cmd.pop(0 )} '''
    if len(snake_case ) == 0 or len(snake_case ) + len(cmd[0] ) + 1 > max_width - 1:
      lines.append(snake_case )
      SCREAMING_SNAKE_CASE:Tuple						      =					""
  return "\\\n".join(snake_case )





def    A_  ( snake_case      ,       snake_case ):
  # unwrap multi-line input
  SCREAMING_SNAKE_CASE:List[Any]						      =					re.sub(r"[\\\n]+"      ,       " "      ,       args.base_cmd )

  # remove --output_dir if any and set our own
  SCREAMING_SNAKE_CASE:int						      =					re.sub("--output_dir\s+[^\s]+"      ,       ""      ,       args.base_cmd )
  args.base_cmd += F''' --output_dir {output_dir}'''

  # ensure we have --overwrite_output_dir
  SCREAMING_SNAKE_CASE:Union[str, Any]						      =					re.sub("--overwrite_output_dir\s+"      ,       ""      ,       args.base_cmd )
  args.base_cmd += " --overwrite_output_dir"

  return [sys.executable] + shlex.split(args.base_cmd )





def    A_  ( snake_case      ,       snake_case      ,       snake_case      ,       snake_case      ,       snake_case      ,       snake_case      ,       snake_case ):
  # Enable to debug everything but the run itself, to do it fast and see the progress.
  # This is useful for debugging the output formatting quickly - we can remove it later once
  # everybody is happy with the output
  if 0:
    import random
    from time import sleep

    sleep(0 )
    return dict(
        {k: random.uniform(0      ,       100 ) for k in metric_keys}      ,       **{target_metric_key: random.choice([nan, 10.31, 100.2, 55.6666, 222.2222_2222] )}      ,       )

  SCREAMING_SNAKE_CASE:Union[str, Any]						      =					subprocess.run(snake_case      ,       capture_output=snake_case      ,       text=snake_case )

  if verbose:
    print("STDOUT"      ,       result.stdout )
    print("STDERR"      ,       result.stderr )

  # save the streams
  SCREAMING_SNAKE_CASE:Optional[Any]						      =					variation.replace(" "      ,       "-" )
  with open(Path(snake_case ) / F'''log.{prefix}.stdout.txt'''      ,       "w" ) as f:
    f.write(result.stdout )
  with open(Path(snake_case ) / F'''log.{prefix}.stderr.txt'''      ,       "w" ) as f:
    f.write(result.stderr )

  if result.returncode != 0:
    if verbose:
      print("failed" )
    return {target_metric_key: nan}

  with io.open(F'''{output_dir}/all_results.json'''      ,       "r"      ,       encoding="utf-8" ) as f:
    SCREAMING_SNAKE_CASE:List[str]						      =					json.load(snake_case )

  # filter out just the keys we want
  return {k: v for k, v in metrics.items() if k in metric_keys}





def    A_  ( snake_case      ,       snake_case      ,       snake_case      ,       snake_case      ,       snake_case      ,       snake_case      ,       snake_case      ,       snake_case      ,       snake_case      ,       snake_case      ,       ):
  SCREAMING_SNAKE_CASE:str						      =					[]
  SCREAMING_SNAKE_CASE:List[str]						      =					[]
  SCREAMING_SNAKE_CASE:Any						      =					F'''{id}: {variation:<{longest_variation_len}}'''
  SCREAMING_SNAKE_CASE:Tuple						      =					F'''{preamble}: '''
  SCREAMING_SNAKE_CASE:Any						      =					set(report_metric_keys + [target_metric_key] )
  for i in tqdm(range(snake_case )      ,       desc=snake_case      ,       leave=snake_case ):
    SCREAMING_SNAKE_CASE:Dict						      =					process_run_single(
        snake_case      ,       snake_case      ,       snake_case      ,       snake_case      ,       snake_case      ,       snake_case      ,       snake_case )
    SCREAMING_SNAKE_CASE:Tuple						      =					single_run_metrics[target_metric_key]
    if not math.isnan(snake_case ):
      metrics.append(snake_case )
      results.append(snake_case )
      outcome += "✓"
    else:
      outcome += "✘"
  SCREAMING_SNAKE_CASE:Union[str, Any]						      =					F'''\33[2K\r{outcome}'''
  if len(snake_case ) > 0:
    SCREAMING_SNAKE_CASE:Dict						      =					{k: fmean([x[k] for x in metrics] ) for k in metrics[0].keys()}
    SCREAMING_SNAKE_CASE:Union[str, Any]						      =					round(mean_metrics[target_metric_key]      ,       2 )
    SCREAMING_SNAKE_CASE:int						      =					F'''{outcome} {mean_target}'''
    if len(snake_case ) > 1:
      results_str += F''' {tuple(round(snake_case      ,       2 ) for x in results )}'''
    print(snake_case )
    SCREAMING_SNAKE_CASE:List[str]						      =					variation
    return mean_metrics
  else:
    print(snake_case )
    return {variation_key: variation, target_metric_key: nan}





def    A_  ( ):
  SCREAMING_SNAKE_CASE:int						      =					torch.cuda.get_device_properties(torch.device("cuda" ) )
  return F'''
Datetime    : {datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S" )}

Software:
transformers: {transformers.__version__}
torch       : {torch.__version__}
cuda        : {torch.version.cuda}
python      : {platform.python_version()}

Hardware:
{torch.cuda.device_count()} GPUs      : {properties.name}, {properties.total_memory/2**30:0.2f}GB
'''





def    A_  ( snake_case      ,       snake_case      ,       snake_case      ,       snake_case      ,       snake_case ):
  SCREAMING_SNAKE_CASE:Optional[Any]						      =					pd.DataFrame(snake_case )
  SCREAMING_SNAKE_CASE:str						      =					"variation"
  SCREAMING_SNAKE_CASE:str						      =					"diff_%"

  SCREAMING_SNAKE_CASE:Union[str, Any]						      =					nan
  if base_variation is not None and len(df[df[variation_key] == base_variation] ):
    # this may still return nan
    SCREAMING_SNAKE_CASE:Any						      =					df.loc[df[variation_key] == base_variation][target_metric_key].item()
  if math.isnan(snake_case ):
    # as a fallback, use the minimal value as the sentinel
    SCREAMING_SNAKE_CASE:Any						      =					df.loc[df[target_metric_key] != nan][target_metric_key].min()

  # create diff column if possible
  if not math.isnan(snake_case ):
    SCREAMING_SNAKE_CASE:Tuple						      =					df.apply(
        lambda snake_case : round(100 * (r[target_metric_key] - sentinel_value) / sentinel_value )
        if not math.isnan(r[target_metric_key] )
        else 0      ,       axis="columns"      ,       )

  # re-order columns
  SCREAMING_SNAKE_CASE:Tuple						      =					[variation_key, target_metric_key, diff_key, *report_metric_keys]
  SCREAMING_SNAKE_CASE:Union[str, Any]						      =					df.reindex(snake_case      ,       axis="columns" )  # reorder cols

  # capitalize
  SCREAMING_SNAKE_CASE:str						      =					df.rename(str.capitalize      ,       axis="columns" )

  # make the cols as narrow as possible
  SCREAMING_SNAKE_CASE:int						      =					df.rename(lambda snake_case : c.replace("_"      ,       "<br>" )      ,       axis="columns" )
  SCREAMING_SNAKE_CASE:Dict						      =					df.rename(lambda snake_case : c.replace("_"      ,       "\n" )      ,       axis="columns" )

  SCREAMING_SNAKE_CASE:List[Any]						      =					["", "Copy between the cut-here-lines and paste as is to github or a forum"]
  report += ["----------8<-----------------8<--------"]
  report += ["*** Results:", df_github.to_markdown(index=snake_case      ,       floatfmt=".2f" )]
  report += ["```"]
  report += ["*** Setup:", get_versions()]
  report += ["*** The benchmark command line was:", get_original_command()]
  report += ["```"]
  report += ["----------8<-----------------8<--------"]
  report += ["*** Results (console):", df_console.to_markdown(index=snake_case      ,       floatfmt=".2f" )]

  print("\n\n".join(snake_case ) )





def    A_  ( ):
  SCREAMING_SNAKE_CASE:Union[str, Any]						      =					argparse.ArgumentParser()
  parser.add_argument(
      "--base-cmd"      ,       default=snake_case      ,       type=snake_case      ,       required=snake_case      ,       help="Base cmd"      ,       )
  parser.add_argument(
      "--variations"      ,       default=snake_case      ,       type=snake_case      ,       nargs="+"      ,       required=snake_case      ,       help="Multi-dimensional variations, example: '|--fp16|--bf16' '|--tf32'"      ,       )
  parser.add_argument(
      "--base-variation"      ,       default=snake_case      ,       type=snake_case      ,       help="Baseline variation to compare to. if None the minimal target value will be used to compare against"      ,       )
  parser.add_argument(
      "--target-metric-key"      ,       default=snake_case      ,       type=snake_case      ,       required=snake_case      ,       help="Target metric key in output_dir/all_results.json, e.g., train_samples_per_second"      ,       )
  parser.add_argument(
      "--report-metric-keys"      ,       default=""      ,       type=snake_case      ,       help="Report metric keys - other metric keys from output_dir/all_results.json to report, e.g., train_loss. Use a single argument e.g., 'train_loss train_samples"      ,       )
  parser.add_argument(
      "--repeat-times"      ,       default=1      ,       type=snake_case      ,       help="How many times to re-run each variation - an average will be reported"      ,       )
  parser.add_argument(
      "--output_dir"      ,       default="output_benchmark"      ,       type=snake_case      ,       help="The output directory where all the benchmark reports will go to and additionally this directory will be used to override --output_dir in the script that is being benchmarked"      ,       )
  parser.add_argument(
      "--verbose"      ,       default=snake_case      ,       action="store_true"      ,       help="Whether to show the outputs of each run or just the benchmark progress"      ,       )
  SCREAMING_SNAKE_CASE:int						      =					parser.parse_args()

  SCREAMING_SNAKE_CASE:int						      =					args.output_dir
  Path(snake_case ).mkdir(exist_ok=snake_case )
  SCREAMING_SNAKE_CASE:Optional[int]						      =					get_base_command(snake_case      ,       snake_case )

  # split each dimension into its --foo variations
  SCREAMING_SNAKE_CASE:Union[str, Any]						      =					[list(map(str.strip      ,       re.split(r"\|"      ,       snake_case ) ) ) for x in args.variations]
  # build a cartesian product of dimensions and convert those back into cmd-line arg strings,
  # while stripping white space for inputs that were empty
  SCREAMING_SNAKE_CASE:str						      =					list(map(str.strip      ,       map(" ".join      ,       itertools.product(*snake_case ) ) ) )
  SCREAMING_SNAKE_CASE:Dict						      =					max(len(snake_case ) for x in variations )

  # split wanted keys
  SCREAMING_SNAKE_CASE:Any						      =					args.report_metric_keys.split()

  # capture prints into a log file for convenience
  SCREAMING_SNAKE_CASE:Any						      =					F'''benchmark-report-{datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S" )}.txt'''
  print(F'''\nNote: each run\'s output is also logged under {output_dir}/log.*.std*.txt''' )
  print(F'''and this script\'s output is also piped into {report_fn}''' )

  SCREAMING_SNAKE_CASE:Any						      =					Tee(snake_case )

  print(F'''\n*** Running {len(snake_case )} benchmarks:''' )
  print(F'''Base command: {" ".join(snake_case )}''' )

  SCREAMING_SNAKE_CASE:str						      =					"variation"
  SCREAMING_SNAKE_CASE:Tuple						      =					[]
  for id, variation in enumerate(tqdm(snake_case      ,       desc="Total completion: "      ,       leave=snake_case ) ):
    SCREAMING_SNAKE_CASE:List[str]						      =					base_cmd + variation.split()
    results.append(
        process_run(
            id + 1      ,       snake_case      ,       snake_case      ,       snake_case      ,       snake_case      ,       args.target_metric_key      ,       snake_case      ,       args.repeat_times      ,       snake_case      ,       args.verbose      ,       ) )

  process_results(snake_case      ,       args.target_metric_key      ,       snake_case      ,       args.base_variation      ,       snake_case )


if __name__ == "__main__":
    main()









'''simple docstring'''

import os
import tempfile
import unittest

from transformers import FlaubertConfig, is_torch_available
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import (
        FlaubertForMultipleChoice,
        FlaubertForQuestionAnswering,
        FlaubertForQuestionAnsweringSimple,
        FlaubertForSequenceClassification,
        FlaubertForTokenClassification,
        FlaubertModel,
        FlaubertWithLMHeadModel,
    )
    from transformers.models.flaubert.modeling_flaubert import FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST





class 							_snake_case (       _a						):
   def __init__( self			:      Tuple     ,SCREAMING_SNAKE_CASE__			:      Optional[int]     ,SCREAMING_SNAKE_CASE__			:      Tuple=13     ,SCREAMING_SNAKE_CASE__			:      Optional[int]=7     ,SCREAMING_SNAKE_CASE__			:      List[Any]=True     ,SCREAMING_SNAKE_CASE__			:      List[Any]=True     ,SCREAMING_SNAKE_CASE__			:      str=True     ,SCREAMING_SNAKE_CASE__			:      Union[str, Any]=True     ,SCREAMING_SNAKE_CASE__			:      Any=True     ,SCREAMING_SNAKE_CASE__			:      Optional[Any]=False     ,SCREAMING_SNAKE_CASE__			:      Union[str, Any]=False     ,SCREAMING_SNAKE_CASE__			:      int=False     ,SCREAMING_SNAKE_CASE__			:      int=2     ,SCREAMING_SNAKE_CASE__			:      Any=99     ,SCREAMING_SNAKE_CASE__			:      Optional[int]=0     ,SCREAMING_SNAKE_CASE__			:      Optional[Any]=32     ,SCREAMING_SNAKE_CASE__			:      List[str]=5     ,SCREAMING_SNAKE_CASE__			:      Tuple=4     ,SCREAMING_SNAKE_CASE__			:      Tuple=0.1     ,SCREAMING_SNAKE_CASE__			:      str=0.1     ,SCREAMING_SNAKE_CASE__			:      Dict=512     ,SCREAMING_SNAKE_CASE__			:      Optional[int]=12     ,SCREAMING_SNAKE_CASE__			:      List[Any]=2     ,SCREAMING_SNAKE_CASE__			:      Optional[Any]=0.02     ,SCREAMING_SNAKE_CASE__			:      Union[str, Any]=3     ,SCREAMING_SNAKE_CASE__			:      Dict=4     ,SCREAMING_SNAKE_CASE__			:      Dict="last"     ,SCREAMING_SNAKE_CASE__			:      Optional[Any]=None     ,SCREAMING_SNAKE_CASE__			:      List[Any]=None     ,):
     SCREAMING_SNAKE_CASE:Union[str, Any]						      =					parent
     SCREAMING_SNAKE_CASE:Dict						      =					batch_size
     SCREAMING_SNAKE_CASE:int						      =					seq_length
     SCREAMING_SNAKE_CASE:Any						      =					is_training
     SCREAMING_SNAKE_CASE:Union[str, Any]						      =					use_input_lengths
     SCREAMING_SNAKE_CASE:List[Any]						      =					use_token_type_ids
     SCREAMING_SNAKE_CASE:Dict						      =					use_labels
     SCREAMING_SNAKE_CASE:Dict						      =					gelu_activation
     SCREAMING_SNAKE_CASE:Dict						      =					sinusoidal_embeddings
     SCREAMING_SNAKE_CASE:List[str]						      =					causal
     SCREAMING_SNAKE_CASE:List[Any]						      =					asm
     SCREAMING_SNAKE_CASE:List[str]						      =					n_langs
     SCREAMING_SNAKE_CASE:Optional[Any]						      =					vocab_size
     SCREAMING_SNAKE_CASE:Optional[int]						      =					n_special
     SCREAMING_SNAKE_CASE:str						      =					hidden_size
     SCREAMING_SNAKE_CASE:Dict						      =					num_hidden_layers
     SCREAMING_SNAKE_CASE:Dict						      =					num_attention_heads
     SCREAMING_SNAKE_CASE:Optional[Any]						      =					hidden_dropout_prob
     SCREAMING_SNAKE_CASE:Tuple						      =					attention_probs_dropout_prob
     SCREAMING_SNAKE_CASE:Tuple						      =					max_position_embeddings
     SCREAMING_SNAKE_CASE:List[str]						      =					type_vocab_size
     SCREAMING_SNAKE_CASE:int						      =					type_sequence_label_size
     SCREAMING_SNAKE_CASE:Any						      =					initializer_range
     SCREAMING_SNAKE_CASE:str						      =					num_labels
     SCREAMING_SNAKE_CASE:Dict						      =					num_choices
     SCREAMING_SNAKE_CASE:Any						      =					summary_type
     SCREAMING_SNAKE_CASE:int						      =					use_proj
     SCREAMING_SNAKE_CASE:int						      =					scope
   def     __UpperCamelCase						( self			:      Optional[int]							):
     SCREAMING_SNAKE_CASE:int						      =					ids_tensor([self.batch_size, self.seq_length]     ,self.vocab_size							)
     SCREAMING_SNAKE_CASE:str						      =					random_attention_mask([self.batch_size, self.seq_length]							)

     SCREAMING_SNAKE_CASE:Tuple						      =					None
     if self.use_input_lengths:
       SCREAMING_SNAKE_CASE:List[Any]						      =					(
           ids_tensor([self.batch_size]     ,vocab_size=2							) + self.seq_length - 2
       )  # small variation of seq_length

     SCREAMING_SNAKE_CASE:Any						      =					None
     if self.use_token_type_ids:
       SCREAMING_SNAKE_CASE:Optional[int]						      =					ids_tensor([self.batch_size, self.seq_length]     ,self.n_langs							)

     SCREAMING_SNAKE_CASE:Dict						      =					None
     SCREAMING_SNAKE_CASE:List[Any]						      =					None
     SCREAMING_SNAKE_CASE:List[Any]						      =					None
     if self.use_labels:
       SCREAMING_SNAKE_CASE:Union[str, Any]						      =					ids_tensor([self.batch_size]     ,self.type_sequence_label_size							)
       SCREAMING_SNAKE_CASE:str						      =					ids_tensor([self.batch_size, self.seq_length]     ,self.num_labels							)
       SCREAMING_SNAKE_CASE:int						      =					ids_tensor([self.batch_size]     ,2							).float()
       SCREAMING_SNAKE_CASE:Optional[Any]						      =					ids_tensor([self.batch_size]     ,self.num_choices							)

     SCREAMING_SNAKE_CASE:List[Any]						      =					self.get_config()

     return (
         config,
         input_ids,
         token_type_ids,
         input_lengths,
         sequence_labels,
         token_labels,
         is_impossible_labels,
         choice_labels,
         input_mask,
     )
   def     __UpperCamelCase						( self			:      Optional[int]							):
     return FlaubertConfig(
         vocab_size=self.vocab_size     ,n_special=self.n_special     ,emb_dim=self.hidden_size     ,n_layers=self.num_hidden_layers     ,n_heads=self.num_attention_heads     ,dropout=self.hidden_dropout_prob     ,attention_dropout=self.attention_probs_dropout_prob     ,gelu_activation=self.gelu_activation     ,sinusoidal_embeddings=self.sinusoidal_embeddings     ,asm=self.asm     ,causal=self.causal     ,n_langs=self.n_langs     ,max_position_embeddings=self.max_position_embeddings     ,initializer_range=self.initializer_range     ,summary_type=self.summary_type     ,use_proj=self.use_proj     ,)
   def     __UpperCamelCase						( self			:      Any     ,SCREAMING_SNAKE_CASE__			:      Union[str, Any]     ,SCREAMING_SNAKE_CASE__			:      str     ,SCREAMING_SNAKE_CASE__			:      str     ,SCREAMING_SNAKE_CASE__			:      Dict     ,SCREAMING_SNAKE_CASE__			:      List[str]     ,SCREAMING_SNAKE_CASE__			:      Optional[int]     ,SCREAMING_SNAKE_CASE__			:      Tuple     ,SCREAMING_SNAKE_CASE__			:      Optional[Any]     ,SCREAMING_SNAKE_CASE__			:      List[Any]     ,):
     SCREAMING_SNAKE_CASE:Union[str, Any]						      =					FlaubertModel(config=SCREAMING_SNAKE_CASE__							)
     model.to(SCREAMING_SNAKE_CASE__							)
     model.eval()
     SCREAMING_SNAKE_CASE:Any						      =					model(SCREAMING_SNAKE_CASE__     ,lengths=SCREAMING_SNAKE_CASE__     ,langs=SCREAMING_SNAKE_CASE__							)
     SCREAMING_SNAKE_CASE:Any						      =					model(SCREAMING_SNAKE_CASE__     ,langs=SCREAMING_SNAKE_CASE__							)
     SCREAMING_SNAKE_CASE:Optional[int]						      =					model(SCREAMING_SNAKE_CASE__							)
     self.parent.assertEqual(result.last_hidden_state.shape     ,(self.batch_size, self.seq_length, self.hidden_size)							)
   def     __UpperCamelCase						( self			:      Dict     ,SCREAMING_SNAKE_CASE__			:      Optional[int]     ,SCREAMING_SNAKE_CASE__			:      Union[str, Any]     ,SCREAMING_SNAKE_CASE__			:      Union[str, Any]     ,SCREAMING_SNAKE_CASE__			:      Any     ,SCREAMING_SNAKE_CASE__			:      Tuple     ,SCREAMING_SNAKE_CASE__			:      Any     ,SCREAMING_SNAKE_CASE__			:      Union[str, Any]     ,SCREAMING_SNAKE_CASE__			:      Any     ,SCREAMING_SNAKE_CASE__			:      str     ,):
     SCREAMING_SNAKE_CASE:Optional[int]						      =					FlaubertWithLMHeadModel(SCREAMING_SNAKE_CASE__							)
     model.to(SCREAMING_SNAKE_CASE__							)
     model.eval()

     SCREAMING_SNAKE_CASE:Optional[int]						      =					model(SCREAMING_SNAKE_CASE__     ,token_type_ids=SCREAMING_SNAKE_CASE__     ,labels=SCREAMING_SNAKE_CASE__							)
     self.parent.assertEqual(result.loss.shape     ,()							)
     self.parent.assertEqual(result.logits.shape     ,(self.batch_size, self.seq_length, self.vocab_size)							)
   def     __UpperCamelCase						( self			:      Any     ,SCREAMING_SNAKE_CASE__			:      List[str]     ,SCREAMING_SNAKE_CASE__			:      List[str]     ,SCREAMING_SNAKE_CASE__			:      str     ,SCREAMING_SNAKE_CASE__			:      List[Any]     ,SCREAMING_SNAKE_CASE__			:      Optional[Any]     ,SCREAMING_SNAKE_CASE__			:      Union[str, Any]     ,SCREAMING_SNAKE_CASE__			:      Union[str, Any]     ,SCREAMING_SNAKE_CASE__			:      Optional[Any]     ,SCREAMING_SNAKE_CASE__			:      Optional[int]     ,):
     SCREAMING_SNAKE_CASE:Optional[int]						      =					FlaubertForQuestionAnsweringSimple(SCREAMING_SNAKE_CASE__							)
     model.to(SCREAMING_SNAKE_CASE__							)
     model.eval()

     SCREAMING_SNAKE_CASE:Union[str, Any]						      =					model(SCREAMING_SNAKE_CASE__							)

     SCREAMING_SNAKE_CASE:Dict						      =					model(SCREAMING_SNAKE_CASE__     ,start_positions=SCREAMING_SNAKE_CASE__     ,end_positions=SCREAMING_SNAKE_CASE__							)
     self.parent.assertEqual(result.start_logits.shape     ,(self.batch_size, self.seq_length)							)
     self.parent.assertEqual(result.end_logits.shape     ,(self.batch_size, self.seq_length)							)
   def     __UpperCamelCase						( self			:      List[str]     ,SCREAMING_SNAKE_CASE__			:      List[str]     ,SCREAMING_SNAKE_CASE__			:      Optional[Any]     ,SCREAMING_SNAKE_CASE__			:      Optional[int]     ,SCREAMING_SNAKE_CASE__			:      Tuple     ,SCREAMING_SNAKE_CASE__			:      Any     ,SCREAMING_SNAKE_CASE__			:      Union[str, Any]     ,SCREAMING_SNAKE_CASE__			:      Dict     ,SCREAMING_SNAKE_CASE__			:      Any     ,SCREAMING_SNAKE_CASE__			:      Tuple     ,):
     SCREAMING_SNAKE_CASE:str						      =					FlaubertForQuestionAnswering(SCREAMING_SNAKE_CASE__							)
     model.to(SCREAMING_SNAKE_CASE__							)
     model.eval()

     SCREAMING_SNAKE_CASE:List[Any]						      =					model(SCREAMING_SNAKE_CASE__							)

     SCREAMING_SNAKE_CASE:Optional[Any]						      =					model(
         SCREAMING_SNAKE_CASE__     ,start_positions=SCREAMING_SNAKE_CASE__     ,end_positions=SCREAMING_SNAKE_CASE__     ,cls_index=SCREAMING_SNAKE_CASE__     ,is_impossible=SCREAMING_SNAKE_CASE__     ,p_mask=SCREAMING_SNAKE_CASE__     ,)

     SCREAMING_SNAKE_CASE:str						      =					model(
         SCREAMING_SNAKE_CASE__     ,start_positions=SCREAMING_SNAKE_CASE__     ,end_positions=SCREAMING_SNAKE_CASE__     ,cls_index=SCREAMING_SNAKE_CASE__     ,is_impossible=SCREAMING_SNAKE_CASE__     ,)

     ((SCREAMING_SNAKE_CASE)      ,    ):List[Any]						      =					result_with_labels.to_tuple()

     SCREAMING_SNAKE_CASE:Optional[int]						      =					model(SCREAMING_SNAKE_CASE__     ,start_positions=SCREAMING_SNAKE_CASE__     ,end_positions=SCREAMING_SNAKE_CASE__							)

     ((SCREAMING_SNAKE_CASE)      ,    ):List[Any]						      =					result_with_labels.to_tuple()

     self.parent.assertEqual(result_with_labels.loss.shape     ,()							)
     self.parent.assertEqual(result.start_top_log_probs.shape     ,(self.batch_size, model.config.start_n_top)							)
     self.parent.assertEqual(result.start_top_index.shape     ,(self.batch_size, model.config.start_n_top)							)
     self.parent.assertEqual(
         result.end_top_log_probs.shape     ,(self.batch_size, model.config.start_n_top * model.config.end_n_top)							)
     self.parent.assertEqual(
         result.end_top_index.shape     ,(self.batch_size, model.config.start_n_top * model.config.end_n_top)							)
     self.parent.assertEqual(result.cls_logits.shape     ,(self.batch_size,)							)
   def     __UpperCamelCase						( self			:      Tuple     ,SCREAMING_SNAKE_CASE__			:      Union[str, Any]     ,SCREAMING_SNAKE_CASE__			:      Optional[Any]     ,SCREAMING_SNAKE_CASE__			:      Optional[Any]     ,SCREAMING_SNAKE_CASE__			:      str     ,SCREAMING_SNAKE_CASE__			:      str     ,SCREAMING_SNAKE_CASE__			:      Any     ,SCREAMING_SNAKE_CASE__			:      Union[str, Any]     ,SCREAMING_SNAKE_CASE__			:      Tuple     ,SCREAMING_SNAKE_CASE__			:      Dict     ,):
     SCREAMING_SNAKE_CASE:List[Any]						      =					FlaubertForSequenceClassification(SCREAMING_SNAKE_CASE__							)
     model.to(SCREAMING_SNAKE_CASE__							)
     model.eval()

     SCREAMING_SNAKE_CASE:Tuple						      =					model(SCREAMING_SNAKE_CASE__							)
     SCREAMING_SNAKE_CASE:Union[str, Any]						      =					model(SCREAMING_SNAKE_CASE__     ,labels=SCREAMING_SNAKE_CASE__							)

     self.parent.assertEqual(result.loss.shape     ,()							)
     self.parent.assertEqual(result.logits.shape     ,(self.batch_size, self.type_sequence_label_size)							)
   def     __UpperCamelCase						( self			:      int     ,SCREAMING_SNAKE_CASE__			:      List[Any]     ,SCREAMING_SNAKE_CASE__			:      Union[str, Any]     ,SCREAMING_SNAKE_CASE__			:      List[Any]     ,SCREAMING_SNAKE_CASE__			:      List[Any]     ,SCREAMING_SNAKE_CASE__			:      List[Any]     ,SCREAMING_SNAKE_CASE__			:      Any     ,SCREAMING_SNAKE_CASE__			:      Union[str, Any]     ,SCREAMING_SNAKE_CASE__			:      List[Any]     ,SCREAMING_SNAKE_CASE__			:      Optional[Any]     ,):
     SCREAMING_SNAKE_CASE:List[str]						      =					self.num_labels
     SCREAMING_SNAKE_CASE:Optional[int]						      =					FlaubertForTokenClassification(SCREAMING_SNAKE_CASE__							)
     model.to(SCREAMING_SNAKE_CASE__							)
     model.eval()

     SCREAMING_SNAKE_CASE:Optional[Any]						      =					model(SCREAMING_SNAKE_CASE__     ,attention_mask=SCREAMING_SNAKE_CASE__     ,labels=SCREAMING_SNAKE_CASE__							)
     self.parent.assertEqual(result.logits.shape     ,(self.batch_size, self.seq_length, self.num_labels)							)
   def     __UpperCamelCase						( self			:      Any     ,SCREAMING_SNAKE_CASE__			:      str     ,SCREAMING_SNAKE_CASE__			:      Dict     ,SCREAMING_SNAKE_CASE__			:      Optional[Any]     ,SCREAMING_SNAKE_CASE__			:      Optional[int]     ,SCREAMING_SNAKE_CASE__			:      Union[str, Any]     ,SCREAMING_SNAKE_CASE__			:      List[Any]     ,SCREAMING_SNAKE_CASE__			:      str     ,SCREAMING_SNAKE_CASE__			:      Tuple     ,SCREAMING_SNAKE_CASE__			:      List[str]     ,):
     SCREAMING_SNAKE_CASE:List[str]						      =					self.num_choices
     SCREAMING_SNAKE_CASE:Any						      =					FlaubertForMultipleChoice(config=SCREAMING_SNAKE_CASE__							)
     model.to(SCREAMING_SNAKE_CASE__							)
     model.eval()
     SCREAMING_SNAKE_CASE:str						      =					input_ids.unsqueeze(1							).expand(-1     ,self.num_choices     ,-1							).contiguous()
     SCREAMING_SNAKE_CASE:Dict						      =					token_type_ids.unsqueeze(1							).expand(-1     ,self.num_choices     ,-1							).contiguous()
     SCREAMING_SNAKE_CASE:List[Any]						      =					input_mask.unsqueeze(1							).expand(-1     ,self.num_choices     ,-1							).contiguous()
     SCREAMING_SNAKE_CASE:int						      =					model(
         SCREAMING_SNAKE_CASE__     ,attention_mask=SCREAMING_SNAKE_CASE__     ,token_type_ids=SCREAMING_SNAKE_CASE__     ,labels=SCREAMING_SNAKE_CASE__     ,)
     self.parent.assertEqual(result.logits.shape     ,(self.batch_size, self.num_choices)							)







   def     __UpperCamelCase						( self			:      Any							):
     SCREAMING_SNAKE_CASE:str						      =					self.prepare_config_and_inputs()
     (
         (
         SCREAMING_SNAKE_CASE
     )      ,    (
         SCREAMING_SNAKE_CASE
     )      ,    (
         SCREAMING_SNAKE_CASE
     )      ,    (
         SCREAMING_SNAKE_CASE
     )      ,    (
         SCREAMING_SNAKE_CASE
     )      ,    (
         SCREAMING_SNAKE_CASE
     )      ,    (
         SCREAMING_SNAKE_CASE
     )      ,    (
         SCREAMING_SNAKE_CASE
     )      ,    (
         SCREAMING_SNAKE_CASE
     )      ,    
     ):Tuple						      =					config_and_inputs
     SCREAMING_SNAKE_CASE:Tuple						      =					{
         "input_ids": input_ids,
         "token_type_ids": token_type_ids,
         "lengths": input_lengths,
         "attention_mask": input_mask,
     }
     return config, inputs_dict





@require_torch
class 							_snake_case (       _a	,  _a	,  unittest.TestCase						):
   _A       :     int       =					(
       (
           FlaubertModel,
           FlaubertWithLMHeadModel,
           FlaubertForQuestionAnswering,
           FlaubertForQuestionAnsweringSimple,
           FlaubertForSequenceClassification,
           FlaubertForTokenClassification,
           FlaubertForMultipleChoice,
       )
       if is_torch_available()
       else ()
   )
   _A       :     Optional[Any]       =					(
       {
           '''feature-extraction''': FlaubertModel,
           '''fill-mask''': FlaubertWithLMHeadModel,
           '''question-answering''': FlaubertForQuestionAnsweringSimple,
           '''text-classification''': FlaubertForSequenceClassification,
           '''token-classification''': FlaubertForTokenClassification,
           '''zero-shot''': FlaubertForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )
   def     __UpperCamelCase						( self			:      str     ,SCREAMING_SNAKE_CASE__			:      Optional[int]     ,SCREAMING_SNAKE_CASE__			:      Tuple     ,SCREAMING_SNAKE_CASE__			:      str     ,SCREAMING_SNAKE_CASE__			:      Optional[int]     ,SCREAMING_SNAKE_CASE__			:      Optional[Any]							):
     if (
         pipeline_test_casse_name == "QAPipelineTests"
         and tokenizer_name is not None
         and not tokenizer_name.endswith("Fast"							)
     ):
       # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
       # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
       # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
       return True

     return False
   def     __UpperCamelCase						( self			:      Union[str, Any]     ,SCREAMING_SNAKE_CASE__			:      Optional[int]     ,SCREAMING_SNAKE_CASE__			:      Union[str, Any]     ,SCREAMING_SNAKE_CASE__			:      int=False							):
     SCREAMING_SNAKE_CASE:Any						      =					super()._prepare_for_class(SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__     ,return_labels=SCREAMING_SNAKE_CASE__							)

     if return_labels:
       if model_class.__name__ == "FlaubertForQuestionAnswering":
         SCREAMING_SNAKE_CASE:Dict						      =					torch.zeros(
             self.model_tester.batch_size     ,dtype=torch.long     ,device=SCREAMING_SNAKE_CASE__							)
         SCREAMING_SNAKE_CASE:int						      =					torch.zeros(
             self.model_tester.batch_size     ,dtype=torch.long     ,device=SCREAMING_SNAKE_CASE__							)

     return inputs_dict
   def     __UpperCamelCase						( self			:      Optional[int]							):
     SCREAMING_SNAKE_CASE:int						      =					FlaubertModelTester(self							)
     SCREAMING_SNAKE_CASE:Dict						      =					ConfigTester(self     ,config_class=SCREAMING_SNAKE_CASE__     ,emb_dim=37							)
   def     __UpperCamelCase						( self			:      Any							):
     self.config_tester.run_common_tests()
   def     __UpperCamelCase						( self			:      Any							):
     SCREAMING_SNAKE_CASE:Any						      =					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_flaubert_model(*SCREAMING_SNAKE_CASE__							)
   def     __UpperCamelCase						( self			:      Optional[int]							):
     SCREAMING_SNAKE_CASE:Dict						      =					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_flaubert_lm_head(*SCREAMING_SNAKE_CASE__							)
   def     __UpperCamelCase						( self			:      str							):
     SCREAMING_SNAKE_CASE:str						      =					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_flaubert_simple_qa(*SCREAMING_SNAKE_CASE__							)
   def     __UpperCamelCase						( self			:      Any							):
     SCREAMING_SNAKE_CASE:Dict						      =					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_flaubert_qa(*SCREAMING_SNAKE_CASE__							)
   def     __UpperCamelCase						( self			:      Optional[Any]							):
     SCREAMING_SNAKE_CASE:Dict						      =					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_flaubert_sequence_classif(*SCREAMING_SNAKE_CASE__							)
   def     __UpperCamelCase						( self			:      List[Any]							):
     SCREAMING_SNAKE_CASE:Optional[Any]						      =					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_flaubert_token_classif(*SCREAMING_SNAKE_CASE__							)
   def     __UpperCamelCase						( self			:      int							):
     SCREAMING_SNAKE_CASE:List[Any]						      =					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_flaubert_multiple_choice(*SCREAMING_SNAKE_CASE__							)
   @slow
   def     __UpperCamelCase						( self			:      List[str]							):
     for model_name in FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
       SCREAMING_SNAKE_CASE:Tuple						      =					FlaubertModel.from_pretrained(SCREAMING_SNAKE_CASE__							)
       self.assertIsNotNone(SCREAMING_SNAKE_CASE__							)







   @slow
   @require_torch_gpu
   def     __UpperCamelCase						( self			:      str							):
     SCREAMING_SNAKE_CASE      ,    SCREAMING_SNAKE_CASE:int						      =					self.model_tester.prepare_config_and_inputs_for_common()
     for model_class in self.all_model_classes:
       # FlauBertForMultipleChoice behaves incorrectly in JIT environments.
       if model_class == FlaubertForMultipleChoice:
         return

       SCREAMING_SNAKE_CASE:Union[str, Any]						      =					True
       SCREAMING_SNAKE_CASE:Optional[int]						      =					model_class(config=SCREAMING_SNAKE_CASE__							)

       SCREAMING_SNAKE_CASE:Any						      =					self._prepare_for_class(SCREAMING_SNAKE_CASE__     ,SCREAMING_SNAKE_CASE__							)
       SCREAMING_SNAKE_CASE:Any						      =					torch.jit.trace(
           SCREAMING_SNAKE_CASE__     ,(inputs_dict["input_ids"].to("cpu"							), inputs_dict["attention_mask"].to("cpu"							))							)

       with tempfile.TemporaryDirectory() as tmp:
         torch.jit.save(SCREAMING_SNAKE_CASE__     ,os.path.join(SCREAMING_SNAKE_CASE__     ,"traced_model.pt"							)							)
         SCREAMING_SNAKE_CASE:Optional[Any]						      =					torch.jit.load(os.path.join(SCREAMING_SNAKE_CASE__     ,"traced_model.pt"							)     ,map_location=SCREAMING_SNAKE_CASE__							)
         loaded(inputs_dict["input_ids"].to(SCREAMING_SNAKE_CASE__							)     ,inputs_dict["attention_mask"].to(SCREAMING_SNAKE_CASE__							)							)





@require_torch
class 							_snake_case (       unittest.TestCase						):
   @slow
   def     __UpperCamelCase						( self			:      List[str]							):
     SCREAMING_SNAKE_CASE:Optional[int]						      =					FlaubertModel.from_pretrained("flaubert/flaubert_base_cased"							)
     SCREAMING_SNAKE_CASE:Optional[int]						      =					torch.tensor([[0, 345, 232, 328, 740, 140, 1_695, 69, 6_078, 1_588, 2]]							)
     with torch.no_grad():
       SCREAMING_SNAKE_CASE:Union[str, Any]						      =					model(SCREAMING_SNAKE_CASE__							)[0]
     SCREAMING_SNAKE_CASE:Union[str, Any]						      =					torch.Size((1, 11, 768)							)
     self.assertEqual(output.shape     ,SCREAMING_SNAKE_CASE__							)
     SCREAMING_SNAKE_CASE:int						      =					torch.tensor(
         [[[-2.6_251, -1.4_298, -0.0_227], [-2.8_510, -1.6_387, 0.2_258], [-2.8_114, -1.1_832, -0.3_066]]]							)

     self.assertTrue(torch.allclose(output[:, :3, :3]     ,SCREAMING_SNAKE_CASE__     ,atol=1e-4							)							)






