
"""simple docstring"""



from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
 import numpy
 import tensorflow as tf

 from transformers import (
     TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
     TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
     TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST,
     BertConfig,
     DPRConfig,
     TFDPRContextEncoder,
     TFDPRQuestionEncoder,
     TFDPRReader,
 )




class 		a_						:







    def __init__(			self			:       List[str]     ,					__UpperCamelCase			:       List[str]     ,					__UpperCamelCase			:       List[str]=13     ,					__UpperCamelCase			:       Optional[int]=7     ,					__UpperCamelCase			:       Any=True     ,					__UpperCamelCase			:       Optional[int]=True     ,					__UpperCamelCase			:       int=True     ,					__UpperCamelCase			:       Any=True     ,					__UpperCamelCase			:       Dict=99     ,					__UpperCamelCase			:       str=32     ,					__UpperCamelCase			:       List[Any]=2     ,					__UpperCamelCase			:       List[str]=4     ,					__UpperCamelCase			:       Tuple=37     ,					__UpperCamelCase			:       int="gelu"     ,					__UpperCamelCase			:       str=0.1     ,					__UpperCamelCase			:       Union[str, Any]=0.1     ,					__UpperCamelCase			:       List[str]=5_12     ,					__UpperCamelCase			:       Union[str, Any]=16     ,					__UpperCamelCase			:       Optional[Any]=2     ,					__UpperCamelCase			:       Union[str, Any]=0.0_2     ,					__UpperCamelCase			:       Tuple=3     ,					__UpperCamelCase			:       List[Any]=4     ,					__UpperCamelCase			:       List[Any]=None     ,					__UpperCamelCase			:       Dict=0     ,					)		->List[Any]:






        '''simple docstring'''






        _UpperCAmelCase          =   parent
        _UpperCAmelCase          =   batch_size
        _UpperCAmelCase          =   seq_length
        _UpperCAmelCase          =   is_training
        _UpperCAmelCase          =   use_input_mask
        _UpperCAmelCase          =   use_token_type_ids
        _UpperCAmelCase          =   use_labels
        _UpperCAmelCase          =   vocab_size
        _UpperCAmelCase          =   hidden_size
        _UpperCAmelCase          =   num_hidden_layers
        _UpperCAmelCase          =   num_attention_heads
        _UpperCAmelCase          =   intermediate_size
        _UpperCAmelCase          =   hidden_act
        _UpperCAmelCase          =   hidden_dropout_prob
        _UpperCAmelCase          =   attention_probs_dropout_prob
        _UpperCAmelCase          =   max_position_embeddings
        _UpperCAmelCase          =   type_vocab_size
        _UpperCAmelCase          =   type_sequence_label_size
        _UpperCAmelCase          =   initializer_range
        _UpperCAmelCase          =   num_labels
        _UpperCAmelCase          =   num_choices
        _UpperCAmelCase          =   scope
        _UpperCAmelCase          =   projection_dim







    def      _snake_case					(			self			:       List[str]     )		->Dict:






        '''simple docstring'''






        _UpperCAmelCase          =   ids_tensor([self.batch_size, self.seq_length]     ,					self.vocab_size     )

        _UpperCAmelCase          =   None
        if self.use_input_mask:
            # follow test_modeling_tf_ctrl.py
            _UpperCAmelCase          =   random_attention_mask([self.batch_size, self.seq_length]     )

        _UpperCAmelCase          =   None
        if self.use_token_type_ids:
            _UpperCAmelCase          =   ids_tensor([self.batch_size, self.seq_length]     ,					self.type_vocab_size     )

        _UpperCAmelCase          =   None
        _UpperCAmelCase          =   None
        _UpperCAmelCase          =   None
        if self.use_labels:
            _UpperCAmelCase          =   ids_tensor([self.batch_size]     ,					self.type_sequence_label_size     )
            _UpperCAmelCase          =   ids_tensor([self.batch_size, self.seq_length]     ,					self.num_labels     )
            _UpperCAmelCase          =   ids_tensor([self.batch_size]     ,					self.num_choices     )

        _UpperCAmelCase          =   BertConfig(
            vocab_size=self.vocab_size     ,					hidden_size=self.hidden_size     ,					num_hidden_layers=self.num_hidden_layers     ,					num_attention_heads=self.num_attention_heads     ,					intermediate_size=self.intermediate_size     ,					hidden_act=self.hidden_act     ,					hidden_dropout_prob=self.hidden_dropout_prob     ,					attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,					max_position_embeddings=self.max_position_embeddings     ,					type_vocab_size=self.type_vocab_size     ,					is_decoder=__UpperCamelCase     ,					initializer_range=self.initializer_range     ,					)
        _UpperCAmelCase          =   DPRConfig(projection_dim=self.projection_dim     ,					**config.to_dict()     )

        return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







    def      _snake_case					(			self			:       Optional[Any]     ,					__UpperCamelCase			:       Optional[Any]     ,					__UpperCamelCase			:       Any     ,					__UpperCamelCase			:       Optional[int]     ,					__UpperCamelCase			:       Any     ,					__UpperCamelCase			:       List[Any]     ,					__UpperCamelCase			:       Optional[int]     ,					__UpperCamelCase			:       Optional[int]     )		->int:






        '''simple docstring'''






        _UpperCAmelCase          =   TFDPRContextEncoder(config=__UpperCamelCase     )
        _UpperCAmelCase          =   model(__UpperCamelCase     ,					attention_mask=__UpperCamelCase     ,					token_type_ids=__UpperCamelCase     )
        _UpperCAmelCase          =   model(__UpperCamelCase     ,					token_type_ids=__UpperCamelCase     )
        _UpperCAmelCase          =   model(__UpperCamelCase     )
        self.parent.assertEqual(result.pooler_output.shape     ,					(self.batch_size, self.projection_dim or self.hidden_size)     )







    def      _snake_case					(			self			:       List[str]     ,					__UpperCamelCase			:       Tuple     ,					__UpperCamelCase			:       Union[str, Any]     ,					__UpperCamelCase			:       Dict     ,					__UpperCamelCase			:       int     ,					__UpperCamelCase			:       List[str]     ,					__UpperCamelCase			:       Optional[Any]     ,					__UpperCamelCase			:       Union[str, Any]     )		->Tuple:






        '''simple docstring'''






        _UpperCAmelCase          =   TFDPRQuestionEncoder(config=__UpperCamelCase     )
        _UpperCAmelCase          =   model(__UpperCamelCase     ,					attention_mask=__UpperCamelCase     ,					token_type_ids=__UpperCamelCase     )
        _UpperCAmelCase          =   model(__UpperCamelCase     ,					token_type_ids=__UpperCamelCase     )
        _UpperCAmelCase          =   model(__UpperCamelCase     )
        self.parent.assertEqual(result.pooler_output.shape     ,					(self.batch_size, self.projection_dim or self.hidden_size)     )







    def      _snake_case					(			self			:       int     ,					__UpperCamelCase			:       List[Any]     ,					__UpperCamelCase			:       Optional[int]     ,					__UpperCamelCase			:       Optional[int]     ,					__UpperCamelCase			:       Dict     ,					__UpperCamelCase			:       Optional[Any]     ,					__UpperCamelCase			:       List[Any]     ,					__UpperCamelCase			:       Dict     )		->Union[str, Any]:






        '''simple docstring'''






        _UpperCAmelCase          =   TFDPRReader(config=__UpperCamelCase     )
        _UpperCAmelCase          =   model(__UpperCamelCase     ,					attention_mask=__UpperCamelCase     )

        self.parent.assertEqual(result.start_logits.shape     ,					(self.batch_size, self.seq_length)     )
        self.parent.assertEqual(result.end_logits.shape     ,					(self.batch_size, self.seq_length)     )
        self.parent.assertEqual(result.relevance_logits.shape     ,					(self.batch_size,)     )







    def      _snake_case					(			self			:       Optional[Any]     )		->Union[str, Any]:






        '''simple docstring'''






        _UpperCAmelCase          =   self.prepare_config_and_inputs()
        (
            (
            _UpperCAmelCase
        )    ,(
            _UpperCAmelCase
        )    ,(
            _UpperCAmelCase
        )    ,(
            _UpperCAmelCase
        )    ,(
            _UpperCAmelCase
        )    ,(
            _UpperCAmelCase
        )    ,(
            _UpperCAmelCase
        )    ,
        )          =   config_and_inputs
        _UpperCAmelCase          =   {"""input_ids""": input_ids}
        return config, inputs_dict







@require_tf
class 		a_						( _UpperCAmelCase     ,						_UpperCAmelCase     ,						unittest.TestCase	):
    a      :				str     =				(
        (
            TFDPRContextEncoder,
            TFDPRQuestionEncoder,
            TFDPRReader,
        )
        if is_tf_available()
        else ()
    )
    a      :				Optional[int]     =				{'feature-extraction': TFDPRQuestionEncoder} if is_tf_available() else {}

    a      :				Tuple     =				False
    a      :				List[str]     =				False
    a      :				int     =				False
    a      :				List[Any]     =				False
    a      :				Any     =				False







    def      _snake_case					(			self			:       Optional[Any]     )		->Dict:






        '''simple docstring'''






        _UpperCAmelCase          =   TFDPRModelTester(self     )
        _UpperCAmelCase          =   ConfigTester(self     ,					config_class=__UpperCamelCase     ,					hidden_size=37     )







    def      _snake_case					(			self			:       Union[str, Any]     )		->Union[str, Any]:






        '''simple docstring'''






        self.config_tester.run_common_tests()







    def      _snake_case					(			self			:       int     )		->Dict:






        '''simple docstring'''






        _UpperCAmelCase          =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_dpr_context_encoder(*__UpperCamelCase     )







    def      _snake_case					(			self			:       Tuple     )		->Optional[Any]:






        '''simple docstring'''






        _UpperCAmelCase          =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_dpr_question_encoder(*__UpperCamelCase     )







    def      _snake_case					(			self			:       Dict     )		->Dict:






        '''simple docstring'''






        _UpperCAmelCase          =   self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_dpr_reader(*__UpperCamelCase     )







    @slow
    def      _snake_case					(			self			:       Tuple     )		->str:






        '''simple docstring'''






        for model_name in TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            _UpperCAmelCase          =   TFDPRContextEncoder.from_pretrained(__UpperCamelCase     )
            self.assertIsNotNone(__UpperCamelCase     )

        for model_name in TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            _UpperCAmelCase          =   TFDPRContextEncoder.from_pretrained(__UpperCamelCase     )
            self.assertIsNotNone(__UpperCamelCase     )

        for model_name in TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            _UpperCAmelCase          =   TFDPRQuestionEncoder.from_pretrained(__UpperCamelCase     )
            self.assertIsNotNone(__UpperCamelCase     )

        for model_name in TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            _UpperCAmelCase          =   TFDPRReader.from_pretrained(__UpperCamelCase     )
            self.assertIsNotNone(__UpperCamelCase     )







@require_tf
class 		a_						( unittest.TestCase	):







    @slow
    def      _snake_case					(			self			:       int     )		->Optional[int]:






        '''simple docstring'''






        _UpperCAmelCase          =   TFDPRQuestionEncoder.from_pretrained("""facebook/dpr-question_encoder-single-nq-base"""     )

        _UpperCAmelCase          =   tf.constant(
            [[1_01, 75_92, 10_10, 20_03, 20_26, 38_99, 1_01_40, 10_29, 1_02]]     )  # [CLS] hello, is my dog cute? [SEP]
        _UpperCAmelCase          =   model(__UpperCamelCase     )[0]  # embedding shape = (1, 768)
        # compare the actual values for a slice.
        _UpperCAmelCase          =   tf.constant(
            [
                [
                    0.0_3_2_3_6_2_5_3,
                    0.1_2_7_5_3_3_3_5,
                    0.1_6_8_1_8_5_0_9,
                    0.0_0_2_7_9_7_8_6,
                    0.3_8_9_6_9_3_3,
                    0.2_4_2_6_4_9_4_5,
                    0.2_1_7_8_9_7_1,
                    -0.0_2_3_3_5_2_2_7,
                    -0.0_8_4_8_1_9_5_9,
                    -0.1_4_3_2_4_1_1_7,
                ]
            ]     )
        self.assertTrue(numpy.allclose(output[:, :10].numpy()     ,					expected_slice.numpy()     ,					atol=1e-4     )     )

"""simple docstring"""



import argparse
import json
import math
import os
import time
import traceback
import zipfile
from collections import Counter

import requests


def  _UpperCamelCase	(						_A					,      _A=None      )    ->  Union[str, Any]:





    """simple docstring"""

    _UpperCAmelCase          =   None
    if token is not None:
        _UpperCAmelCase          =   {"""Accept""": """application/vnd.github+json""", """Authorization""": F"""Bearer {token}"""}

    _UpperCAmelCase          =   F"""https://api.github.com/repos/huggingface/transformers/actions/runs/{workflow_run_id}/jobs?per_page=100"""
    _UpperCAmelCase          =   requests.get(_A					,      headers=_A      ).json()
    _UpperCAmelCase          =   {}

    try:
        job_links.update({job["""name"""]: job["""html_url"""] for job in result["""jobs"""]}      )
        _UpperCAmelCase          =   math.ceil((result["""total_count"""] - 1_0_0) / 1_0_0      )

        for i in range(_A      ):
            _UpperCAmelCase          =   requests.get(url + F"""&page={i + 2}"""					,      headers=_A      ).json()
            job_links.update({job["""name"""]: job["""html_url"""] for job in result["""jobs"""]}      )

        return job_links
    except Exception:
        print(F"""Unknown error, could not fetch links:\n{traceback.format_exc()}"""      )

    return {}


def  _UpperCamelCase	(						_A					,      _A=None      )    ->  Dict:





    """simple docstring"""

    _UpperCAmelCase          =   None
    if token is not None:
        _UpperCAmelCase          =   {"""Accept""": """application/vnd.github+json""", """Authorization""": F"""Bearer {token}"""}

    _UpperCAmelCase          =   F"""https://api.github.com/repos/huggingface/transformers/actions/runs/{worflow_run_id}/artifacts?per_page=100"""
    _UpperCAmelCase          =   requests.get(_A					,      headers=_A      ).json()
    _UpperCAmelCase          =   {}

    try:
        artifacts.update({artifact["""name"""]: artifact["""archive_download_url"""] for artifact in result["""artifacts"""]}      )
        _UpperCAmelCase          =   math.ceil((result["""total_count"""] - 1_0_0) / 1_0_0      )

        for i in range(_A      ):
            _UpperCAmelCase          =   requests.get(url + F"""&page={i + 2}"""					,      headers=_A      ).json()
            artifacts.update({artifact["""name"""]: artifact["""archive_download_url"""] for artifact in result["""artifacts"""]}      )

        return artifacts
    except Exception:
        print(F"""Unknown error, could not fetch links:\n{traceback.format_exc()}"""      )

    return {}


def  _UpperCamelCase	(						_A					,      _A					,      _A					,      _A      )    ->  List[str]:





    """simple docstring"""

    _UpperCAmelCase          =   None
    if token is not None:
        _UpperCAmelCase          =   {"""Accept""": """application/vnd.github+json""", """Authorization""": F"""Bearer {token}"""}

    _UpperCAmelCase          =   requests.get(_A					,      headers=_A					,      allow_redirects=_A      )
    _UpperCAmelCase          =   result.headers["""Location"""]
    _UpperCAmelCase          =   requests.get(_A					,      allow_redirects=_A      )
    _UpperCAmelCase          =   os.path.join(_A					,      F"""{artifact_name}.zip"""      )
    with open(_A					,      """wb"""      ) as fp:
        fp.write(response.content      )


def  _UpperCamelCase	(						_A					,      _A=None      )    ->  Dict:





    """simple docstring"""

    _UpperCAmelCase          =   []
    _UpperCAmelCase          =   []
    _UpperCAmelCase          =   None

    with zipfile.ZipFile(_A      ) as z:
        for filename in z.namelist():
            if not os.path.isdir(_A      ):
                # read the file
                if filename in ["failures_line.txt", "summary_short.txt", "job_name.txt"]:
                    with z.open(_A      ) as f:
                        for line in f:
                            _UpperCAmelCase          =   line.decode("""UTF-8"""      ).strip()
                            if filename == "failures_line.txt":
                                try:
                                    # `error_line` is the place where `error` occurs
                                    _UpperCAmelCase          =   line[: line.index(""": """      )]
                                    _UpperCAmelCase          =   line[line.index(""": """      ) + len(""": """      ) :]
                                    errors.append([error_line, error]      )
                                except Exception:
                                    # skip un-related lines
                                    pass
                            elif filename == "summary_short.txt" and line.startswith("""FAILED """      ):
                                # `test` is the test method that failed
                                _UpperCAmelCase          =   line[len("""FAILED """      ) :]
                                failed_tests.append(_A      )
                            elif filename == "job_name.txt":
                                _UpperCAmelCase          =   line

    if len(_A      ) != len(_A      ):
        raise ValueError(
            F"""`errors` and `failed_tests` should have the same number of elements. Got {len(_A      )} for `errors` """
            F"""and {len(_A      )} for `failed_tests` instead. The test reports in {artifact_zip_path} have some"""
            """ problem."""      )

    _UpperCAmelCase          =   None
    if job_name and job_links:
        _UpperCAmelCase          =   job_links.get(_A					,      _A      )

    # A list with elements of the form (line of error, error, failed test)
    _UpperCAmelCase          =   [x + [y] + [job_link] for x, y in zip(_A					,      _A      )]

    return result


def  _UpperCamelCase	(						_A					,      _A=None      )    ->  Union[str, Any]:





    """simple docstring"""

    _UpperCAmelCase          =   []

    _UpperCAmelCase          =   [os.path.join(_A					,      _A      ) for p in os.listdir(_A      ) if p.endswith(""".zip"""      )]
    for p in paths:
        errors.extend(get_errors_from_single_artifact(_A					,      job_links=_A      )      )

    return errors


def  _UpperCamelCase	(						_A					,      _A=None      )    ->  Optional[Any]:





    """simple docstring"""

    _UpperCAmelCase          =   Counter()
    counter.update([x[1] for x in logs]      )
    _UpperCAmelCase          =   counter.most_common()
    _UpperCAmelCase          =   {}
    for error, count in counts:
        if error_filter is None or error not in error_filter:
            _UpperCAmelCase          =   {"""count""": count, """failed_tests""": [(x[2], x[0]) for x in logs if x[1] == error]}

    _UpperCAmelCase          =   dict(sorted(r.items()					,      key=lambda _A      : item[1]["count"]					,      reverse=_A      )      )
    return r


def  _UpperCamelCase	(						_A      )    ->  Dict:





    """simple docstring"""

    _UpperCAmelCase          =   test.split("""::"""      )[0]
    if test.startswith("""tests/models/"""      ):
        _UpperCAmelCase          =   test.split("""/"""      )[2]
    else:
        _UpperCAmelCase          =   None

    return test


def  _UpperCamelCase	(						_A					,      _A=None      )    ->  Any:





    """simple docstring"""

    _UpperCAmelCase          =   [(x[0], x[1], get_model(x[2]      )) for x in logs]
    _UpperCAmelCase          =   [x for x in logs if x[2] is not None]
    _UpperCAmelCase          =   {x[2] for x in logs}

    _UpperCAmelCase          =   {}
    for test in tests:
        _UpperCAmelCase          =   Counter()
        # count by errors in `test`
        counter.update([x[1] for x in logs if x[2] == test]      )
        _UpperCAmelCase          =   counter.most_common()
        _UpperCAmelCase          =   {error: count for error, count in counts if (error_filter is None or error not in error_filter)}
        _UpperCAmelCase          =   sum(error_counts.values()      )
        if n_errors > 0:
            _UpperCAmelCase          =   {"""count""": n_errors, """errors""": error_counts}

    _UpperCAmelCase          =   dict(sorted(r.items()					,      key=lambda _A      : item[1]["count"]					,      reverse=_A      )      )
    return r


def  _UpperCamelCase	(						_A      )    ->  Optional[int]:





    """simple docstring"""

    _UpperCAmelCase          =   """| no. | error | status |"""
    _UpperCAmelCase          =   """|-:|:-|:-|"""
    _UpperCAmelCase          =   [header, sep]
    for error in reduced_by_error:
        _UpperCAmelCase          =   reduced_by_error[error]["""count"""]
        _UpperCAmelCase          =   F"""| {count} | {error[:1_0_0]} |  |"""
        lines.append(_A      )

    return "\n".join(_A      )


def  _UpperCamelCase	(						_A      )    ->  Optional[Any]:





    """simple docstring"""

    _UpperCAmelCase          =   """| model | no. of errors | major error | count |"""
    _UpperCAmelCase          =   """|-:|-:|-:|-:|"""
    _UpperCAmelCase          =   [header, sep]
    for model in reduced_by_model:
        _UpperCAmelCase          =   reduced_by_model[model]["""count"""]
        _UpperCAmelCase    ,_UpperCAmelCase          =   list(reduced_by_model[model]["""errors"""].items()      )[0]
        _UpperCAmelCase          =   F"""| {model} | {count} | {error[:6_0]} | {_count} |"""
        lines.append(_A      )

    return "\n".join(_A      )


if __name__ == "__main__":
 a					:	Tuple							=		argparse.ArgumentParser()
 # Required parameters
 parser.add_argument('''--workflow_run_id''', type=str, required=True, help='''A GitHub Actions workflow run id.''')
 parser.add_argument(
     '''--output_dir''',
     type=str,
     required=True,
     help='''Where to store the downloaded artifacts and other result files.''',
 )
 parser.add_argument('''--token''', default=None, type=str, help='''A token that has actions:read permission.''')
 a					:	Dict							=		parser.parse_args()

 os.makedirs(args.output_dir, exist_ok=True)

 a					:	Tuple							=		get_job_links(args.workflow_run_id, token=args.token)
 a					:	Tuple							=		{}
 # To deal with `workflow_call` event, where a job name is the combination of the job names in the caller and callee.
 # For example, `PyTorch 1.11 / Model tests (models/albert, single-gpu)`.
 if _job_links:
  for k, v in _job_links.items():
   # This is how GitHub actions combine job names.
   if " / " in k:
    a					:	List[Any]							=		k.find(''' / ''')
    a					:	Tuple							=		k[index + len(''' / ''') :]
   a					:	int							=		v
 with open(os.path.join(args.output_dir, '''job_links.json'''), '''w''', encoding='''UTF-8''') as fp:
  json.dump(job_links, fp, ensure_ascii=False, indent=4)

 a					:	Tuple							=		get_artifacts_links(args.workflow_run_id, token=args.token)
 with open(os.path.join(args.output_dir, '''artifacts.json'''), '''w''', encoding='''UTF-8''') as fp:
  json.dump(artifacts, fp, ensure_ascii=False, indent=4)

 for idx, (name, url) in enumerate(artifacts.items()):
  download_artifact(name, url, args.output_dir, args.token)
  # Be gentle to GitHub
  time.sleep(1)

 a					:	Optional[int]							=		get_all_errors(args.output_dir, job_links=job_links)

 # `e[1]` is the error
 a					:	Union[str, Any]							=		Counter()
 counter.update([e[1] for e in errors])

 # print the top 30 most common test errors
 a					:	Optional[int]							=		counter.most_common(3_0)
 for item in most_common:
  print(item)

 with open(os.path.join(args.output_dir, '''errors.json'''), '''w''', encoding='''UTF-8''') as fp:
  json.dump(errors, fp, ensure_ascii=False, indent=4)

 a					:	int							=		reduce_by_error(errors)
 a					:	str							=		reduce_by_model(errors)

 a					:	int							=		make_github_table(reduced_by_error)
 a					:	Optional[int]							=		make_github_table_per_model(reduced_by_model)

 with open(os.path.join(args.output_dir, '''reduced_by_error.txt'''), '''w''', encoding='''UTF-8''') as fp:
  fp.write(sa)
 with open(os.path.join(args.output_dir, '''reduced_by_model.txt'''), '''w''', encoding='''UTF-8''') as fp:
  fp.write(sa)