We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[RFE] Results to include positive NACK - CVEs to which a system is NOT vulnerable
Use case: VMaaS provides list of CVEs to which a system is vulnerable, based on list of packages' NEVRAs installed on the system.  This is overridden by results from the rules engine evaluation, which can check if a certain port is closed, or a configuration value prevents exploit of a vulnerability.  Unless the rules engine results includes "we evaluated for vulnerability to CVE-x and the system is NOT vulnerable", we (vulnerability-engine) won't know to override the VMaaS results that report the system is vulnerable based on the version of the package installed.

Vulnerability-engine will be able to assume that a system is no longer vulnerable to a rules engine-reported CVE if it was reported as vulnerable in previous evaluation.  However, a new system checking in with a fix already in place... vulnerability-engine won't know that the vulnerability is mitigated by a config setting, closed port, etc. unless rules engine explicitly reports a CVE that was evaluated and found the system NOT vulnerable.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `insights/core/evaluators.py`
Content:
```
1 import logging
2 import sys
3 
4 from ..formats import Formatter
5 from ..specs import Specs
6 from ..combiners.hostname import hostname as combiner_hostname
7 from ..parsers.branch_info import BranchInfo
8 from . import dr, plugins
9 
10 log = logging.getLogger(__name__)
11 
12 
13 def get_simple_module_name(obj):
14     return dr.BASE_MODULE_NAMES.get(obj, None)
15 
16 
17 class Evaluator(Formatter):
18     def __init__(self, broker=None, stream=sys.stdout, incremental=False):
19         super(Evaluator, self).__init__(broker or dr.Broker(), stream)
20         self.rule_skips = []
21         self.rule_results = []
22         self.fingerprint_results = []
23         self.hostname = None
24         self.metadata = {}
25         self.metadata_keys = {}
26         self.incremental = incremental
27 
28     def observer(self, comp, broker):
29         if comp is combiner_hostname and comp in broker:
30             self.hostname = broker[comp].fqdn
31 
32         if plugins.is_rule(comp) and comp in broker:
33             self.handle_result(comp, broker[comp])
34 
35     def preprocess(self):
36         self.broker.add_observer(self.observer)
37 
38     def run_serial(self, graph=None):
39         dr.run(graph or dr.COMPONENTS[dr.GROUPS.single], broker=self.broker)
40 
41     def run_incremental(self, graph=None):
42         for _ in dr.run_incremental(graph or dr.COMPONENTS[dr.GROUPS.single], broker=self.broker):
43             pass
44 
45     def format_response(self, response):
46         """
47         To be overridden by subclasses to format the response sent back to the
48         client.
49         """
50         return response
51 
52     def format_result(self, result):
53         """
54         To be overridden by subclasses to format individual rule results.
55         """
56         return result
57 
58     def process(self, graph=None):
59         with self:
60             if self.incremental:
61                 self.run_incremental(graph)
62             else:
63                 self.run_serial(graph)
64         return self.get_response()
65 
66 
67 class SingleEvaluator(Evaluator):
68     def append_metadata(self, r):
69         for k, v in r.items():
70             if k != "type":
71                 self.metadata[k] = v
72 
73     def format_response(self, response):
74         return response
75 
76     def get_response(self):
77         r = dict(self.metadata_keys)
78         r.update({
79             "system": {
80                 "metadata": self.metadata,
81                 "hostname": self.hostname
82             },
83             "reports": self.rule_results,
84             "fingerprints": self.fingerprint_results,
85             "skips": self.rule_skips,
86         })
87         return self.format_response(r)
88 
89     def handle_result(self, plugin, r):
90         type_ = r["type"]
91         if type_ == "metadata":
92             self.append_metadata(r)
93         elif type_ == "rule":
94             self.rule_results.append(self.format_result({
95                 "rule_id": "{0}|{1}".format(get_simple_module_name(plugin), r["error_key"]),
96                 "details": r
97             }))
98         elif type_ == "fingerprint":
99             self.fingerprint_results.append(self.format_result({
100                 "fingerprint_id": "{0}|{1}".format(get_simple_module_name(plugin), r["fingerprint_key"]),
101                 "details": r
102             }))
103         elif type_ == "skip":
104             self.rule_skips.append(r)
105         elif type_ == "metadata_key":
106             self.metadata_keys[r["key"]] = r["value"]
107 
108 
109 class InsightsEvaluator(SingleEvaluator):
110     def __init__(self, broker=None, system_id=None, stream=sys.stdout, incremental=False):
111         super(InsightsEvaluator, self).__init__(broker, stream=sys.stdout, incremental=incremental)
112         self.system_id = system_id
113         self.branch_info = {}
114         self.product = "rhel"
115         self.type = "host"
116         self.release = None
117 
118     def observer(self, comp, broker):
119         super(InsightsEvaluator, self).observer(comp, broker)
120         if comp is Specs.machine_id and comp in broker:
121             self.system_id = broker[Specs.machine_id].content[0].strip()
122 
123         if comp is Specs.redhat_release and comp in broker:
124             self.release = broker[comp].content[0].strip()
125 
126         if comp is BranchInfo and BranchInfo in broker:
127             self.branch_info = broker[comp].data
128 
129         if comp is Specs.metadata_json and comp in broker:
130             md = broker[comp]
131             self.product = md.get("product_code")
132             self.type = md.get("role")
133 
134     def format_result(self, result):
135         result["system_id"] = self.system_id
136         return result
137 
138     def format_response(self, response):
139         system = response["system"]
140         system["remote_branch"] = self.branch_info.get("remote_branch")
141         system["remote_leaf"] = self.branch_info.get("remote_leaf")
142         system["system_id"] = self.system_id
143         system["product"] = self.product
144         system["type"] = self.type
145         if self.release:
146             system["metadata"]["release"] = self.release
147 
148         return response
149
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/insights/core/evaluators.py b/insights/core/evaluators.py
--- a/insights/core/evaluators.py
+++ b/insights/core/evaluators.py
@@ -1,6 +1,9 @@
 import logging
+import six
 import sys
 
+from collections import defaultdict
+
 from ..formats import Formatter
 from ..specs import Specs
 from ..combiners.hostname import hostname as combiner_hostname
@@ -17,9 +20,8 @@
 class Evaluator(Formatter):
     def __init__(self, broker=None, stream=sys.stdout, incremental=False):
         super(Evaluator, self).__init__(broker or dr.Broker(), stream)
+        self.results = defaultdict(list)
         self.rule_skips = []
-        self.rule_results = []
-        self.fingerprint_results = []
         self.hostname = None
         self.metadata = {}
         self.metadata_keys = {}
@@ -80,30 +82,32 @@
                 "metadata": self.metadata,
                 "hostname": self.hostname
             },
-            "reports": self.rule_results,
-            "fingerprints": self.fingerprint_results,
+            "reports": self.results["rule"],
+            "fingerprints": self.results["fingerprint"],
             "skips": self.rule_skips,
         })
+
+        for k, v in six.iteritems(self.results):
+            if k not in ("rule", "fingerprint"):
+                r[k] = v
+
         return self.format_response(r)
 
     def handle_result(self, plugin, r):
         type_ = r["type"]
-        if type_ == "metadata":
+
+        if type_ == "skip":
+            self.rule_skips.append(r)
+        elif type_ == "metadata":
             self.append_metadata(r)
-        elif type_ == "rule":
-            self.rule_results.append(self.format_result({
-                "rule_id": "{0}|{1}".format(get_simple_module_name(plugin), r["error_key"]),
-                "details": r
-            }))
-        elif type_ == "fingerprint":
-            self.fingerprint_results.append(self.format_result({
-                "fingerprint_id": "{0}|{1}".format(get_simple_module_name(plugin), r["fingerprint_key"]),
+        elif type_ == "metadata_key":
+            self.metadata_keys[r.get_key()] = r["value"]
+        else:
+            response_id = "%s_id" % r.response_type
+            self.results[type_].append(self.format_result({
+                response_id: "{0}|{1}".format(get_simple_module_name(plugin), r.get_key()),
                 "details": r
             }))
-        elif type_ == "skip":
-            self.rule_skips.append(r)
-        elif type_ == "metadata_key":
-            self.metadata_keys[r["key"]] = r["value"]
 
 
 class InsightsEvaluator(SingleEvaluator):

{"golden_diff": "diff --git a/insights/core/evaluators.py b/insights/core/evaluators.py\n--- a/insights/core/evaluators.py\n+++ b/insights/core/evaluators.py\n@@ -1,6 +1,9 @@\n import logging\n+import six\n import sys\n \n+from collections import defaultdict\n+\n from ..formats import Formatter\n from ..specs import Specs\n from ..combiners.hostname import hostname as combiner_hostname\n@@ -17,9 +20,8 @@\n class Evaluator(Formatter):\n     def __init__(self, broker=None, stream=sys.stdout, incremental=False):\n         super(Evaluator, self).__init__(broker or dr.Broker(), stream)\n+        self.results = defaultdict(list)\n         self.rule_skips = []\n-        self.rule_results = []\n-        self.fingerprint_results = []\n         self.hostname = None\n         self.metadata = {}\n         self.metadata_keys = {}\n@@ -80,30 +82,32 @@\n                 \"metadata\": self.metadata,\n                 \"hostname\": self.hostname\n             },\n-            \"reports\": self.rule_results,\n-            \"fingerprints\": self.fingerprint_results,\n+            \"reports\": self.results[\"rule\"],\n+            \"fingerprints\": self.results[\"fingerprint\"],\n             \"skips\": self.rule_skips,\n         })\n+\n+        for k, v in six.iteritems(self.results):\n+            if k not in (\"rule\", \"fingerprint\"):\n+                r[k] = v\n+\n         return self.format_response(r)\n \n     def handle_result(self, plugin, r):\n         type_ = r[\"type\"]\n-        if type_ == \"metadata\":\n+\n+        if type_ == \"skip\":\n+            self.rule_skips.append(r)\n+        elif type_ == \"metadata\":\n             self.append_metadata(r)\n-        elif type_ == \"rule\":\n-            self.rule_results.append(self.format_result({\n-                \"rule_id\": \"{0}|{1}\".format(get_simple_module_name(plugin), r[\"error_key\"]),\n-                \"details\": r\n-            }))\n-        elif type_ == \"fingerprint\":\n-            self.fingerprint_results.append(self.format_result({\n-                \"fingerprint_id\": \"{0}|{1}\".format(get_simple_module_name(plugin), r[\"fingerprint_key\"]),\n+        elif type_ == \"metadata_key\":\n+            self.metadata_keys[r.get_key()] = r[\"value\"]\n+        else:\n+            response_id = \"%s_id\" % r.response_type\n+            self.results[type_].append(self.format_result({\n+                response_id: \"{0}|{1}\".format(get_simple_module_name(plugin), r.get_key()),\n                 \"details\": r\n             }))\n-        elif type_ == \"skip\":\n-            self.rule_skips.append(r)\n-        elif type_ == \"metadata_key\":\n-            self.metadata_keys[r[\"key\"]] = r[\"value\"]\n \n \n class InsightsEvaluator(SingleEvaluator):\n", "issue": "[RFE] Results to include positive NACK - CVEs to which a system is NOT vulnerable\nUse case: VMaaS provides list of CVEs to which a system is vulnerable, based on list of packages' NEVRAs installed on the system.  This is overridden by results from the rules engine evaluation, which can check if a certain port is closed, or a configuration value prevents exploit of a vulnerability.  Unless the rules engine results includes \"we evaluated for vulnerability to CVE-x and the system is NOT vulnerable\", we (vulnerability-engine) won't know to override the VMaaS results that report the system is vulnerable based on the version of the package installed.\r\n\r\nVulnerability-engine will be able to assume that a system is no longer vulnerable to a rules engine-reported CVE if it was reported as vulnerable in previous evaluation.  However, a new system checking in with a fix already in place... vulnerability-engine won't know that the vulnerability is mitigated by a config setting, closed port, etc. unless rules engine explicitly reports a CVE that was evaluated and found the system NOT vulnerable.\n", "before_files": [{"content": "import logging\nimport sys\n\nfrom ..formats import Formatter\nfrom ..specs import Specs\nfrom ..combiners.hostname import hostname as combiner_hostname\nfrom ..parsers.branch_info import BranchInfo\nfrom . import dr, plugins\n\nlog = logging.getLogger(__name__)\n\n\ndef get_simple_module_name(obj):\n    return dr.BASE_MODULE_NAMES.get(obj, None)\n\n\nclass Evaluator(Formatter):\n    def __init__(self, broker=None, stream=sys.stdout, incremental=False):\n        super(Evaluator, self).__init__(broker or dr.Broker(), stream)\n        self.rule_skips = []\n        self.rule_results = []\n        self.fingerprint_results = []\n        self.hostname = None\n        self.metadata = {}\n        self.metadata_keys = {}\n        self.incremental = incremental\n\n    def observer(self, comp, broker):\n        if comp is combiner_hostname and comp in broker:\n            self.hostname = broker[comp].fqdn\n\n        if plugins.is_rule(comp) and comp in broker:\n            self.handle_result(comp, broker[comp])\n\n    def preprocess(self):\n        self.broker.add_observer(self.observer)\n\n    def run_serial(self, graph=None):\n        dr.run(graph or dr.COMPONENTS[dr.GROUPS.single], broker=self.broker)\n\n    def run_incremental(self, graph=None):\n        for _ in dr.run_incremental(graph or dr.COMPONENTS[dr.GROUPS.single], broker=self.broker):\n            pass\n\n    def format_response(self, response):\n        \"\"\"\n        To be overridden by subclasses to format the response sent back to the\n        client.\n        \"\"\"\n        return response\n\n    def format_result(self, result):\n        \"\"\"\n        To be overridden by subclasses to format individual rule results.\n        \"\"\"\n        return result\n\n    def process(self, graph=None):\n        with self:\n            if self.incremental:\n                self.run_incremental(graph)\n            else:\n                self.run_serial(graph)\n        return self.get_response()\n\n\nclass SingleEvaluator(Evaluator):\n    def append_metadata(self, r):\n        for k, v in r.items():\n            if k != \"type\":\n                self.metadata[k] = v\n\n    def format_response(self, response):\n        return response\n\n    def get_response(self):\n        r = dict(self.metadata_keys)\n        r.update({\n            \"system\": {\n                \"metadata\": self.metadata,\n                \"hostname\": self.hostname\n            },\n            \"reports\": self.rule_results,\n            \"fingerprints\": self.fingerprint_results,\n            \"skips\": self.rule_skips,\n        })\n        return self.format_response(r)\n\n    def handle_result(self, plugin, r):\n        type_ = r[\"type\"]\n        if type_ == \"metadata\":\n            self.append_metadata(r)\n        elif type_ == \"rule\":\n            self.rule_results.append(self.format_result({\n                \"rule_id\": \"{0}|{1}\".format(get_simple_module_name(plugin), r[\"error_key\"]),\n                \"details\": r\n            }))\n        elif type_ == \"fingerprint\":\n            self.fingerprint_results.append(self.format_result({\n                \"fingerprint_id\": \"{0}|{1}\".format(get_simple_module_name(plugin), r[\"fingerprint_key\"]),\n                \"details\": r\n            }))\n        elif type_ == \"skip\":\n            self.rule_skips.append(r)\n        elif type_ == \"metadata_key\":\n            self.metadata_keys[r[\"key\"]] = r[\"value\"]\n\n\nclass InsightsEvaluator(SingleEvaluator):\n    def __init__(self, broker=None, system_id=None, stream=sys.stdout, incremental=False):\n        super(InsightsEvaluator, self).__init__(broker, stream=sys.stdout, incremental=incremental)\n        self.system_id = system_id\n        self.branch_info = {}\n        self.product = \"rhel\"\n        self.type = \"host\"\n        self.release = None\n\n    def observer(self, comp, broker):\n        super(InsightsEvaluator, self).observer(comp, broker)\n        if comp is Specs.machine_id and comp in broker:\n            self.system_id = broker[Specs.machine_id].content[0].strip()\n\n        if comp is Specs.redhat_release and comp in broker:\n            self.release = broker[comp].content[0].strip()\n\n        if comp is BranchInfo and BranchInfo in broker:\n            self.branch_info = broker[comp].data\n\n        if comp is Specs.metadata_json and comp in broker:\n            md = broker[comp]\n            self.product = md.get(\"product_code\")\n            self.type = md.get(\"role\")\n\n    def format_result(self, result):\n        result[\"system_id\"] = self.system_id\n        return result\n\n    def format_response(self, response):\n        system = response[\"system\"]\n        system[\"remote_branch\"] = self.branch_info.get(\"remote_branch\")\n        system[\"remote_leaf\"] = self.branch_info.get(\"remote_leaf\")\n        system[\"system_id\"] = self.system_id\n        system[\"product\"] = self.product\n        system[\"type\"] = self.type\n        if self.release:\n            system[\"metadata\"][\"release\"] = self.release\n\n        return response\n", "path": "insights/core/evaluators.py"}], "after_files": [{"content": "import logging\nimport six\nimport sys\n\nfrom collections import defaultdict\n\nfrom ..formats import Formatter\nfrom ..specs import Specs\nfrom ..combiners.hostname import hostname as combiner_hostname\nfrom ..parsers.branch_info import BranchInfo\nfrom . import dr, plugins\n\nlog = logging.getLogger(__name__)\n\n\ndef get_simple_module_name(obj):\n    return dr.BASE_MODULE_NAMES.get(obj, None)\n\n\nclass Evaluator(Formatter):\n    def __init__(self, broker=None, stream=sys.stdout, incremental=False):\n        super(Evaluator, self).__init__(broker or dr.Broker(), stream)\n        self.results = defaultdict(list)\n        self.rule_skips = []\n        self.hostname = None\n        self.metadata = {}\n        self.metadata_keys = {}\n        self.incremental = incremental\n\n    def observer(self, comp, broker):\n        if comp is combiner_hostname and comp in broker:\n            self.hostname = broker[comp].fqdn\n\n        if plugins.is_rule(comp) and comp in broker:\n            self.handle_result(comp, broker[comp])\n\n    def preprocess(self):\n        self.broker.add_observer(self.observer)\n\n    def run_serial(self, graph=None):\n        dr.run(graph or dr.COMPONENTS[dr.GROUPS.single], broker=self.broker)\n\n    def run_incremental(self, graph=None):\n        for _ in dr.run_incremental(graph or dr.COMPONENTS[dr.GROUPS.single], broker=self.broker):\n            pass\n\n    def format_response(self, response):\n        \"\"\"\n        To be overridden by subclasses to format the response sent back to the\n        client.\n        \"\"\"\n        return response\n\n    def format_result(self, result):\n        \"\"\"\n        To be overridden by subclasses to format individual rule results.\n        \"\"\"\n        return result\n\n    def process(self, graph=None):\n        with self:\n            if self.incremental:\n                self.run_incremental(graph)\n            else:\n                self.run_serial(graph)\n        return self.get_response()\n\n\nclass SingleEvaluator(Evaluator):\n    def append_metadata(self, r):\n        for k, v in r.items():\n            if k != \"type\":\n                self.metadata[k] = v\n\n    def format_response(self, response):\n        return response\n\n    def get_response(self):\n        r = dict(self.metadata_keys)\n        r.update({\n            \"system\": {\n                \"metadata\": self.metadata,\n                \"hostname\": self.hostname\n            },\n            \"reports\": self.results[\"rule\"],\n            \"fingerprints\": self.results[\"fingerprint\"],\n            \"skips\": self.rule_skips,\n        })\n\n        for k, v in six.iteritems(self.results):\n            if k not in (\"rule\", \"fingerprint\"):\n                r[k] = v\n\n        return self.format_response(r)\n\n    def handle_result(self, plugin, r):\n        type_ = r[\"type\"]\n\n        if type_ == \"skip\":\n            self.rule_skips.append(r)\n        elif type_ == \"metadata\":\n            self.append_metadata(r)\n        elif type_ == \"metadata_key\":\n            self.metadata_keys[r.get_key()] = r[\"value\"]\n        else:\n            response_id = \"%s_id\" % r.response_type\n            self.results[type_].append(self.format_result({\n                response_id: \"{0}|{1}\".format(get_simple_module_name(plugin), r.get_key()),\n                \"details\": r\n            }))\n\n\nclass InsightsEvaluator(SingleEvaluator):\n    def __init__(self, broker=None, system_id=None, stream=sys.stdout, incremental=False):\n        super(InsightsEvaluator, self).__init__(broker, stream=sys.stdout, incremental=incremental)\n        self.system_id = system_id\n        self.branch_info = {}\n        self.product = \"rhel\"\n        self.type = \"host\"\n        self.release = None\n\n    def observer(self, comp, broker):\n        super(InsightsEvaluator, self).observer(comp, broker)\n        if comp is Specs.machine_id and comp in broker:\n            self.system_id = broker[Specs.machine_id].content[0].strip()\n\n        if comp is Specs.redhat_release and comp in broker:\n            self.release = broker[comp].content[0].strip()\n\n        if comp is BranchInfo and BranchInfo in broker:\n            self.branch_info = broker[comp].data\n\n        if comp is Specs.metadata_json and comp in broker:\n            md = broker[comp]\n            self.product = md.get(\"product_code\")\n            self.type = md.get(\"role\")\n\n    def format_result(self, result):\n        result[\"system_id\"] = self.system_id\n        return result\n\n    def format_response(self, response):\n        system = response[\"system\"]\n        system[\"remote_branch\"] = self.branch_info.get(\"remote_branch\")\n        system[\"remote_leaf\"] = self.branch_info.get(\"remote_leaf\")\n        system[\"system_id\"] = self.system_id\n        system[\"product\"] = self.product\n        system[\"type\"] = self.type\n        if self.release:\n            system[\"metadata\"][\"release\"] = self.release\n\n        return response\n", "path": "insights/core/evaluators.py"}]}