We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
/fuzzer-weights cron is failing on OSS-Fuzz
@mbarbella-chromium I can get to this a bit later, but if you're interested to check this out, I'd be grateful

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `src/appengine/handlers/cron/fuzzer_weights.py`
Content:
```
1 # Copyright 2019 Google LLC
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #      http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 """Manage automatic weight adjustments."""
15 
16 import collections
17 import datetime
18 import six
19 
20 from base import utils
21 from datastore import data_handler
22 from datastore import data_types
23 from datastore import ndb_utils
24 from google_cloud_utils import big_query
25 from handlers import base_handler
26 from libs import handler
27 from metrics import fuzzer_stats
28 from metrics import logs
29 
30 QuerySpecification = collections.namedtuple(
31     'QuerySpecification', ['query_format', 'formatter', 'reason'])
32 
33 SpecificationMatch = collections.namedtuple('SpecificationMatch',
34                                             ['new_weight', 'reason'])
35 
36 
37 # Formatters for query specifications.
38 def _past_day_formatter(query_format, dataset):
39   """Simple formatter to get stats for the past day."""
40   end_time = utils.utcnow().date()
41   start_time = end_time - datetime.timedelta(days=1)
42   return query_format.format(
43       dataset=dataset, start_time=start_time, end_time=end_time)
44 
45 
46 def _new_fuzzer_formatter(query_format, dataset):
47   """Prepare a query to check for new fuzzers from the past week."""
48   now = utils.utcnow().date()
49   cutoff_time = now - datetime.timedelta(days=7)
50   return query_format.format(dataset=dataset, cutoff_time=cutoff_time)
51 
52 
53 def _coverage_formatter(query_format, dataset):
54   """Prepare a query to check for changes in coverage week over week."""
55   end_date = utils.utcnow().date() - datetime.timedelta(days=1)
56   middle_date = end_date - datetime.timedelta(days=7)
57   start_date = end_date - datetime.timedelta(days=14)
58   return query_format.format(
59       dataset=dataset,
60       start_date=start_date,
61       middle_date=middle_date,
62       end_date=end_date)
63 
64 
65 # Most of our queries should simply average a field name to get a ratio showing
66 # how often some behavior occurs.
67 GENERIC_QUERY_FORMAT = """
68 SELECT
69   fuzzer,
70   job,
71   1.0 - (1.0 - {min_weight}) * AVG({field_name}) AS new_weight
72 FROM
73   {{dataset}}.TestcaseRun
74 WHERE
75   _PARTITIONTIME BETWEEN TIMESTAMP('{{start_time}}')
76   AND TIMESTAMP('{{end_time}}')
77 GROUP BY
78   fuzzer,
79   job
80 """
81 
82 # Heavily reduce the weight for fuzzers which frequently crash on startup. This
83 # is indicitave of a very serious problem that makes it highly unlikely that
84 # we'll find anything during fuzzing.
85 STARTUP_CRASH_SPECIFICATION = QuerySpecification(
86     query_format=GENERIC_QUERY_FORMAT.format(
87         field_name='startup_crash_count', min_weight=0.10),
88     formatter=_past_day_formatter,
89     reason='frequent startup crashes')
90 
91 # Reduce weight somewhat for fuzzers with many slow units. If a particular unit
92 # runs for so long that we detect it as a slow unit, it usually means that the
93 # fuzzer is not making good use of its cycles while running or needs a fix.
94 SLOW_UNIT_SPECIFICATION = QuerySpecification(
95     query_format=GENERIC_QUERY_FORMAT.format(
96         field_name='slow_unit_count', min_weight=0.50),
97     formatter=_past_day_formatter,
98     reason='frequent slow units')
99 
100 # This should end up being very similar to the slow unit specification, and is
101 # included for the same reason.
102 TIMEOUT_SPECIFICATION = QuerySpecification(
103     query_format=GENERIC_QUERY_FORMAT.format(
104         field_name='timeout_count', min_weight=0.50),
105     formatter=_past_day_formatter,
106     reason='frequent timeouts')
107 
108 # Fuzzers with extremely frequent OOMs may contain leaks or other issues that
109 # signal that they need some improvement. Run with a slightly reduced weight
110 # until the issues are fixed.
111 OOM_SPECIFICATION = QuerySpecification(
112     query_format=GENERIC_QUERY_FORMAT.format(
113         field_name='oom_count', min_weight=0.50),
114     formatter=_past_day_formatter,
115     reason='frequent OOMs')
116 
117 # Fuzzers which are crashing frequently may not be making full use of their
118 # allotted time for fuzzing, and may end up being more effective once the known
119 # issues are fixed. This rule is more lenient than some of the others as even
120 # healthy fuzzers are expected to have some crashes.
121 CRASH_SPECIFICATION = QuerySpecification(
122     query_format=GENERIC_QUERY_FORMAT.format(
123         field_name='crash_count', min_weight=0.70),
124     formatter=_past_day_formatter,
125     reason='frequent crashes')
126 
127 # New fuzzers/jobs should run much more frequently than others. In this case, we
128 # test the fraction of days for which we have no stats for this fuzzer/job pair
129 # and increase if it's nonzero.
130 NEW_FUZZER_FORMAT = """
131 SELECT
132   fuzzer,
133   job,
134   5.0 as new_weight,
135   MIN(_PARTITIONTIME) as first_time
136 FROM
137   {dataset}.TestcaseRun
138 GROUP BY
139   fuzzer,
140   job
141 HAVING
142   first_time >= TIMESTAMP('{cutoff_time}')
143 """
144 
145 NEW_FUZZER_SPECIFICATION = QuerySpecification(
146     query_format=NEW_FUZZER_FORMAT,
147     formatter=_new_fuzzer_formatter,
148     reason='new fuzzer')
149 
150 # Format to query for fuzzers with minimal change in week to week coverage.
151 COVERAGE_UNCHANGED_FORMAT = """
152 SELECT
153   recent.fuzzer AS fuzzer,
154   recent.job AS job,
155   0.70 as new_weight
156 FROM (
157   SELECT
158     fuzzer,
159     job,
160     MAX(edge_coverage / edges_total) AS coverage
161   FROM
162     {dataset}.TestcaseRun
163   WHERE
164     _PARTITIONTIME BETWEEN TIMESTAMP('{middle_date}')
165     AND TIMESTAMP('{end_date}')
166     AND edges_total > 0
167     AND edge_coverage > 0
168   GROUP BY
169     fuzzer,
170     job
171   HAVING
172     coverage <= 1.0) AS recent
173 JOIN (
174   SELECT
175     fuzzer,
176     job,
177     MAX(edge_coverage / edges_total) AS coverage
178   FROM
179     {dataset}.TestcaseRun
180   WHERE
181     _PARTITIONTIME BETWEEN TIMESTAMP('{start_date}')
182     AND TIMESTAMP('{middle_date}')
183     AND edges_total > 0
184     AND edge_coverage > 0
185   GROUP BY
186     fuzzer,
187     job
188   HAVING
189     coverage <= 1.0) AS older
190 ON
191   recent.fuzzer = older.fuzzer
192   AND recent.job = older.job
193 WHERE
194   ABS((recent.coverage - older.coverage) / recent.coverage) < 0.01
195 """
196 
197 COVERAGE_UNCHANGED_SPECIFICATION = QuerySpecification(
198     query_format=COVERAGE_UNCHANGED_FORMAT,
199     formatter=_coverage_formatter,
200     reason='coverage flat over past 2 weeks')
201 
202 # Mappings for which specifications to use for which
203 LIBFUZZER_SPECIFICATIONS = [
204     COVERAGE_UNCHANGED_SPECIFICATION,
205     CRASH_SPECIFICATION,
206     NEW_FUZZER_SPECIFICATION,
207     OOM_SPECIFICATION,
208     SLOW_UNIT_SPECIFICATION,
209     STARTUP_CRASH_SPECIFICATION,
210     TIMEOUT_SPECIFICATION,
211 ]
212 
213 AFL_SPECIFICATIONS = [
214     CRASH_SPECIFICATION,
215     NEW_FUZZER_SPECIFICATION,
216     STARTUP_CRASH_SPECIFICATION,
217 ]
218 
219 RESTORE_DEFAULT_MATCH = SpecificationMatch(
220     new_weight=1.0, reason='no longer matches any weight adjustment rules')
221 
222 
223 def _query_helper(client, query):
224   """Helper function to get fuzzer stats."""
225   return client.query(query=query).rows
226 
227 
228 def _update_match(matches, fuzzer, job, match):
229   """Update the weight for a fuzzer/job."""
230   key = (fuzzer, job)
231   old_match = matches.get(key, RESTORE_DEFAULT_MATCH)
232 
233   new_weight = match.new_weight
234   old_weight = old_match.new_weight
235 
236   # Rules that increase weights are expected to take precedence over any that
237   # lower the weight. Issues with new fuzzers may be fixed intraday and other
238   # issues like crashes shouldn't be penalized for them.
239   if old_weight > 1.0:
240     return
241 
242   # Always update the weight if the previous value is the default. This is
243   # required to deal with specifications that are meant to set the weight above
244   # 1.0. Otherwise, prioritize only the most penalizing match for this pairing.
245   if old_match == RESTORE_DEFAULT_MATCH or new_weight < old_weight:
246     matches[key] = match
247 
248 
249 def update_weight_for_target(fuzz_target_name, job, match):
250   """Set the weight for a particular target."""
251   target_job = data_handler.get_fuzz_target_job(fuzz_target_name, job)
252 
253   if not target_job:
254     logs.log_error('FuzzTargetJob for target %s and job %s does not exist.' %
255                    (fuzz_target_name, job))
256     return
257 
258   weight = match.new_weight
259   logs.log('Adjusted weight to %f for target %s and job %s (%s).' %
260            (weight, fuzz_target_name, job, match.reason))
261 
262   target_job.weight = weight
263   target_job.put()
264 
265 
266 def update_matches_for_specification(specification, client, engine, matches,
267                                      run_set):
268   """Run a query and adjust weights based on a given query specification."""
269   query = specification.formatter(specification.query_format,
270                                   fuzzer_stats.dataset_name(engine))
271   results = _query_helper(client, query)
272   for result in results:
273     fuzzer = result['fuzzer']
274     job = result['job']
275     new_weight = result['new_weight']
276 
277     run_set.add((fuzzer, job))
278     if new_weight != 1.0:
279       match = SpecificationMatch(
280           new_weight=new_weight, reason=specification.reason)
281       _update_match(matches, fuzzer, job, match)
282 
283 
284 def update_target_weights_for_engine(client, engine, specifications):
285   """Update all fuzz target weights for the specified engine."""
286   matches = {}
287   run_set = set()
288 
289   # All fuzzers with non-default weights must be tracked with a special
290   # specification. This ensures that they will be restored to normal weight
291   # once conditions causing adjustments are no longer met.
292   target_jobs = data_types.FuzzTargetJob.query(
293       data_types.FuzzTarget.engine == engine).filter(
294           data_types.FuzzTargetJob.weight != 1.0)
295 
296   for target_job in target_jobs:
297     matches[(target_job.fuzz_target_name,
298              target_job.job)] = RESTORE_DEFAULT_MATCH
299 
300   for match in specifications:
301     update_matches_for_specification(match, client, engine, matches, run_set)
302 
303   for (fuzzer, job), match in six.iteritems(matches):
304     if (fuzzer, job) not in run_set:
305       # This ensures that we don't reset weights for fuzzers with problems if
306       # they didn't run in the time covered by our queries.
307       continue
308 
309     update_weight_for_target(fuzzer, job, match)
310 
311   logs.log('Weight adjustments complete for engine %s.' % engine)
312 
313 
314 def store_current_weights_in_bigquery():
315   """Update a bigquery table containing the daily stats."""
316   rows = []
317   target_jobs = ndb_utils.get_all_from_model(data_types.FuzzTargetJob)
318   for target_job in target_jobs:
319     row = {
320         'fuzzer': target_job.fuzz_target_name,
321         'job': target_job.job,
322         'weight': target_job.weight
323     }
324     rows.append(big_query.Insert(row=row, insert_id=None))
325 
326   client = big_query.Client(dataset_id='main', table_id='fuzzer_weights')
327   client.insert(rows)
328 
329 
330 class Handler(base_handler.Handler):
331   """Handler to periodically update fuzz target weights based on performance."""
332 
333   @handler.check_cron()
334   def get(self):
335     """Process all fuzz targets and update FuzzTargetJob weights."""
336     client = big_query.Client()
337     update_target_weights_for_engine(client, 'libFuzzer',
338                                      LIBFUZZER_SPECIFICATIONS)
339     update_target_weights_for_engine(client, 'afl', AFL_SPECIFICATIONS)
340 
341     store_current_weights_in_bigquery()
342
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/src/appengine/handlers/cron/fuzzer_weights.py b/src/appengine/handlers/cron/fuzzer_weights.py
--- a/src/appengine/handlers/cron/fuzzer_weights.py
+++ b/src/appengine/handlers/cron/fuzzer_weights.py
@@ -251,8 +251,7 @@
   target_job = data_handler.get_fuzz_target_job(fuzz_target_name, job)
 
   if not target_job:
-    logs.log_error('FuzzTargetJob for target %s and job %s does not exist.' %
-                   (fuzz_target_name, job))
+    # Bail out. This is expected if any fuzzer/job combinations become outdated.
     return
 
   weight = match.new_weight

{"golden_diff": "diff --git a/src/appengine/handlers/cron/fuzzer_weights.py b/src/appengine/handlers/cron/fuzzer_weights.py\n--- a/src/appengine/handlers/cron/fuzzer_weights.py\n+++ b/src/appengine/handlers/cron/fuzzer_weights.py\n@@ -251,8 +251,7 @@\n   target_job = data_handler.get_fuzz_target_job(fuzz_target_name, job)\n \n   if not target_job:\n-    logs.log_error('FuzzTargetJob for target %s and job %s does not exist.' %\n-                   (fuzz_target_name, job))\n+    # Bail out. This is expected if any fuzzer/job combinations become outdated.\n     return\n \n   weight = match.new_weight\n", "issue": "/fuzzer-weights cron is failing on OSS-Fuzz\n@mbarbella-chromium I can get to this a bit later, but if you're interested to check this out, I'd be grateful\n", "before_files": [{"content": "# Copyright 2019 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"Manage automatic weight adjustments.\"\"\"\n\nimport collections\nimport datetime\nimport six\n\nfrom base import utils\nfrom datastore import data_handler\nfrom datastore import data_types\nfrom datastore import ndb_utils\nfrom google_cloud_utils import big_query\nfrom handlers import base_handler\nfrom libs import handler\nfrom metrics import fuzzer_stats\nfrom metrics import logs\n\nQuerySpecification = collections.namedtuple(\n    'QuerySpecification', ['query_format', 'formatter', 'reason'])\n\nSpecificationMatch = collections.namedtuple('SpecificationMatch',\n                                            ['new_weight', 'reason'])\n\n\n# Formatters for query specifications.\ndef _past_day_formatter(query_format, dataset):\n  \"\"\"Simple formatter to get stats for the past day.\"\"\"\n  end_time = utils.utcnow().date()\n  start_time = end_time - datetime.timedelta(days=1)\n  return query_format.format(\n      dataset=dataset, start_time=start_time, end_time=end_time)\n\n\ndef _new_fuzzer_formatter(query_format, dataset):\n  \"\"\"Prepare a query to check for new fuzzers from the past week.\"\"\"\n  now = utils.utcnow().date()\n  cutoff_time = now - datetime.timedelta(days=7)\n  return query_format.format(dataset=dataset, cutoff_time=cutoff_time)\n\n\ndef _coverage_formatter(query_format, dataset):\n  \"\"\"Prepare a query to check for changes in coverage week over week.\"\"\"\n  end_date = utils.utcnow().date() - datetime.timedelta(days=1)\n  middle_date = end_date - datetime.timedelta(days=7)\n  start_date = end_date - datetime.timedelta(days=14)\n  return query_format.format(\n      dataset=dataset,\n      start_date=start_date,\n      middle_date=middle_date,\n      end_date=end_date)\n\n\n# Most of our queries should simply average a field name to get a ratio showing\n# how often some behavior occurs.\nGENERIC_QUERY_FORMAT = \"\"\"\nSELECT\n  fuzzer,\n  job,\n  1.0 - (1.0 - {min_weight}) * AVG({field_name}) AS new_weight\nFROM\n  {{dataset}}.TestcaseRun\nWHERE\n  _PARTITIONTIME BETWEEN TIMESTAMP('{{start_time}}')\n  AND TIMESTAMP('{{end_time}}')\nGROUP BY\n  fuzzer,\n  job\n\"\"\"\n\n# Heavily reduce the weight for fuzzers which frequently crash on startup. This\n# is indicitave of a very serious problem that makes it highly unlikely that\n# we'll find anything during fuzzing.\nSTARTUP_CRASH_SPECIFICATION = QuerySpecification(\n    query_format=GENERIC_QUERY_FORMAT.format(\n        field_name='startup_crash_count', min_weight=0.10),\n    formatter=_past_day_formatter,\n    reason='frequent startup crashes')\n\n# Reduce weight somewhat for fuzzers with many slow units. If a particular unit\n# runs for so long that we detect it as a slow unit, it usually means that the\n# fuzzer is not making good use of its cycles while running or needs a fix.\nSLOW_UNIT_SPECIFICATION = QuerySpecification(\n    query_format=GENERIC_QUERY_FORMAT.format(\n        field_name='slow_unit_count', min_weight=0.50),\n    formatter=_past_day_formatter,\n    reason='frequent slow units')\n\n# This should end up being very similar to the slow unit specification, and is\n# included for the same reason.\nTIMEOUT_SPECIFICATION = QuerySpecification(\n    query_format=GENERIC_QUERY_FORMAT.format(\n        field_name='timeout_count', min_weight=0.50),\n    formatter=_past_day_formatter,\n    reason='frequent timeouts')\n\n# Fuzzers with extremely frequent OOMs may contain leaks or other issues that\n# signal that they need some improvement. Run with a slightly reduced weight\n# until the issues are fixed.\nOOM_SPECIFICATION = QuerySpecification(\n    query_format=GENERIC_QUERY_FORMAT.format(\n        field_name='oom_count', min_weight=0.50),\n    formatter=_past_day_formatter,\n    reason='frequent OOMs')\n\n# Fuzzers which are crashing frequently may not be making full use of their\n# allotted time for fuzzing, and may end up being more effective once the known\n# issues are fixed. This rule is more lenient than some of the others as even\n# healthy fuzzers are expected to have some crashes.\nCRASH_SPECIFICATION = QuerySpecification(\n    query_format=GENERIC_QUERY_FORMAT.format(\n        field_name='crash_count', min_weight=0.70),\n    formatter=_past_day_formatter,\n    reason='frequent crashes')\n\n# New fuzzers/jobs should run much more frequently than others. In this case, we\n# test the fraction of days for which we have no stats for this fuzzer/job pair\n# and increase if it's nonzero.\nNEW_FUZZER_FORMAT = \"\"\"\nSELECT\n  fuzzer,\n  job,\n  5.0 as new_weight,\n  MIN(_PARTITIONTIME) as first_time\nFROM\n  {dataset}.TestcaseRun\nGROUP BY\n  fuzzer,\n  job\nHAVING\n  first_time >= TIMESTAMP('{cutoff_time}')\n\"\"\"\n\nNEW_FUZZER_SPECIFICATION = QuerySpecification(\n    query_format=NEW_FUZZER_FORMAT,\n    formatter=_new_fuzzer_formatter,\n    reason='new fuzzer')\n\n# Format to query for fuzzers with minimal change in week to week coverage.\nCOVERAGE_UNCHANGED_FORMAT = \"\"\"\nSELECT\n  recent.fuzzer AS fuzzer,\n  recent.job AS job,\n  0.70 as new_weight\nFROM (\n  SELECT\n    fuzzer,\n    job,\n    MAX(edge_coverage / edges_total) AS coverage\n  FROM\n    {dataset}.TestcaseRun\n  WHERE\n    _PARTITIONTIME BETWEEN TIMESTAMP('{middle_date}')\n    AND TIMESTAMP('{end_date}')\n    AND edges_total > 0\n    AND edge_coverage > 0\n  GROUP BY\n    fuzzer,\n    job\n  HAVING\n    coverage <= 1.0) AS recent\nJOIN (\n  SELECT\n    fuzzer,\n    job,\n    MAX(edge_coverage / edges_total) AS coverage\n  FROM\n    {dataset}.TestcaseRun\n  WHERE\n    _PARTITIONTIME BETWEEN TIMESTAMP('{start_date}')\n    AND TIMESTAMP('{middle_date}')\n    AND edges_total > 0\n    AND edge_coverage > 0\n  GROUP BY\n    fuzzer,\n    job\n  HAVING\n    coverage <= 1.0) AS older\nON\n  recent.fuzzer = older.fuzzer\n  AND recent.job = older.job\nWHERE\n  ABS((recent.coverage - older.coverage) / recent.coverage) < 0.01\n\"\"\"\n\nCOVERAGE_UNCHANGED_SPECIFICATION = QuerySpecification(\n    query_format=COVERAGE_UNCHANGED_FORMAT,\n    formatter=_coverage_formatter,\n    reason='coverage flat over past 2 weeks')\n\n# Mappings for which specifications to use for which\nLIBFUZZER_SPECIFICATIONS = [\n    COVERAGE_UNCHANGED_SPECIFICATION,\n    CRASH_SPECIFICATION,\n    NEW_FUZZER_SPECIFICATION,\n    OOM_SPECIFICATION,\n    SLOW_UNIT_SPECIFICATION,\n    STARTUP_CRASH_SPECIFICATION,\n    TIMEOUT_SPECIFICATION,\n]\n\nAFL_SPECIFICATIONS = [\n    CRASH_SPECIFICATION,\n    NEW_FUZZER_SPECIFICATION,\n    STARTUP_CRASH_SPECIFICATION,\n]\n\nRESTORE_DEFAULT_MATCH = SpecificationMatch(\n    new_weight=1.0, reason='no longer matches any weight adjustment rules')\n\n\ndef _query_helper(client, query):\n  \"\"\"Helper function to get fuzzer stats.\"\"\"\n  return client.query(query=query).rows\n\n\ndef _update_match(matches, fuzzer, job, match):\n  \"\"\"Update the weight for a fuzzer/job.\"\"\"\n  key = (fuzzer, job)\n  old_match = matches.get(key, RESTORE_DEFAULT_MATCH)\n\n  new_weight = match.new_weight\n  old_weight = old_match.new_weight\n\n  # Rules that increase weights are expected to take precedence over any that\n  # lower the weight. Issues with new fuzzers may be fixed intraday and other\n  # issues like crashes shouldn't be penalized for them.\n  if old_weight > 1.0:\n    return\n\n  # Always update the weight if the previous value is the default. This is\n  # required to deal with specifications that are meant to set the weight above\n  # 1.0. Otherwise, prioritize only the most penalizing match for this pairing.\n  if old_match == RESTORE_DEFAULT_MATCH or new_weight < old_weight:\n    matches[key] = match\n\n\ndef update_weight_for_target(fuzz_target_name, job, match):\n  \"\"\"Set the weight for a particular target.\"\"\"\n  target_job = data_handler.get_fuzz_target_job(fuzz_target_name, job)\n\n  if not target_job:\n    logs.log_error('FuzzTargetJob for target %s and job %s does not exist.' %\n                   (fuzz_target_name, job))\n    return\n\n  weight = match.new_weight\n  logs.log('Adjusted weight to %f for target %s and job %s (%s).' %\n           (weight, fuzz_target_name, job, match.reason))\n\n  target_job.weight = weight\n  target_job.put()\n\n\ndef update_matches_for_specification(specification, client, engine, matches,\n                                     run_set):\n  \"\"\"Run a query and adjust weights based on a given query specification.\"\"\"\n  query = specification.formatter(specification.query_format,\n                                  fuzzer_stats.dataset_name(engine))\n  results = _query_helper(client, query)\n  for result in results:\n    fuzzer = result['fuzzer']\n    job = result['job']\n    new_weight = result['new_weight']\n\n    run_set.add((fuzzer, job))\n    if new_weight != 1.0:\n      match = SpecificationMatch(\n          new_weight=new_weight, reason=specification.reason)\n      _update_match(matches, fuzzer, job, match)\n\n\ndef update_target_weights_for_engine(client, engine, specifications):\n  \"\"\"Update all fuzz target weights for the specified engine.\"\"\"\n  matches = {}\n  run_set = set()\n\n  # All fuzzers with non-default weights must be tracked with a special\n  # specification. This ensures that they will be restored to normal weight\n  # once conditions causing adjustments are no longer met.\n  target_jobs = data_types.FuzzTargetJob.query(\n      data_types.FuzzTarget.engine == engine).filter(\n          data_types.FuzzTargetJob.weight != 1.0)\n\n  for target_job in target_jobs:\n    matches[(target_job.fuzz_target_name,\n             target_job.job)] = RESTORE_DEFAULT_MATCH\n\n  for match in specifications:\n    update_matches_for_specification(match, client, engine, matches, run_set)\n\n  for (fuzzer, job), match in six.iteritems(matches):\n    if (fuzzer, job) not in run_set:\n      # This ensures that we don't reset weights for fuzzers with problems if\n      # they didn't run in the time covered by our queries.\n      continue\n\n    update_weight_for_target(fuzzer, job, match)\n\n  logs.log('Weight adjustments complete for engine %s.' % engine)\n\n\ndef store_current_weights_in_bigquery():\n  \"\"\"Update a bigquery table containing the daily stats.\"\"\"\n  rows = []\n  target_jobs = ndb_utils.get_all_from_model(data_types.FuzzTargetJob)\n  for target_job in target_jobs:\n    row = {\n        'fuzzer': target_job.fuzz_target_name,\n        'job': target_job.job,\n        'weight': target_job.weight\n    }\n    rows.append(big_query.Insert(row=row, insert_id=None))\n\n  client = big_query.Client(dataset_id='main', table_id='fuzzer_weights')\n  client.insert(rows)\n\n\nclass Handler(base_handler.Handler):\n  \"\"\"Handler to periodically update fuzz target weights based on performance.\"\"\"\n\n  @handler.check_cron()\n  def get(self):\n    \"\"\"Process all fuzz targets and update FuzzTargetJob weights.\"\"\"\n    client = big_query.Client()\n    update_target_weights_for_engine(client, 'libFuzzer',\n                                     LIBFUZZER_SPECIFICATIONS)\n    update_target_weights_for_engine(client, 'afl', AFL_SPECIFICATIONS)\n\n    store_current_weights_in_bigquery()\n", "path": "src/appengine/handlers/cron/fuzzer_weights.py"}], "after_files": [{"content": "# Copyright 2019 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"Manage automatic weight adjustments.\"\"\"\n\nimport collections\nimport datetime\nimport six\n\nfrom base import utils\nfrom datastore import data_handler\nfrom datastore import data_types\nfrom datastore import ndb_utils\nfrom google_cloud_utils import big_query\nfrom handlers import base_handler\nfrom libs import handler\nfrom metrics import fuzzer_stats\nfrom metrics import logs\n\nQuerySpecification = collections.namedtuple(\n    'QuerySpecification', ['query_format', 'formatter', 'reason'])\n\nSpecificationMatch = collections.namedtuple('SpecificationMatch',\n                                            ['new_weight', 'reason'])\n\n\n# Formatters for query specifications.\ndef _past_day_formatter(query_format, dataset):\n  \"\"\"Simple formatter to get stats for the past day.\"\"\"\n  end_time = utils.utcnow().date()\n  start_time = end_time - datetime.timedelta(days=1)\n  return query_format.format(\n      dataset=dataset, start_time=start_time, end_time=end_time)\n\n\ndef _new_fuzzer_formatter(query_format, dataset):\n  \"\"\"Prepare a query to check for new fuzzers from the past week.\"\"\"\n  now = utils.utcnow().date()\n  cutoff_time = now - datetime.timedelta(days=7)\n  return query_format.format(dataset=dataset, cutoff_time=cutoff_time)\n\n\ndef _coverage_formatter(query_format, dataset):\n  \"\"\"Prepare a query to check for changes in coverage week over week.\"\"\"\n  end_date = utils.utcnow().date() - datetime.timedelta(days=1)\n  middle_date = end_date - datetime.timedelta(days=7)\n  start_date = end_date - datetime.timedelta(days=14)\n  return query_format.format(\n      dataset=dataset,\n      start_date=start_date,\n      middle_date=middle_date,\n      end_date=end_date)\n\n\n# Most of our queries should simply average a field name to get a ratio showing\n# how often some behavior occurs.\nGENERIC_QUERY_FORMAT = \"\"\"\nSELECT\n  fuzzer,\n  job,\n  1.0 - (1.0 - {min_weight}) * AVG({field_name}) AS new_weight\nFROM\n  {{dataset}}.TestcaseRun\nWHERE\n  _PARTITIONTIME BETWEEN TIMESTAMP('{{start_time}}')\n  AND TIMESTAMP('{{end_time}}')\nGROUP BY\n  fuzzer,\n  job\n\"\"\"\n\n# Heavily reduce the weight for fuzzers which frequently crash on startup. This\n# is indicitave of a very serious problem that makes it highly unlikely that\n# we'll find anything during fuzzing.\nSTARTUP_CRASH_SPECIFICATION = QuerySpecification(\n    query_format=GENERIC_QUERY_FORMAT.format(\n        field_name='startup_crash_count', min_weight=0.10),\n    formatter=_past_day_formatter,\n    reason='frequent startup crashes')\n\n# Reduce weight somewhat for fuzzers with many slow units. If a particular unit\n# runs for so long that we detect it as a slow unit, it usually means that the\n# fuzzer is not making good use of its cycles while running or needs a fix.\nSLOW_UNIT_SPECIFICATION = QuerySpecification(\n    query_format=GENERIC_QUERY_FORMAT.format(\n        field_name='slow_unit_count', min_weight=0.50),\n    formatter=_past_day_formatter,\n    reason='frequent slow units')\n\n# This should end up being very similar to the slow unit specification, and is\n# included for the same reason.\nTIMEOUT_SPECIFICATION = QuerySpecification(\n    query_format=GENERIC_QUERY_FORMAT.format(\n        field_name='timeout_count', min_weight=0.50),\n    formatter=_past_day_formatter,\n    reason='frequent timeouts')\n\n# Fuzzers with extremely frequent OOMs may contain leaks or other issues that\n# signal that they need some improvement. Run with a slightly reduced weight\n# until the issues are fixed.\nOOM_SPECIFICATION = QuerySpecification(\n    query_format=GENERIC_QUERY_FORMAT.format(\n        field_name='oom_count', min_weight=0.50),\n    formatter=_past_day_formatter,\n    reason='frequent OOMs')\n\n# Fuzzers which are crashing frequently may not be making full use of their\n# allotted time for fuzzing, and may end up being more effective once the known\n# issues are fixed. This rule is more lenient than some of the others as even\n# healthy fuzzers are expected to have some crashes.\nCRASH_SPECIFICATION = QuerySpecification(\n    query_format=GENERIC_QUERY_FORMAT.format(\n        field_name='crash_count', min_weight=0.70),\n    formatter=_past_day_formatter,\n    reason='frequent crashes')\n\n# New fuzzers/jobs should run much more frequently than others. In this case, we\n# test the fraction of days for which we have no stats for this fuzzer/job pair\n# and increase if it's nonzero.\nNEW_FUZZER_FORMAT = \"\"\"\nSELECT\n  fuzzer,\n  job,\n  5.0 as new_weight,\n  MIN(_PARTITIONTIME) as first_time\nFROM\n  {dataset}.TestcaseRun\nGROUP BY\n  fuzzer,\n  job\nHAVING\n  first_time >= TIMESTAMP('{cutoff_time}')\n\"\"\"\n\nNEW_FUZZER_SPECIFICATION = QuerySpecification(\n    query_format=NEW_FUZZER_FORMAT,\n    formatter=_new_fuzzer_formatter,\n    reason='new fuzzer')\n\n# Format to query for fuzzers with minimal change in week to week coverage.\nCOVERAGE_UNCHANGED_FORMAT = \"\"\"\nSELECT\n  recent.fuzzer AS fuzzer,\n  recent.job AS job,\n  0.70 as new_weight\nFROM (\n  SELECT\n    fuzzer,\n    job,\n    MAX(edge_coverage / edges_total) AS coverage\n  FROM\n    {dataset}.TestcaseRun\n  WHERE\n    _PARTITIONTIME BETWEEN TIMESTAMP('{middle_date}')\n    AND TIMESTAMP('{end_date}')\n    AND edges_total > 0\n    AND edge_coverage > 0\n  GROUP BY\n    fuzzer,\n    job\n  HAVING\n    coverage <= 1.0) AS recent\nJOIN (\n  SELECT\n    fuzzer,\n    job,\n    MAX(edge_coverage / edges_total) AS coverage\n  FROM\n    {dataset}.TestcaseRun\n  WHERE\n    _PARTITIONTIME BETWEEN TIMESTAMP('{start_date}')\n    AND TIMESTAMP('{middle_date}')\n    AND edges_total > 0\n    AND edge_coverage > 0\n  GROUP BY\n    fuzzer,\n    job\n  HAVING\n    coverage <= 1.0) AS older\nON\n  recent.fuzzer = older.fuzzer\n  AND recent.job = older.job\nWHERE\n  ABS((recent.coverage - older.coverage) / recent.coverage) < 0.01\n\"\"\"\n\nCOVERAGE_UNCHANGED_SPECIFICATION = QuerySpecification(\n    query_format=COVERAGE_UNCHANGED_FORMAT,\n    formatter=_coverage_formatter,\n    reason='coverage flat over past 2 weeks')\n\n# Mappings for which specifications to use for which\nLIBFUZZER_SPECIFICATIONS = [\n    COVERAGE_UNCHANGED_SPECIFICATION,\n    CRASH_SPECIFICATION,\n    NEW_FUZZER_SPECIFICATION,\n    OOM_SPECIFICATION,\n    SLOW_UNIT_SPECIFICATION,\n    STARTUP_CRASH_SPECIFICATION,\n    TIMEOUT_SPECIFICATION,\n]\n\nAFL_SPECIFICATIONS = [\n    CRASH_SPECIFICATION,\n    NEW_FUZZER_SPECIFICATION,\n    STARTUP_CRASH_SPECIFICATION,\n]\n\nRESTORE_DEFAULT_MATCH = SpecificationMatch(\n    new_weight=1.0, reason='no longer matches any weight adjustment rules')\n\n\ndef _query_helper(client, query):\n  \"\"\"Helper function to get fuzzer stats.\"\"\"\n  return client.query(query=query).rows\n\n\ndef _update_match(matches, fuzzer, job, match):\n  \"\"\"Update the weight for a fuzzer/job.\"\"\"\n  key = (fuzzer, job)\n  old_match = matches.get(key, RESTORE_DEFAULT_MATCH)\n\n  new_weight = match.new_weight\n  old_weight = old_match.new_weight\n\n  # Rules that increase weights are expected to take precedence over any that\n  # lower the weight. Issues with new fuzzers may be fixed intraday and other\n  # issues like crashes shouldn't be penalized for them.\n  if old_weight > 1.0:\n    return\n\n  # Always update the weight if the previous value is the default. This is\n  # required to deal with specifications that are meant to set the weight above\n  # 1.0. Otherwise, prioritize only the most penalizing match for this pairing.\n  if old_match == RESTORE_DEFAULT_MATCH or new_weight < old_weight:\n    matches[key] = match\n\n\ndef update_weight_for_target(fuzz_target_name, job, match):\n  \"\"\"Set the weight for a particular target.\"\"\"\n  target_job = data_handler.get_fuzz_target_job(fuzz_target_name, job)\n\n  if not target_job:\n    # Bail out. This is expected if any fuzzer/job combinations become outdated.\n    return\n\n  weight = match.new_weight\n  logs.log('Adjusted weight to %f for target %s and job %s (%s).' %\n           (weight, fuzz_target_name, job, match.reason))\n\n  target_job.weight = weight\n  target_job.put()\n\n\ndef update_matches_for_specification(specification, client, engine, matches,\n                                     run_set):\n  \"\"\"Run a query and adjust weights based on a given query specification.\"\"\"\n  query = specification.formatter(specification.query_format,\n                                  fuzzer_stats.dataset_name(engine))\n  results = _query_helper(client, query)\n  for result in results:\n    fuzzer = result['fuzzer']\n    job = result['job']\n    new_weight = result['new_weight']\n\n    run_set.add((fuzzer, job))\n    if new_weight != 1.0:\n      match = SpecificationMatch(\n          new_weight=new_weight, reason=specification.reason)\n      _update_match(matches, fuzzer, job, match)\n\n\ndef update_target_weights_for_engine(client, engine, specifications):\n  \"\"\"Update all fuzz target weights for the specified engine.\"\"\"\n  matches = {}\n  run_set = set()\n\n  # All fuzzers with non-default weights must be tracked with a special\n  # specification. This ensures that they will be restored to normal weight\n  # once conditions causing adjustments are no longer met.\n  target_jobs = data_types.FuzzTargetJob.query(\n      data_types.FuzzTarget.engine == engine).filter(\n          data_types.FuzzTargetJob.weight != 1.0)\n\n  for target_job in target_jobs:\n    matches[(target_job.fuzz_target_name,\n             target_job.job)] = RESTORE_DEFAULT_MATCH\n\n  for match in specifications:\n    update_matches_for_specification(match, client, engine, matches, run_set)\n\n  for (fuzzer, job), match in six.iteritems(matches):\n    if (fuzzer, job) not in run_set:\n      # This ensures that we don't reset weights for fuzzers with problems if\n      # they didn't run in the time covered by our queries.\n      continue\n\n    update_weight_for_target(fuzzer, job, match)\n\n  logs.log('Weight adjustments complete for engine %s.' % engine)\n\n\ndef store_current_weights_in_bigquery():\n  \"\"\"Update a bigquery table containing the daily stats.\"\"\"\n  rows = []\n  target_jobs = ndb_utils.get_all_from_model(data_types.FuzzTargetJob)\n  for target_job in target_jobs:\n    row = {\n        'fuzzer': target_job.fuzz_target_name,\n        'job': target_job.job,\n        'weight': target_job.weight\n    }\n    rows.append(big_query.Insert(row=row, insert_id=None))\n\n  client = big_query.Client(dataset_id='main', table_id='fuzzer_weights')\n  client.insert(rows)\n\n\nclass Handler(base_handler.Handler):\n  \"\"\"Handler to periodically update fuzz target weights based on performance.\"\"\"\n\n  @handler.check_cron()\n  def get(self):\n    \"\"\"Process all fuzz targets and update FuzzTargetJob weights.\"\"\"\n    client = big_query.Client()\n    update_target_weights_for_engine(client, 'libFuzzer',\n                                     LIBFUZZER_SPECIFICATIONS)\n    update_target_weights_for_engine(client, 'afl', AFL_SPECIFICATIONS)\n\n    store_current_weights_in_bigquery()\n", "path": "src/appengine/handlers/cron/fuzzer_weights.py"}]}