We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Make ecosystem counts in UI more accurate. 
Currently these counts include withdrawn/invalid entries.

Context: https://github.com/google/osv/issues/468#issuecomment-1169755263

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `gcp/appengine/frontend_handlers.py`
Content:
```
1 # Copyright 2021 Google LLC
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #      http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 """Handlers for the OSV web frontend."""
15 
16 import json
17 import os
18 import math
19 
20 from flask import abort
21 from flask import Blueprint
22 from flask import make_response
23 from flask import redirect
24 from flask import render_template
25 from flask import request
26 import markdown2
27 
28 import cache
29 import osv
30 import rate_limiter
31 import source_mapper
32 import utils
33 
34 blueprint = Blueprint('frontend_handlers', __name__)
35 
36 _PAGE_SIZE = 16
37 _PAGE_LOOKAHEAD = 4
38 _REQUESTS_PER_MIN = 30
39 
40 if utils.is_prod():
41   redis_host = os.environ.get('REDISHOST', 'localhost')
42   redis_port = int(os.environ.get('REDISPORT', 6379))
43   limiter = rate_limiter.RateLimiter(
44       redis_host, redis_port, requests_per_min=_REQUESTS_PER_MIN)
45 
46   @blueprint.before_request
47   def check_rate_limit():
48     ip_addr = request.headers.get('X-Appengine-User-Ip', 'unknown')
49     if not limiter.check_request(ip_addr):
50       abort(429)
51 
52 
53 @blueprint.before_request
54 def check_cors_preflight():
55   """Handle CORS preflight requests."""
56   if request.method != 'OPTIONS':
57     return None
58 
59   response = make_response()
60   response.headers.add('Access-Control-Allow-Origin', 'http://localhost:8080')
61   response.headers.add('Access-Control-Allow-Methods', '*')
62   response.headers.add('Access-Control-Allow-Headers', '*')
63   return response
64 
65 
66 @blueprint.after_request
67 def add_cors_headers(response):
68   """Add CORS headers."""
69   response.headers.add('Access-Control-Allow-Origin', 'http://localhost:8080')
70   return response
71 
72 
73 @blueprint.route('/v2/')
74 def index_v2():
75   return redirect('/')
76 
77 
78 @blueprint.route('/v2/<path:subpath>')
79 def index_v2_with_subpath(subpath):
80   return redirect('/' + subpath)
81 
82 
83 @blueprint.route('/')
84 def index():
85   return render_template(
86       'home.html', ecosystem_counts=osv_get_ecosystem_counts_cached())
87 
88 
89 @blueprint.route('/about')
90 def about():
91   return render_template('about.html')
92 
93 
94 @blueprint.route('/list')
95 def list_vulnerabilities():
96   """Main page."""
97   query = request.args.get('q', '')
98   page = int(request.args.get('page', 1))
99   ecosystem = request.args.get('ecosystem')
100   results = osv_query(query, page, False, ecosystem)
101 
102   # Fetch ecosystems by default. As an optimization, skip when rendering page
103   # fragments.
104   ecosystem_counts = osv_get_ecosystem_counts_cached(
105   ) if not request.headers.get('Turbo-Frame') else None
106 
107   return render_template(
108       'list.html',
109       page=page,
110       query=query,
111       selected_ecosystem=ecosystem,
112       ecosystem_counts=ecosystem_counts,
113       vulnerabilities=results['items'])
114 
115 
116 @blueprint.route('/vulnerability/<vuln_id>')
117 def vulnerability(vuln_id):
118   """Vulnerability page."""
119   vuln = osv_get_by_id(vuln_id)
120   return render_template('vulnerability.html', vulnerability=vuln)
121 
122 
123 def bug_to_response(bug, detailed=True):
124   """Convert a Bug entity to a response object."""
125   response = osv.vulnerability_to_dict(bug.to_vulnerability())
126   response.update({
127       'isFixed': bug.is_fixed,
128       'invalid': bug.status == osv.BugStatus.INVALID
129   })
130 
131   if detailed:
132     add_links(response)
133     add_source_info(bug, response)
134   return response
135 
136 
137 def add_links(bug):
138   """Add VCS links where possible."""
139 
140   for entry in bug.get('affected', []):
141     for i, affected_range in enumerate(entry.get('ranges', [])):
142       affected_range['id'] = i
143       if affected_range['type'] != 'GIT':
144         continue
145 
146       repo_url = affected_range.get('repo')
147       if not repo_url:
148         continue
149 
150       for event in affected_range.get('events', []):
151         if event.get('introduced'):
152           event['introduced_link'] = _commit_to_link(repo_url,
153                                                      event['introduced'])
154           continue
155 
156         if event.get('fixed'):
157           event['fixed_link'] = _commit_to_link(repo_url, event['fixed'])
158           continue
159 
160         if event.get('limit'):
161           event['limit_link'] = _commit_to_link(repo_url, event['limit'])
162           continue
163 
164 
165 def add_source_info(bug, response):
166   """Add source information to `response`."""
167   if bug.source_of_truth == osv.SourceOfTruth.INTERNAL:
168     response['source'] = 'INTERNAL'
169     return
170 
171   source_repo = osv.get_source_repository(bug.source)
172   if not source_repo or not source_repo.link:
173     return
174 
175   source_path = osv.source_path(source_repo, bug)
176   response['source'] = source_repo.link + source_path
177   response['source_link'] = response['source']
178 
179 
180 def _commit_to_link(repo_url, commit):
181   """Convert commit to link."""
182   vcs = source_mapper.get_vcs_viewer_for_url(repo_url)
183   if not vcs:
184     return None
185 
186   if ':' not in commit:
187     return vcs.get_source_url_for_revision(commit)
188 
189   commit_parts = commit.split(':')
190   if len(commit_parts) != 2:
191     return None
192 
193   start, end = commit_parts
194   if start == 'unknown':
195     return None
196 
197   return vcs.get_source_url_for_revision_diff(start, end)
198 
199 
200 def osv_get_ecosystems():
201   """Get list of ecosystems."""
202   query = osv.Bug.query(projection=[osv.Bug.ecosystem], distinct=True)
203   return sorted([bug.ecosystem[0] for bug in query if bug.ecosystem],
204                 key=str.lower)
205 
206 
207 # TODO: Figure out how to skip cache when testing
208 @cache.instance.cached(
209     timeout=24 * 60 * 60, key_prefix='osv_get_ecosystem_counts')
210 def osv_get_ecosystem_counts_cached():
211   """Get count of vulnerabilities per ecosystem, cached"""
212   return osv_get_ecosystem_counts()
213 
214 
215 def osv_get_ecosystem_counts():
216   """Get count of vulnerabilities per ecosystem."""
217   counts = {}
218   ecosystems = osv_get_ecosystems()
219   for ecosystem in ecosystems:
220     base_ecosystem = ecosystem.split(':')[0]
221     if base_ecosystem not in counts:
222       counts[base_ecosystem] = 0
223 
224     counts[base_ecosystem] += osv.Bug.query(
225         osv.Bug.ecosystem == ecosystem).count()
226 
227   return counts
228 
229 
230 def osv_query(search_string, page, affected_only, ecosystem):
231   """Run an OSV query."""
232   query = osv.Bug.query(osv.Bug.status == osv.BugStatus.PROCESSED,
233                         osv.Bug.public == True)  # pylint: disable=singleton-comparison
234 
235   if search_string:
236     query = query.filter(osv.Bug.search_indices == search_string.lower())
237 
238   if affected_only:
239     query = query.filter(osv.Bug.has_affected == True)  # pylint: disable=singleton-comparison
240 
241   if ecosystem:
242     query = query.filter(osv.Bug.ecosystem == ecosystem)
243 
244   query = query.order(-osv.Bug.last_modified)
245   total = query.count()
246   results = {
247       'total': total,
248       'items': [],
249   }
250 
251   bugs, _, _ = query.fetch_page(
252       page_size=_PAGE_SIZE, offset=(page - 1) * _PAGE_SIZE)
253   for bug in bugs:
254     results['items'].append(bug_to_response(bug, detailed=False))
255 
256   return results
257 
258 
259 def osv_get_by_id(vuln_id):
260   """Gets bug details from its id. If invalid, aborts the request."""
261   if not vuln_id:
262     abort(400)
263     return None
264 
265   bug = osv.Bug.get_by_id(vuln_id)
266   if not bug:
267     abort(404)
268     return None
269 
270   if bug.status == osv.BugStatus.UNPROCESSED:
271     abort(404)
272     return None
273 
274   if not bug.public:
275     abort(403)
276     return None
277 
278   return bug_to_response(bug)
279 
280 
281 @blueprint.app_template_filter('event_type')
282 def event_type(event):
283   """Get the type from an event."""
284   if event.get('introduced'):
285     return 'Introduced'
286   if event.get('fixed'):
287     return 'Fixed'
288   if event.get('limit'):
289     return 'Limit'
290   if event.get('last_affected'):
291     return 'Last affected'
292 
293   return None
294 
295 
296 @blueprint.app_template_filter('event_link')
297 def event_link(event):
298   """Get the link from an event."""
299   if event.get('introduced_link'):
300     return event['introduced_link']
301   if event.get('fixed_link'):
302     return event['fixed_link']
303   if event.get('limit_link'):
304     return event['limit_link']
305   if event.get('last_affected_link'):
306     return event['last_affected_link']
307 
308   return None
309 
310 
311 @blueprint.app_template_filter('event_value')
312 def event_value(event):
313   """Get the value from an event."""
314   if event.get('introduced'):
315     return event['introduced']
316   if event.get('fixed'):
317     return event['fixed']
318   if event.get('limit'):
319     return event['limit']
320   if event.get('last_affected'):
321     return event['last_affected']
322 
323   return None
324 
325 
326 @blueprint.app_template_filter('should_collapse')
327 def should_collapse(affected):
328   """Whether if we should collapse the package tab bar."""
329   total_package_length = sum(
330       [len(entry.get('package', {}).get('name', '')) for entry in affected])
331   return total_package_length > 70 or len(affected) > 5
332 
333 
334 @blueprint.app_template_filter('group_versions')
335 def group_versions(versions):
336   """Group versions by prefix."""
337   groups = {}
338 
339   for version in sorted(versions):
340     if '.' not in version:
341       groups.setdefault('Other', []).append(version)
342       continue
343 
344     label = version.split('.')[0] + '.*'
345     groups.setdefault(label, []).append(version)
346 
347   return groups
348 
349 
350 @blueprint.app_template_filter('markdown')
351 def markdown(text):
352   """Render markdown."""
353   if text:
354     return markdown2.markdown(text, extras=['fenced-code-blocks'])
355 
356   return ''
357 
358 
359 @blueprint.app_template_filter('display_json')
360 def display_json(data):
361   # We can't use the default `tojson` filter as it's intended for code (and
362   # escapes characters like '<' to '\u003c'). We want to render the JSON for
363   # display purposes and use HTML escaping ('&lt;') instead so it's rendered
364   # as '<'.
365   return json.dumps(data, indent=4)
366 
367 
368 @blueprint.app_template_filter('log')
369 def logarithm(n):
370   return math.log(n)
371
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/gcp/appengine/frontend_handlers.py b/gcp/appengine/frontend_handlers.py
--- a/gcp/appengine/frontend_handlers.py
+++ b/gcp/appengine/frontend_handlers.py
@@ -217,12 +217,15 @@
   counts = {}
   ecosystems = osv_get_ecosystems()
   for ecosystem in ecosystems:
-    base_ecosystem = ecosystem.split(':')[0]
-    if base_ecosystem not in counts:
-      counts[base_ecosystem] = 0
+    if ':' in ecosystem:
+      # Count by the base ecosystem index. Otherwise we'll overcount as a
+      # single entry may refer to multiple sub-ecosystems.
+      continue
 
-    counts[base_ecosystem] += osv.Bug.query(
-        osv.Bug.ecosystem == ecosystem).count()
+    counts[ecosystem] = osv.Bug.query(
+        osv.Bug.ecosystem == ecosystem,
+        osv.Bug.public == True,  # pylint: disable=singleton-comparison
+        osv.Bug.status == osv.BugStatus.PROCESSED).count()
 
   return counts

{"golden_diff": "diff --git a/gcp/appengine/frontend_handlers.py b/gcp/appengine/frontend_handlers.py\n--- a/gcp/appengine/frontend_handlers.py\n+++ b/gcp/appengine/frontend_handlers.py\n@@ -217,12 +217,15 @@\n   counts = {}\n   ecosystems = osv_get_ecosystems()\n   for ecosystem in ecosystems:\n-    base_ecosystem = ecosystem.split(':')[0]\n-    if base_ecosystem not in counts:\n-      counts[base_ecosystem] = 0\n+    if ':' in ecosystem:\n+      # Count by the base ecosystem index. Otherwise we'll overcount as a\n+      # single entry may refer to multiple sub-ecosystems.\n+      continue\n \n-    counts[base_ecosystem] += osv.Bug.query(\n-        osv.Bug.ecosystem == ecosystem).count()\n+    counts[ecosystem] = osv.Bug.query(\n+        osv.Bug.ecosystem == ecosystem,\n+        osv.Bug.public == True,  # pylint: disable=singleton-comparison\n+        osv.Bug.status == osv.BugStatus.PROCESSED).count()\n \n   return counts\n", "issue": "Make ecosystem counts in UI more accurate. \nCurrently these counts include withdrawn/invalid entries.\r\n\r\nContext: https://github.com/google/osv/issues/468#issuecomment-1169755263\n", "before_files": [{"content": "# Copyright 2021 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"Handlers for the OSV web frontend.\"\"\"\n\nimport json\nimport os\nimport math\n\nfrom flask import abort\nfrom flask import Blueprint\nfrom flask import make_response\nfrom flask import redirect\nfrom flask import render_template\nfrom flask import request\nimport markdown2\n\nimport cache\nimport osv\nimport rate_limiter\nimport source_mapper\nimport utils\n\nblueprint = Blueprint('frontend_handlers', __name__)\n\n_PAGE_SIZE = 16\n_PAGE_LOOKAHEAD = 4\n_REQUESTS_PER_MIN = 30\n\nif utils.is_prod():\n  redis_host = os.environ.get('REDISHOST', 'localhost')\n  redis_port = int(os.environ.get('REDISPORT', 6379))\n  limiter = rate_limiter.RateLimiter(\n      redis_host, redis_port, requests_per_min=_REQUESTS_PER_MIN)\n\n  @blueprint.before_request\n  def check_rate_limit():\n    ip_addr = request.headers.get('X-Appengine-User-Ip', 'unknown')\n    if not limiter.check_request(ip_addr):\n      abort(429)\n\n\n@blueprint.before_request\ndef check_cors_preflight():\n  \"\"\"Handle CORS preflight requests.\"\"\"\n  if request.method != 'OPTIONS':\n    return None\n\n  response = make_response()\n  response.headers.add('Access-Control-Allow-Origin', 'http://localhost:8080')\n  response.headers.add('Access-Control-Allow-Methods', '*')\n  response.headers.add('Access-Control-Allow-Headers', '*')\n  return response\n\n\n@blueprint.after_request\ndef add_cors_headers(response):\n  \"\"\"Add CORS headers.\"\"\"\n  response.headers.add('Access-Control-Allow-Origin', 'http://localhost:8080')\n  return response\n\n\n@blueprint.route('/v2/')\ndef index_v2():\n  return redirect('/')\n\n\n@blueprint.route('/v2/<path:subpath>')\ndef index_v2_with_subpath(subpath):\n  return redirect('/' + subpath)\n\n\n@blueprint.route('/')\ndef index():\n  return render_template(\n      'home.html', ecosystem_counts=osv_get_ecosystem_counts_cached())\n\n\n@blueprint.route('/about')\ndef about():\n  return render_template('about.html')\n\n\n@blueprint.route('/list')\ndef list_vulnerabilities():\n  \"\"\"Main page.\"\"\"\n  query = request.args.get('q', '')\n  page = int(request.args.get('page', 1))\n  ecosystem = request.args.get('ecosystem')\n  results = osv_query(query, page, False, ecosystem)\n\n  # Fetch ecosystems by default. As an optimization, skip when rendering page\n  # fragments.\n  ecosystem_counts = osv_get_ecosystem_counts_cached(\n  ) if not request.headers.get('Turbo-Frame') else None\n\n  return render_template(\n      'list.html',\n      page=page,\n      query=query,\n      selected_ecosystem=ecosystem,\n      ecosystem_counts=ecosystem_counts,\n      vulnerabilities=results['items'])\n\n\n@blueprint.route('/vulnerability/<vuln_id>')\ndef vulnerability(vuln_id):\n  \"\"\"Vulnerability page.\"\"\"\n  vuln = osv_get_by_id(vuln_id)\n  return render_template('vulnerability.html', vulnerability=vuln)\n\n\ndef bug_to_response(bug, detailed=True):\n  \"\"\"Convert a Bug entity to a response object.\"\"\"\n  response = osv.vulnerability_to_dict(bug.to_vulnerability())\n  response.update({\n      'isFixed': bug.is_fixed,\n      'invalid': bug.status == osv.BugStatus.INVALID\n  })\n\n  if detailed:\n    add_links(response)\n    add_source_info(bug, response)\n  return response\n\n\ndef add_links(bug):\n  \"\"\"Add VCS links where possible.\"\"\"\n\n  for entry in bug.get('affected', []):\n    for i, affected_range in enumerate(entry.get('ranges', [])):\n      affected_range['id'] = i\n      if affected_range['type'] != 'GIT':\n        continue\n\n      repo_url = affected_range.get('repo')\n      if not repo_url:\n        continue\n\n      for event in affected_range.get('events', []):\n        if event.get('introduced'):\n          event['introduced_link'] = _commit_to_link(repo_url,\n                                                     event['introduced'])\n          continue\n\n        if event.get('fixed'):\n          event['fixed_link'] = _commit_to_link(repo_url, event['fixed'])\n          continue\n\n        if event.get('limit'):\n          event['limit_link'] = _commit_to_link(repo_url, event['limit'])\n          continue\n\n\ndef add_source_info(bug, response):\n  \"\"\"Add source information to `response`.\"\"\"\n  if bug.source_of_truth == osv.SourceOfTruth.INTERNAL:\n    response['source'] = 'INTERNAL'\n    return\n\n  source_repo = osv.get_source_repository(bug.source)\n  if not source_repo or not source_repo.link:\n    return\n\n  source_path = osv.source_path(source_repo, bug)\n  response['source'] = source_repo.link + source_path\n  response['source_link'] = response['source']\n\n\ndef _commit_to_link(repo_url, commit):\n  \"\"\"Convert commit to link.\"\"\"\n  vcs = source_mapper.get_vcs_viewer_for_url(repo_url)\n  if not vcs:\n    return None\n\n  if ':' not in commit:\n    return vcs.get_source_url_for_revision(commit)\n\n  commit_parts = commit.split(':')\n  if len(commit_parts) != 2:\n    return None\n\n  start, end = commit_parts\n  if start == 'unknown':\n    return None\n\n  return vcs.get_source_url_for_revision_diff(start, end)\n\n\ndef osv_get_ecosystems():\n  \"\"\"Get list of ecosystems.\"\"\"\n  query = osv.Bug.query(projection=[osv.Bug.ecosystem], distinct=True)\n  return sorted([bug.ecosystem[0] for bug in query if bug.ecosystem],\n                key=str.lower)\n\n\n# TODO: Figure out how to skip cache when testing\n@cache.instance.cached(\n    timeout=24 * 60 * 60, key_prefix='osv_get_ecosystem_counts')\ndef osv_get_ecosystem_counts_cached():\n  \"\"\"Get count of vulnerabilities per ecosystem, cached\"\"\"\n  return osv_get_ecosystem_counts()\n\n\ndef osv_get_ecosystem_counts():\n  \"\"\"Get count of vulnerabilities per ecosystem.\"\"\"\n  counts = {}\n  ecosystems = osv_get_ecosystems()\n  for ecosystem in ecosystems:\n    base_ecosystem = ecosystem.split(':')[0]\n    if base_ecosystem not in counts:\n      counts[base_ecosystem] = 0\n\n    counts[base_ecosystem] += osv.Bug.query(\n        osv.Bug.ecosystem == ecosystem).count()\n\n  return counts\n\n\ndef osv_query(search_string, page, affected_only, ecosystem):\n  \"\"\"Run an OSV query.\"\"\"\n  query = osv.Bug.query(osv.Bug.status == osv.BugStatus.PROCESSED,\n                        osv.Bug.public == True)  # pylint: disable=singleton-comparison\n\n  if search_string:\n    query = query.filter(osv.Bug.search_indices == search_string.lower())\n\n  if affected_only:\n    query = query.filter(osv.Bug.has_affected == True)  # pylint: disable=singleton-comparison\n\n  if ecosystem:\n    query = query.filter(osv.Bug.ecosystem == ecosystem)\n\n  query = query.order(-osv.Bug.last_modified)\n  total = query.count()\n  results = {\n      'total': total,\n      'items': [],\n  }\n\n  bugs, _, _ = query.fetch_page(\n      page_size=_PAGE_SIZE, offset=(page - 1) * _PAGE_SIZE)\n  for bug in bugs:\n    results['items'].append(bug_to_response(bug, detailed=False))\n\n  return results\n\n\ndef osv_get_by_id(vuln_id):\n  \"\"\"Gets bug details from its id. If invalid, aborts the request.\"\"\"\n  if not vuln_id:\n    abort(400)\n    return None\n\n  bug = osv.Bug.get_by_id(vuln_id)\n  if not bug:\n    abort(404)\n    return None\n\n  if bug.status == osv.BugStatus.UNPROCESSED:\n    abort(404)\n    return None\n\n  if not bug.public:\n    abort(403)\n    return None\n\n  return bug_to_response(bug)\n\n\n@blueprint.app_template_filter('event_type')\ndef event_type(event):\n  \"\"\"Get the type from an event.\"\"\"\n  if event.get('introduced'):\n    return 'Introduced'\n  if event.get('fixed'):\n    return 'Fixed'\n  if event.get('limit'):\n    return 'Limit'\n  if event.get('last_affected'):\n    return 'Last affected'\n\n  return None\n\n\n@blueprint.app_template_filter('event_link')\ndef event_link(event):\n  \"\"\"Get the link from an event.\"\"\"\n  if event.get('introduced_link'):\n    return event['introduced_link']\n  if event.get('fixed_link'):\n    return event['fixed_link']\n  if event.get('limit_link'):\n    return event['limit_link']\n  if event.get('last_affected_link'):\n    return event['last_affected_link']\n\n  return None\n\n\n@blueprint.app_template_filter('event_value')\ndef event_value(event):\n  \"\"\"Get the value from an event.\"\"\"\n  if event.get('introduced'):\n    return event['introduced']\n  if event.get('fixed'):\n    return event['fixed']\n  if event.get('limit'):\n    return event['limit']\n  if event.get('last_affected'):\n    return event['last_affected']\n\n  return None\n\n\n@blueprint.app_template_filter('should_collapse')\ndef should_collapse(affected):\n  \"\"\"Whether if we should collapse the package tab bar.\"\"\"\n  total_package_length = sum(\n      [len(entry.get('package', {}).get('name', '')) for entry in affected])\n  return total_package_length > 70 or len(affected) > 5\n\n\n@blueprint.app_template_filter('group_versions')\ndef group_versions(versions):\n  \"\"\"Group versions by prefix.\"\"\"\n  groups = {}\n\n  for version in sorted(versions):\n    if '.' not in version:\n      groups.setdefault('Other', []).append(version)\n      continue\n\n    label = version.split('.')[0] + '.*'\n    groups.setdefault(label, []).append(version)\n\n  return groups\n\n\n@blueprint.app_template_filter('markdown')\ndef markdown(text):\n  \"\"\"Render markdown.\"\"\"\n  if text:\n    return markdown2.markdown(text, extras=['fenced-code-blocks'])\n\n  return ''\n\n\n@blueprint.app_template_filter('display_json')\ndef display_json(data):\n  # We can't use the default `tojson` filter as it's intended for code (and\n  # escapes characters like '<' to '\\u003c'). We want to render the JSON for\n  # display purposes and use HTML escaping ('&lt;') instead so it's rendered\n  # as '<'.\n  return json.dumps(data, indent=4)\n\n\n@blueprint.app_template_filter('log')\ndef logarithm(n):\n  return math.log(n)\n", "path": "gcp/appengine/frontend_handlers.py"}], "after_files": [{"content": "# Copyright 2021 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"Handlers for the OSV web frontend.\"\"\"\n\nimport json\nimport os\nimport math\n\nfrom flask import abort\nfrom flask import Blueprint\nfrom flask import make_response\nfrom flask import redirect\nfrom flask import render_template\nfrom flask import request\nimport markdown2\n\nimport cache\nimport osv\nimport rate_limiter\nimport source_mapper\nimport utils\n\nblueprint = Blueprint('frontend_handlers', __name__)\n\n_PAGE_SIZE = 16\n_PAGE_LOOKAHEAD = 4\n_REQUESTS_PER_MIN = 30\n\nif utils.is_prod():\n  redis_host = os.environ.get('REDISHOST', 'localhost')\n  redis_port = int(os.environ.get('REDISPORT', 6379))\n  limiter = rate_limiter.RateLimiter(\n      redis_host, redis_port, requests_per_min=_REQUESTS_PER_MIN)\n\n  @blueprint.before_request\n  def check_rate_limit():\n    ip_addr = request.headers.get('X-Appengine-User-Ip', 'unknown')\n    if not limiter.check_request(ip_addr):\n      abort(429)\n\n\n@blueprint.before_request\ndef check_cors_preflight():\n  \"\"\"Handle CORS preflight requests.\"\"\"\n  if request.method != 'OPTIONS':\n    return None\n\n  response = make_response()\n  response.headers.add('Access-Control-Allow-Origin', 'http://localhost:8080')\n  response.headers.add('Access-Control-Allow-Methods', '*')\n  response.headers.add('Access-Control-Allow-Headers', '*')\n  return response\n\n\n@blueprint.after_request\ndef add_cors_headers(response):\n  \"\"\"Add CORS headers.\"\"\"\n  response.headers.add('Access-Control-Allow-Origin', 'http://localhost:8080')\n  return response\n\n\n@blueprint.route('/v2/')\ndef index_v2():\n  return redirect('/')\n\n\n@blueprint.route('/v2/<path:subpath>')\ndef index_v2_with_subpath(subpath):\n  return redirect('/' + subpath)\n\n\n@blueprint.route('/')\ndef index():\n  return render_template(\n      'home.html', ecosystem_counts=osv_get_ecosystem_counts_cached())\n\n\n@blueprint.route('/about')\ndef about():\n  return render_template('about.html')\n\n\n@blueprint.route('/list')\ndef list_vulnerabilities():\n  \"\"\"Main page.\"\"\"\n  query = request.args.get('q', '')\n  page = int(request.args.get('page', 1))\n  ecosystem = request.args.get('ecosystem')\n  results = osv_query(query, page, False, ecosystem)\n\n  # Fetch ecosystems by default. As an optimization, skip when rendering page\n  # fragments.\n  ecosystem_counts = osv_get_ecosystem_counts_cached(\n  ) if not request.headers.get('Turbo-Frame') else None\n\n  return render_template(\n      'list.html',\n      page=page,\n      query=query,\n      selected_ecosystem=ecosystem,\n      ecosystem_counts=ecosystem_counts,\n      vulnerabilities=results['items'])\n\n\n@blueprint.route('/vulnerability/<vuln_id>')\ndef vulnerability(vuln_id):\n  \"\"\"Vulnerability page.\"\"\"\n  vuln = osv_get_by_id(vuln_id)\n  return render_template('vulnerability.html', vulnerability=vuln)\n\n\ndef bug_to_response(bug, detailed=True):\n  \"\"\"Convert a Bug entity to a response object.\"\"\"\n  response = osv.vulnerability_to_dict(bug.to_vulnerability())\n  response.update({\n      'isFixed': bug.is_fixed,\n      'invalid': bug.status == osv.BugStatus.INVALID\n  })\n\n  if detailed:\n    add_links(response)\n    add_source_info(bug, response)\n  return response\n\n\ndef add_links(bug):\n  \"\"\"Add VCS links where possible.\"\"\"\n\n  for entry in bug.get('affected', []):\n    for i, affected_range in enumerate(entry.get('ranges', [])):\n      affected_range['id'] = i\n      if affected_range['type'] != 'GIT':\n        continue\n\n      repo_url = affected_range.get('repo')\n      if not repo_url:\n        continue\n\n      for event in affected_range.get('events', []):\n        if event.get('introduced'):\n          event['introduced_link'] = _commit_to_link(repo_url,\n                                                     event['introduced'])\n          continue\n\n        if event.get('fixed'):\n          event['fixed_link'] = _commit_to_link(repo_url, event['fixed'])\n          continue\n\n        if event.get('limit'):\n          event['limit_link'] = _commit_to_link(repo_url, event['limit'])\n          continue\n\n\ndef add_source_info(bug, response):\n  \"\"\"Add source information to `response`.\"\"\"\n  if bug.source_of_truth == osv.SourceOfTruth.INTERNAL:\n    response['source'] = 'INTERNAL'\n    return\n\n  source_repo = osv.get_source_repository(bug.source)\n  if not source_repo or not source_repo.link:\n    return\n\n  source_path = osv.source_path(source_repo, bug)\n  response['source'] = source_repo.link + source_path\n  response['source_link'] = response['source']\n\n\ndef _commit_to_link(repo_url, commit):\n  \"\"\"Convert commit to link.\"\"\"\n  vcs = source_mapper.get_vcs_viewer_for_url(repo_url)\n  if not vcs:\n    return None\n\n  if ':' not in commit:\n    return vcs.get_source_url_for_revision(commit)\n\n  commit_parts = commit.split(':')\n  if len(commit_parts) != 2:\n    return None\n\n  start, end = commit_parts\n  if start == 'unknown':\n    return None\n\n  return vcs.get_source_url_for_revision_diff(start, end)\n\n\ndef osv_get_ecosystems():\n  \"\"\"Get list of ecosystems.\"\"\"\n  query = osv.Bug.query(projection=[osv.Bug.ecosystem], distinct=True)\n  return sorted([bug.ecosystem[0] for bug in query if bug.ecosystem],\n                key=str.lower)\n\n\n# TODO: Figure out how to skip cache when testing\n@cache.instance.cached(\n    timeout=24 * 60 * 60, key_prefix='osv_get_ecosystem_counts')\ndef osv_get_ecosystem_counts_cached():\n  \"\"\"Get count of vulnerabilities per ecosystem, cached\"\"\"\n  return osv_get_ecosystem_counts()\n\n\ndef osv_get_ecosystem_counts():\n  \"\"\"Get count of vulnerabilities per ecosystem.\"\"\"\n  counts = {}\n  ecosystems = osv_get_ecosystems()\n  for ecosystem in ecosystems:\n    if ':' in ecosystem:\n      # Count by the base ecosystem index. Otherwise we'll overcount as a\n      # single entry may refer to multiple sub-ecosystems.\n      continue\n\n    counts[ecosystem] = osv.Bug.query(\n        osv.Bug.ecosystem == ecosystem,\n        osv.Bug.public == True,  # pylint: disable=singleton-comparison\n        osv.Bug.status == osv.BugStatus.PROCESSED).count()\n\n  return counts\n\n\ndef osv_query(search_string, page, affected_only, ecosystem):\n  \"\"\"Run an OSV query.\"\"\"\n  query = osv.Bug.query(osv.Bug.status == osv.BugStatus.PROCESSED,\n                        osv.Bug.public == True)  # pylint: disable=singleton-comparison\n\n  if search_string:\n    query = query.filter(osv.Bug.search_indices == search_string.lower())\n\n  if affected_only:\n    query = query.filter(osv.Bug.has_affected == True)  # pylint: disable=singleton-comparison\n\n  if ecosystem:\n    query = query.filter(osv.Bug.ecosystem == ecosystem)\n\n  query = query.order(-osv.Bug.last_modified)\n  total = query.count()\n  results = {\n      'total': total,\n      'items': [],\n  }\n\n  bugs, _, _ = query.fetch_page(\n      page_size=_PAGE_SIZE, offset=(page - 1) * _PAGE_SIZE)\n  for bug in bugs:\n    results['items'].append(bug_to_response(bug, detailed=False))\n\n  return results\n\n\ndef osv_get_by_id(vuln_id):\n  \"\"\"Gets bug details from its id. If invalid, aborts the request.\"\"\"\n  if not vuln_id:\n    abort(400)\n    return None\n\n  bug = osv.Bug.get_by_id(vuln_id)\n  if not bug:\n    abort(404)\n    return None\n\n  if bug.status == osv.BugStatus.UNPROCESSED:\n    abort(404)\n    return None\n\n  if not bug.public:\n    abort(403)\n    return None\n\n  return bug_to_response(bug)\n\n\n@blueprint.app_template_filter('event_type')\ndef event_type(event):\n  \"\"\"Get the type from an event.\"\"\"\n  if event.get('introduced'):\n    return 'Introduced'\n  if event.get('fixed'):\n    return 'Fixed'\n  if event.get('limit'):\n    return 'Limit'\n  if event.get('last_affected'):\n    return 'Last affected'\n\n  return None\n\n\n@blueprint.app_template_filter('event_link')\ndef event_link(event):\n  \"\"\"Get the link from an event.\"\"\"\n  if event.get('introduced_link'):\n    return event['introduced_link']\n  if event.get('fixed_link'):\n    return event['fixed_link']\n  if event.get('limit_link'):\n    return event['limit_link']\n  if event.get('last_affected_link'):\n    return event['last_affected_link']\n\n  return None\n\n\n@blueprint.app_template_filter('event_value')\ndef event_value(event):\n  \"\"\"Get the value from an event.\"\"\"\n  if event.get('introduced'):\n    return event['introduced']\n  if event.get('fixed'):\n    return event['fixed']\n  if event.get('limit'):\n    return event['limit']\n  if event.get('last_affected'):\n    return event['last_affected']\n\n  return None\n\n\n@blueprint.app_template_filter('should_collapse')\ndef should_collapse(affected):\n  \"\"\"Whether if we should collapse the package tab bar.\"\"\"\n  total_package_length = sum(\n      [len(entry.get('package', {}).get('name', '')) for entry in affected])\n  return total_package_length > 70 or len(affected) > 5\n\n\n@blueprint.app_template_filter('group_versions')\ndef group_versions(versions):\n  \"\"\"Group versions by prefix.\"\"\"\n  groups = {}\n\n  for version in sorted(versions):\n    if '.' not in version:\n      groups.setdefault('Other', []).append(version)\n      continue\n\n    label = version.split('.')[0] + '.*'\n    groups.setdefault(label, []).append(version)\n\n  return groups\n\n\n@blueprint.app_template_filter('markdown')\ndef markdown(text):\n  \"\"\"Render markdown.\"\"\"\n  if text:\n    return markdown2.markdown(text, extras=['fenced-code-blocks'])\n\n  return ''\n\n\n@blueprint.app_template_filter('display_json')\ndef display_json(data):\n  # We can't use the default `tojson` filter as it's intended for code (and\n  # escapes characters like '<' to '\\u003c'). We want to render the JSON for\n  # display purposes and use HTML escaping ('&lt;') instead so it's rendered\n  # as '<'.\n  return json.dumps(data, indent=4)\n\n\n@blueprint.app_template_filter('log')\ndef logarithm(n):\n  return math.log(n)\n", "path": "gcp/appengine/frontend_handlers.py"}]}