We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Make version indexes more scalable.
Currently, all indexes for package+version are stored in the same `Bug` entity. 

This has large benefits in terms of read/write latency and consistency, but we are now seeing some entries with large fanout of these indexes (high package count x version count).

Some options here
- Use partial indexes (package count + version count) and rely on Datastore's zig zag merge. Will need to evaluate performance here.
- Break out indexes into its own separate entity. 

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `gcp/api/server.py`
Content:
```
1 # Copyright 2021 Google LLC
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #      http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 """API server implementation."""
15 
16 import argparse
17 from concurrent import futures
18 import functools
19 import logging
20 import os
21 import sys
22 import time
23 
24 from google.cloud import ndb
25 import grpc
26 from packageurl import PackageURL
27 
28 import osv
29 from osv import ecosystems
30 from osv import semver_index
31 import osv_service_v1_pb2
32 import osv_service_v1_pb2_grpc
33 
34 _PROJECT = 'oss-vdb'
35 _OSS_FUZZ_TRACKER_URL = 'https://bugs.chromium.org/p/oss-fuzz/issues/detail?id='
36 
37 _SHUTDOWN_GRACE_DURATION = 5
38 
39 _AUTHORIZATION_HEADER_PREFIX = 'Bearer '
40 _EXPECTED_AUDIENCE = 'https://db.oss-fuzz.com'
41 
42 _ndb_client = ndb.Client()
43 
44 
45 def ndb_context(func):
46   """Wrapper to create an NDB context."""
47 
48   @functools.wraps(func)
49   def wrapper(*args, **kwargs):
50     with _ndb_client.context():
51       return func(*args, **kwargs)
52 
53   return wrapper
54 
55 
56 class OSVServicer(osv_service_v1_pb2_grpc.OSVServicer):
57   """V1 OSV servicer."""
58 
59   @ndb_context
60   def GetVulnById(self, request, context):
61     """Return a `Vulnerability` object for a given OSV ID.
62     """
63     bug = osv.Bug.get_by_id(request.id)
64     if not bug or bug.status == osv.BugStatus.UNPROCESSED:
65       context.abort(grpc.StatusCode.NOT_FOUND, 'Bug not found.')
66       return None
67 
68     if not bug.public:
69       context.abort(grpc.StatusCode.PERMISSION_DENIED, 'Permission denied.')
70       return None
71 
72     return bug_to_response(bug)
73 
74   @ndb_context
75   def QueryAffected(self, request, context):
76     """Query vulnerabilities for a particular project at a given commit or
77     version."""
78     if request.query.HasField('package'):
79       package_name = request.query.package.name
80       ecosystem = request.query.package.ecosystem
81       purl = request.query.package.purl
82     else:
83       package_name = ''
84       ecosystem = ''
85       purl = ''
86 
87     purl_version = None
88     if purl:
89       try:
90         parsed_purl = PackageURL.from_string(purl)
91         purl_version = parsed_purl.version
92         purl = _clean_purl(parsed_purl).to_string()
93       except ValueError:
94         context.abort(grpc.StatusCode.INVALID_ARGUMENT, 'Invalid Package URL.')
95         return None
96 
97     if request.query.WhichOneof('param') == 'commit':
98       bugs = query_by_commit(request.query.commit, to_response=bug_to_response)
99     elif purl and purl_version:
100       bugs = query_by_version(
101           package_name,
102           ecosystem,
103           purl,
104           purl_version,
105           to_response=bug_to_response)
106     elif request.query.WhichOneof('param') == 'version':
107       bugs = query_by_version(
108           package_name,
109           ecosystem,
110           purl,
111           request.query.version,
112           to_response=bug_to_response)
113     else:
114       context.abort(grpc.StatusCode.INVALID_ARGUMENT, 'Invalid query.')
115       return None
116 
117     return osv_service_v1_pb2.VulnerabilityList(vulns=bugs)
118 
119   def GetVulnByIdNew(self, request, context):
120     """Return a `Vulnerability` object for a given OSV ID.
121     """
122     return self.GetVulnById(request, context)
123 
124   def QueryAffectedNew(self, request, context):
125     """Query vulnerabilities for a particular project at a given commit or
126     version."""
127     return self.QueryAffected(request, context)
128 
129 
130 def bug_to_response(bug):
131   """Convert a Bug entity to a response object."""
132   return bug.to_vulnerability(include_source=True)
133 
134 
135 def _get_bugs(bug_ids, to_response=bug_to_response):
136   """Get bugs from bug ids."""
137   bugs = ndb.get_multi([ndb.Key(osv.Bug, bug_id) for bug_id in bug_ids])
138   return [
139       to_response(bug)
140       for bug in bugs
141       if bug and bug.status == osv.BugStatus.PROCESSED
142   ]
143 
144 
145 def _clean_purl(purl):
146   """Clean a purl object."""
147   values = purl.to_dict()
148   values.pop('version', None)
149   values.pop('subpath', None)
150   values.pop('qualifiers', None)
151   return PackageURL(**values)
152 
153 
154 def query_by_commit(commit, to_response=bug_to_response):
155   """Query by commit."""
156   query = osv.AffectedCommit.query(osv.AffectedCommit.commit == commit,
157                                    osv.AffectedCommit.public == True)  # pylint: disable=singleton-comparison
158   bug_ids = []
159   for affected_commit in query:
160     bug_ids.append(affected_commit.bug_id)
161 
162   return _get_bugs(bug_ids, to_response=to_response)
163 
164 
165 def _is_semver_affected(affected_packages, package_name, ecosystem, purl,
166                         version):
167   """Returns whether or not the given version is within an affected SEMVER
168   range."""
169   version = semver_index.parse(version)
170 
171   affected = False
172   for affected_package in affected_packages:
173     if package_name and package_name != affected_package.package.name:
174       continue
175 
176     if ecosystem and ecosystem != affected_package.package.ecosystem:
177       continue
178 
179     if purl and purl != affected_package.package.purl:
180       continue
181 
182     for affected_range in affected_package.ranges:
183       if affected_range.type != 'SEMVER':
184         continue
185 
186       for event in osv.sorted_events('', affected_range.type,
187                                      affected_range.events):
188         if (event.type == 'introduced' and
189             (event.value == '0' or version >= semver_index.parse(event.value))):
190           affected = True
191 
192         if event.type == 'fixed' and version >= semver_index.parse(event.value):
193           affected = False
194 
195   return affected
196 
197 
198 def _query_by_semver(query, package_name, ecosystem, purl, version):
199   """Query by semver."""
200   if not semver_index.is_valid(version):
201     return []
202 
203   query = query.filter(
204       osv.Bug.semver_fixed_indexes > semver_index.normalize(version))
205 
206   return [
207       bug for bug in query if _is_semver_affected(
208           bug.affected_packages, package_name, ecosystem, purl, version)
209   ]
210 
211 
212 def _query_by_generic_version(base_query, version):
213   """Query by generic version."""
214   # Try without normalizing.
215   query = base_query.filter(osv.Bug.affected_fuzzy == version)
216   results = list(query)
217   if results:
218     return results
219 
220   # Try again after normalizing.
221   query = base_query.filter(
222       osv.Bug.affected_fuzzy == osv.normalize_tag(version))
223   return list(query)
224 
225 
226 def query_by_version(project,
227                      ecosystem,
228                      purl,
229                      version,
230                      to_response=bug_to_response):
231   """Query by (fuzzy) version."""
232   ecosystem_info = ecosystems.get(ecosystem)
233   is_semver = ecosystem_info and ecosystem_info.is_semver
234   if project:
235     query = osv.Bug.query(osv.Bug.status == osv.BugStatus.PROCESSED,
236                           osv.Bug.project == project, osv.Bug.public == True)  # pylint: disable=singleton-comparison
237   elif purl:
238     query = osv.Bug.query(osv.Bug.status == osv.BugStatus.PROCESSED,
239                           osv.Bug.purl == purl, osv.Bug.public == True)  # pylint: disable=singleton-comparison
240   else:
241     return []
242 
243   if ecosystem:
244     query = query.filter(osv.Bug.ecosystem == ecosystem)
245 
246   bugs = []
247   if ecosystem:
248     if is_semver:
249       # Ecosystem supports semver only.
250       bugs.extend(_query_by_semver(query, project, ecosystem, purl, version))
251     else:
252       bugs.extend(_query_by_generic_version(query, version))
253   else:
254     # Unspecified ecosystem. Try both.
255     bugs.extend(_query_by_semver(query, project, ecosystem, purl, version))
256     bugs.extend(_query_by_generic_version(query, version))
257 
258   return [to_response(bug) for bug in bugs]
259 
260 
261 def serve(port):
262   """Configures and runs the bookstore API server."""
263   server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
264   osv_service_v1_pb2_grpc.add_OSVServicer_to_server(OSVServicer(), server)
265   server.add_insecure_port('[::]:{}'.format(port))
266   server.start()
267 
268   print('Listening on port {}'.format(port))
269   try:
270     while True:
271       time.sleep(3600)
272   except KeyboardInterrupt:
273     server.stop(_SHUTDOWN_GRACE_DURATION)
274 
275 
276 def main():
277   """Entrypoint."""
278   logging.basicConfig(stream=sys.stderr)
279   logging.getLogger().setLevel(logging.INFO)
280 
281   parser = argparse.ArgumentParser(
282       formatter_class=argparse.RawDescriptionHelpFormatter)
283   parser.add_argument(
284       '--port',
285       type=int,
286       default=None,
287       help='The port to listen on.'
288       'If arg is not set, will listen on the $PORT env var.'
289       'If env var is empty, defaults to 8000.')
290 
291   args = parser.parse_args()
292   port = args.port
293   if not port:
294     port = os.environ.get('PORT')
295   if not port:
296     port = 8000
297 
298   serve(port)
299 
300 
301 if __name__ == '__main__':
302   main()
303
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/gcp/api/server.py b/gcp/api/server.py
--- a/gcp/api/server.py
+++ b/gcp/api/server.py
@@ -195,6 +195,36 @@
   return affected
 
 
+def _is_version_affected(affected_packages,
+                         package_name,
+                         ecosystem,
+                         purl,
+                         version,
+                         normalize=False):
+  """Returns whether or not the given version is within an affected ECOSYSTEM
+  range."""
+  for affected_package in affected_packages:
+    if package_name and package_name != affected_package.package.name:
+      continue
+
+    if ecosystem and ecosystem != affected_package.package.ecosystem:
+      continue
+
+    if purl and purl != affected_package.package.purl:
+      continue
+
+    if normalize:
+      if any(
+          osv.normalize_tag(version) == osv.normalize_tag(v)
+          for v in affected_package.versions):
+        return True
+    else:
+      if version in affected_package.versions:
+        return True
+
+  return False
+
+
 def _query_by_semver(query, package_name, ecosystem, purl, version):
   """Query by semver."""
   if not semver_index.is_valid(version):
@@ -209,18 +239,29 @@
   ]
 
 
-def _query_by_generic_version(base_query, version):
+def _query_by_generic_version(base_query, project, ecosystem, purl, version):
   """Query by generic version."""
   # Try without normalizing.
   query = base_query.filter(osv.Bug.affected_fuzzy == version)
-  results = list(query)
+  results = [
+      bug for bug in query if _is_version_affected(
+          bug.affected_packages, project, ecosystem, purl, version)
+  ]
   if results:
     return results
 
   # Try again after normalizing.
-  query = base_query.filter(
-      osv.Bug.affected_fuzzy == osv.normalize_tag(version))
-  return list(query)
+  version = osv.normalize_tag(version)
+  query = base_query.filter(osv.Bug.affected_fuzzy == version)
+  return [
+      bug for bug in query if _is_version_affected(
+          bug.affected_packages,
+          project,
+          ecosystem,
+          purl,
+          version,
+          normalize=True)
+  ]
 
 
 def query_by_version(project,
@@ -249,11 +290,13 @@
       # Ecosystem supports semver only.
       bugs.extend(_query_by_semver(query, project, ecosystem, purl, version))
     else:
-      bugs.extend(_query_by_generic_version(query, version))
+      bugs.extend(
+          _query_by_generic_version(query, project, ecosystem, purl, version))
   else:
     # Unspecified ecosystem. Try both.
     bugs.extend(_query_by_semver(query, project, ecosystem, purl, version))
-    bugs.extend(_query_by_generic_version(query, version))
+    bugs.extend(
+        _query_by_generic_version(query, project, ecosystem, purl, version))
 
   return [to_response(bug) for bug in bugs]

{"golden_diff": "diff --git a/gcp/api/server.py b/gcp/api/server.py\n--- a/gcp/api/server.py\n+++ b/gcp/api/server.py\n@@ -195,6 +195,36 @@\n   return affected\n \n \n+def _is_version_affected(affected_packages,\n+                         package_name,\n+                         ecosystem,\n+                         purl,\n+                         version,\n+                         normalize=False):\n+  \"\"\"Returns whether or not the given version is within an affected ECOSYSTEM\n+  range.\"\"\"\n+  for affected_package in affected_packages:\n+    if package_name and package_name != affected_package.package.name:\n+      continue\n+\n+    if ecosystem and ecosystem != affected_package.package.ecosystem:\n+      continue\n+\n+    if purl and purl != affected_package.package.purl:\n+      continue\n+\n+    if normalize:\n+      if any(\n+          osv.normalize_tag(version) == osv.normalize_tag(v)\n+          for v in affected_package.versions):\n+        return True\n+    else:\n+      if version in affected_package.versions:\n+        return True\n+\n+  return False\n+\n+\n def _query_by_semver(query, package_name, ecosystem, purl, version):\n   \"\"\"Query by semver.\"\"\"\n   if not semver_index.is_valid(version):\n@@ -209,18 +239,29 @@\n   ]\n \n \n-def _query_by_generic_version(base_query, version):\n+def _query_by_generic_version(base_query, project, ecosystem, purl, version):\n   \"\"\"Query by generic version.\"\"\"\n   # Try without normalizing.\n   query = base_query.filter(osv.Bug.affected_fuzzy == version)\n-  results = list(query)\n+  results = [\n+      bug for bug in query if _is_version_affected(\n+          bug.affected_packages, project, ecosystem, purl, version)\n+  ]\n   if results:\n     return results\n \n   # Try again after normalizing.\n-  query = base_query.filter(\n-      osv.Bug.affected_fuzzy == osv.normalize_tag(version))\n-  return list(query)\n+  version = osv.normalize_tag(version)\n+  query = base_query.filter(osv.Bug.affected_fuzzy == version)\n+  return [\n+      bug for bug in query if _is_version_affected(\n+          bug.affected_packages,\n+          project,\n+          ecosystem,\n+          purl,\n+          version,\n+          normalize=True)\n+  ]\n \n \n def query_by_version(project,\n@@ -249,11 +290,13 @@\n       # Ecosystem supports semver only.\n       bugs.extend(_query_by_semver(query, project, ecosystem, purl, version))\n     else:\n-      bugs.extend(_query_by_generic_version(query, version))\n+      bugs.extend(\n+          _query_by_generic_version(query, project, ecosystem, purl, version))\n   else:\n     # Unspecified ecosystem. Try both.\n     bugs.extend(_query_by_semver(query, project, ecosystem, purl, version))\n-    bugs.extend(_query_by_generic_version(query, version))\n+    bugs.extend(\n+        _query_by_generic_version(query, project, ecosystem, purl, version))\n \n   return [to_response(bug) for bug in bugs]\n", "issue": "Make version indexes more scalable.\nCurrently, all indexes for package+version are stored in the same `Bug` entity. \r\n\r\nThis has large benefits in terms of read/write latency and consistency, but we are now seeing some entries with large fanout of these indexes (high package count x version count).\r\n\r\nSome options here\r\n- Use partial indexes (package count + version count) and rely on Datastore's zig zag merge. Will need to evaluate performance here.\r\n- Break out indexes into its own separate entity. \n", "before_files": [{"content": "# Copyright 2021 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"API server implementation.\"\"\"\n\nimport argparse\nfrom concurrent import futures\nimport functools\nimport logging\nimport os\nimport sys\nimport time\n\nfrom google.cloud import ndb\nimport grpc\nfrom packageurl import PackageURL\n\nimport osv\nfrom osv import ecosystems\nfrom osv import semver_index\nimport osv_service_v1_pb2\nimport osv_service_v1_pb2_grpc\n\n_PROJECT = 'oss-vdb'\n_OSS_FUZZ_TRACKER_URL = 'https://bugs.chromium.org/p/oss-fuzz/issues/detail?id='\n\n_SHUTDOWN_GRACE_DURATION = 5\n\n_AUTHORIZATION_HEADER_PREFIX = 'Bearer '\n_EXPECTED_AUDIENCE = 'https://db.oss-fuzz.com'\n\n_ndb_client = ndb.Client()\n\n\ndef ndb_context(func):\n  \"\"\"Wrapper to create an NDB context.\"\"\"\n\n  @functools.wraps(func)\n  def wrapper(*args, **kwargs):\n    with _ndb_client.context():\n      return func(*args, **kwargs)\n\n  return wrapper\n\n\nclass OSVServicer(osv_service_v1_pb2_grpc.OSVServicer):\n  \"\"\"V1 OSV servicer.\"\"\"\n\n  @ndb_context\n  def GetVulnById(self, request, context):\n    \"\"\"Return a `Vulnerability` object for a given OSV ID.\n    \"\"\"\n    bug = osv.Bug.get_by_id(request.id)\n    if not bug or bug.status == osv.BugStatus.UNPROCESSED:\n      context.abort(grpc.StatusCode.NOT_FOUND, 'Bug not found.')\n      return None\n\n    if not bug.public:\n      context.abort(grpc.StatusCode.PERMISSION_DENIED, 'Permission denied.')\n      return None\n\n    return bug_to_response(bug)\n\n  @ndb_context\n  def QueryAffected(self, request, context):\n    \"\"\"Query vulnerabilities for a particular project at a given commit or\n    version.\"\"\"\n    if request.query.HasField('package'):\n      package_name = request.query.package.name\n      ecosystem = request.query.package.ecosystem\n      purl = request.query.package.purl\n    else:\n      package_name = ''\n      ecosystem = ''\n      purl = ''\n\n    purl_version = None\n    if purl:\n      try:\n        parsed_purl = PackageURL.from_string(purl)\n        purl_version = parsed_purl.version\n        purl = _clean_purl(parsed_purl).to_string()\n      except ValueError:\n        context.abort(grpc.StatusCode.INVALID_ARGUMENT, 'Invalid Package URL.')\n        return None\n\n    if request.query.WhichOneof('param') == 'commit':\n      bugs = query_by_commit(request.query.commit, to_response=bug_to_response)\n    elif purl and purl_version:\n      bugs = query_by_version(\n          package_name,\n          ecosystem,\n          purl,\n          purl_version,\n          to_response=bug_to_response)\n    elif request.query.WhichOneof('param') == 'version':\n      bugs = query_by_version(\n          package_name,\n          ecosystem,\n          purl,\n          request.query.version,\n          to_response=bug_to_response)\n    else:\n      context.abort(grpc.StatusCode.INVALID_ARGUMENT, 'Invalid query.')\n      return None\n\n    return osv_service_v1_pb2.VulnerabilityList(vulns=bugs)\n\n  def GetVulnByIdNew(self, request, context):\n    \"\"\"Return a `Vulnerability` object for a given OSV ID.\n    \"\"\"\n    return self.GetVulnById(request, context)\n\n  def QueryAffectedNew(self, request, context):\n    \"\"\"Query vulnerabilities for a particular project at a given commit or\n    version.\"\"\"\n    return self.QueryAffected(request, context)\n\n\ndef bug_to_response(bug):\n  \"\"\"Convert a Bug entity to a response object.\"\"\"\n  return bug.to_vulnerability(include_source=True)\n\n\ndef _get_bugs(bug_ids, to_response=bug_to_response):\n  \"\"\"Get bugs from bug ids.\"\"\"\n  bugs = ndb.get_multi([ndb.Key(osv.Bug, bug_id) for bug_id in bug_ids])\n  return [\n      to_response(bug)\n      for bug in bugs\n      if bug and bug.status == osv.BugStatus.PROCESSED\n  ]\n\n\ndef _clean_purl(purl):\n  \"\"\"Clean a purl object.\"\"\"\n  values = purl.to_dict()\n  values.pop('version', None)\n  values.pop('subpath', None)\n  values.pop('qualifiers', None)\n  return PackageURL(**values)\n\n\ndef query_by_commit(commit, to_response=bug_to_response):\n  \"\"\"Query by commit.\"\"\"\n  query = osv.AffectedCommit.query(osv.AffectedCommit.commit == commit,\n                                   osv.AffectedCommit.public == True)  # pylint: disable=singleton-comparison\n  bug_ids = []\n  for affected_commit in query:\n    bug_ids.append(affected_commit.bug_id)\n\n  return _get_bugs(bug_ids, to_response=to_response)\n\n\ndef _is_semver_affected(affected_packages, package_name, ecosystem, purl,\n                        version):\n  \"\"\"Returns whether or not the given version is within an affected SEMVER\n  range.\"\"\"\n  version = semver_index.parse(version)\n\n  affected = False\n  for affected_package in affected_packages:\n    if package_name and package_name != affected_package.package.name:\n      continue\n\n    if ecosystem and ecosystem != affected_package.package.ecosystem:\n      continue\n\n    if purl and purl != affected_package.package.purl:\n      continue\n\n    for affected_range in affected_package.ranges:\n      if affected_range.type != 'SEMVER':\n        continue\n\n      for event in osv.sorted_events('', affected_range.type,\n                                     affected_range.events):\n        if (event.type == 'introduced' and\n            (event.value == '0' or version >= semver_index.parse(event.value))):\n          affected = True\n\n        if event.type == 'fixed' and version >= semver_index.parse(event.value):\n          affected = False\n\n  return affected\n\n\ndef _query_by_semver(query, package_name, ecosystem, purl, version):\n  \"\"\"Query by semver.\"\"\"\n  if not semver_index.is_valid(version):\n    return []\n\n  query = query.filter(\n      osv.Bug.semver_fixed_indexes > semver_index.normalize(version))\n\n  return [\n      bug for bug in query if _is_semver_affected(\n          bug.affected_packages, package_name, ecosystem, purl, version)\n  ]\n\n\ndef _query_by_generic_version(base_query, version):\n  \"\"\"Query by generic version.\"\"\"\n  # Try without normalizing.\n  query = base_query.filter(osv.Bug.affected_fuzzy == version)\n  results = list(query)\n  if results:\n    return results\n\n  # Try again after normalizing.\n  query = base_query.filter(\n      osv.Bug.affected_fuzzy == osv.normalize_tag(version))\n  return list(query)\n\n\ndef query_by_version(project,\n                     ecosystem,\n                     purl,\n                     version,\n                     to_response=bug_to_response):\n  \"\"\"Query by (fuzzy) version.\"\"\"\n  ecosystem_info = ecosystems.get(ecosystem)\n  is_semver = ecosystem_info and ecosystem_info.is_semver\n  if project:\n    query = osv.Bug.query(osv.Bug.status == osv.BugStatus.PROCESSED,\n                          osv.Bug.project == project, osv.Bug.public == True)  # pylint: disable=singleton-comparison\n  elif purl:\n    query = osv.Bug.query(osv.Bug.status == osv.BugStatus.PROCESSED,\n                          osv.Bug.purl == purl, osv.Bug.public == True)  # pylint: disable=singleton-comparison\n  else:\n    return []\n\n  if ecosystem:\n    query = query.filter(osv.Bug.ecosystem == ecosystem)\n\n  bugs = []\n  if ecosystem:\n    if is_semver:\n      # Ecosystem supports semver only.\n      bugs.extend(_query_by_semver(query, project, ecosystem, purl, version))\n    else:\n      bugs.extend(_query_by_generic_version(query, version))\n  else:\n    # Unspecified ecosystem. Try both.\n    bugs.extend(_query_by_semver(query, project, ecosystem, purl, version))\n    bugs.extend(_query_by_generic_version(query, version))\n\n  return [to_response(bug) for bug in bugs]\n\n\ndef serve(port):\n  \"\"\"Configures and runs the bookstore API server.\"\"\"\n  server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))\n  osv_service_v1_pb2_grpc.add_OSVServicer_to_server(OSVServicer(), server)\n  server.add_insecure_port('[::]:{}'.format(port))\n  server.start()\n\n  print('Listening on port {}'.format(port))\n  try:\n    while True:\n      time.sleep(3600)\n  except KeyboardInterrupt:\n    server.stop(_SHUTDOWN_GRACE_DURATION)\n\n\ndef main():\n  \"\"\"Entrypoint.\"\"\"\n  logging.basicConfig(stream=sys.stderr)\n  logging.getLogger().setLevel(logging.INFO)\n\n  parser = argparse.ArgumentParser(\n      formatter_class=argparse.RawDescriptionHelpFormatter)\n  parser.add_argument(\n      '--port',\n      type=int,\n      default=None,\n      help='The port to listen on.'\n      'If arg is not set, will listen on the $PORT env var.'\n      'If env var is empty, defaults to 8000.')\n\n  args = parser.parse_args()\n  port = args.port\n  if not port:\n    port = os.environ.get('PORT')\n  if not port:\n    port = 8000\n\n  serve(port)\n\n\nif __name__ == '__main__':\n  main()\n", "path": "gcp/api/server.py"}], "after_files": [{"content": "# Copyright 2021 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"API server implementation.\"\"\"\n\nimport argparse\nfrom concurrent import futures\nimport functools\nimport logging\nimport os\nimport sys\nimport time\n\nfrom google.cloud import ndb\nimport grpc\nfrom packageurl import PackageURL\n\nimport osv\nfrom osv import ecosystems\nfrom osv import semver_index\nimport osv_service_v1_pb2\nimport osv_service_v1_pb2_grpc\n\n_PROJECT = 'oss-vdb'\n_OSS_FUZZ_TRACKER_URL = 'https://bugs.chromium.org/p/oss-fuzz/issues/detail?id='\n\n_SHUTDOWN_GRACE_DURATION = 5\n\n_AUTHORIZATION_HEADER_PREFIX = 'Bearer '\n_EXPECTED_AUDIENCE = 'https://db.oss-fuzz.com'\n\n_ndb_client = ndb.Client()\n\n\ndef ndb_context(func):\n  \"\"\"Wrapper to create an NDB context.\"\"\"\n\n  @functools.wraps(func)\n  def wrapper(*args, **kwargs):\n    with _ndb_client.context():\n      return func(*args, **kwargs)\n\n  return wrapper\n\n\nclass OSVServicer(osv_service_v1_pb2_grpc.OSVServicer):\n  \"\"\"V1 OSV servicer.\"\"\"\n\n  @ndb_context\n  def GetVulnById(self, request, context):\n    \"\"\"Return a `Vulnerability` object for a given OSV ID.\n    \"\"\"\n    bug = osv.Bug.get_by_id(request.id)\n    if not bug or bug.status == osv.BugStatus.UNPROCESSED:\n      context.abort(grpc.StatusCode.NOT_FOUND, 'Bug not found.')\n      return None\n\n    if not bug.public:\n      context.abort(grpc.StatusCode.PERMISSION_DENIED, 'Permission denied.')\n      return None\n\n    return bug_to_response(bug)\n\n  @ndb_context\n  def QueryAffected(self, request, context):\n    \"\"\"Query vulnerabilities for a particular project at a given commit or\n    version.\"\"\"\n    if request.query.HasField('package'):\n      package_name = request.query.package.name\n      ecosystem = request.query.package.ecosystem\n      purl = request.query.package.purl\n    else:\n      package_name = ''\n      ecosystem = ''\n      purl = ''\n\n    purl_version = None\n    if purl:\n      try:\n        parsed_purl = PackageURL.from_string(purl)\n        purl_version = parsed_purl.version\n        purl = _clean_purl(parsed_purl).to_string()\n      except ValueError:\n        context.abort(grpc.StatusCode.INVALID_ARGUMENT, 'Invalid Package URL.')\n        return None\n\n    if request.query.WhichOneof('param') == 'commit':\n      bugs = query_by_commit(request.query.commit, to_response=bug_to_response)\n    elif purl and purl_version:\n      bugs = query_by_version(\n          package_name,\n          ecosystem,\n          purl,\n          purl_version,\n          to_response=bug_to_response)\n    elif request.query.WhichOneof('param') == 'version':\n      bugs = query_by_version(\n          package_name,\n          ecosystem,\n          purl,\n          request.query.version,\n          to_response=bug_to_response)\n    else:\n      context.abort(grpc.StatusCode.INVALID_ARGUMENT, 'Invalid query.')\n      return None\n\n    return osv_service_v1_pb2.VulnerabilityList(vulns=bugs)\n\n  def GetVulnByIdNew(self, request, context):\n    \"\"\"Return a `Vulnerability` object for a given OSV ID.\n    \"\"\"\n    return self.GetVulnById(request, context)\n\n  def QueryAffectedNew(self, request, context):\n    \"\"\"Query vulnerabilities for a particular project at a given commit or\n    version.\"\"\"\n    return self.QueryAffected(request, context)\n\n\ndef bug_to_response(bug):\n  \"\"\"Convert a Bug entity to a response object.\"\"\"\n  return bug.to_vulnerability(include_source=True)\n\n\ndef _get_bugs(bug_ids, to_response=bug_to_response):\n  \"\"\"Get bugs from bug ids.\"\"\"\n  bugs = ndb.get_multi([ndb.Key(osv.Bug, bug_id) for bug_id in bug_ids])\n  return [\n      to_response(bug)\n      for bug in bugs\n      if bug and bug.status == osv.BugStatus.PROCESSED\n  ]\n\n\ndef _clean_purl(purl):\n  \"\"\"Clean a purl object.\"\"\"\n  values = purl.to_dict()\n  values.pop('version', None)\n  values.pop('subpath', None)\n  values.pop('qualifiers', None)\n  return PackageURL(**values)\n\n\ndef query_by_commit(commit, to_response=bug_to_response):\n  \"\"\"Query by commit.\"\"\"\n  query = osv.AffectedCommit.query(osv.AffectedCommit.commit == commit,\n                                   osv.AffectedCommit.public == True)  # pylint: disable=singleton-comparison\n  bug_ids = []\n  for affected_commit in query:\n    bug_ids.append(affected_commit.bug_id)\n\n  return _get_bugs(bug_ids, to_response=to_response)\n\n\ndef _is_semver_affected(affected_packages, package_name, ecosystem, purl,\n                        version):\n  \"\"\"Returns whether or not the given version is within an affected SEMVER\n  range.\"\"\"\n  version = semver_index.parse(version)\n\n  affected = False\n  for affected_package in affected_packages:\n    if package_name and package_name != affected_package.package.name:\n      continue\n\n    if ecosystem and ecosystem != affected_package.package.ecosystem:\n      continue\n\n    if purl and purl != affected_package.package.purl:\n      continue\n\n    for affected_range in affected_package.ranges:\n      if affected_range.type != 'SEMVER':\n        continue\n\n      for event in osv.sorted_events('', affected_range.type,\n                                     affected_range.events):\n        if (event.type == 'introduced' and\n            (event.value == '0' or version >= semver_index.parse(event.value))):\n          affected = True\n\n        if event.type == 'fixed' and version >= semver_index.parse(event.value):\n          affected = False\n\n  return affected\n\n\ndef _is_version_affected(affected_packages,\n                         package_name,\n                         ecosystem,\n                         purl,\n                         version,\n                         normalize=False):\n  \"\"\"Returns whether or not the given version is within an affected ECOSYSTEM\n  range.\"\"\"\n  for affected_package in affected_packages:\n    if package_name and package_name != affected_package.package.name:\n      continue\n\n    if ecosystem and ecosystem != affected_package.package.ecosystem:\n      continue\n\n    if purl and purl != affected_package.package.purl:\n      continue\n\n    if normalize:\n      if any(\n          osv.normalize_tag(version) == osv.normalize_tag(v)\n          for v in affected_package.versions):\n        return True\n    else:\n      if version in affected_package.versions:\n        return True\n\n  return False\n\n\ndef _query_by_semver(query, package_name, ecosystem, purl, version):\n  \"\"\"Query by semver.\"\"\"\n  if not semver_index.is_valid(version):\n    return []\n\n  query = query.filter(\n      osv.Bug.semver_fixed_indexes > semver_index.normalize(version))\n\n  return [\n      bug for bug in query if _is_semver_affected(\n          bug.affected_packages, package_name, ecosystem, purl, version)\n  ]\n\n\ndef _query_by_generic_version(base_query, project, ecosystem, purl, version):\n  \"\"\"Query by generic version.\"\"\"\n  # Try without normalizing.\n  query = base_query.filter(osv.Bug.affected_fuzzy == version)\n  results = [\n      bug for bug in query if _is_version_affected(\n          bug.affected_packages, project, ecosystem, purl, version)\n  ]\n  if results:\n    return results\n\n  # Try again after normalizing.\n  version = osv.normalize_tag(version)\n  query = base_query.filter(osv.Bug.affected_fuzzy == version)\n  return [\n      bug for bug in query if _is_version_affected(\n          bug.affected_packages,\n          project,\n          ecosystem,\n          purl,\n          version,\n          normalize=True)\n  ]\n\n\ndef query_by_version(project,\n                     ecosystem,\n                     purl,\n                     version,\n                     to_response=bug_to_response):\n  \"\"\"Query by (fuzzy) version.\"\"\"\n  ecosystem_info = ecosystems.get(ecosystem)\n  is_semver = ecosystem_info and ecosystem_info.is_semver\n  if project:\n    query = osv.Bug.query(osv.Bug.status == osv.BugStatus.PROCESSED,\n                          osv.Bug.project == project, osv.Bug.public == True)  # pylint: disable=singleton-comparison\n  elif purl:\n    query = osv.Bug.query(osv.Bug.status == osv.BugStatus.PROCESSED,\n                          osv.Bug.purl == purl, osv.Bug.public == True)  # pylint: disable=singleton-comparison\n  else:\n    return []\n\n  if ecosystem:\n    query = query.filter(osv.Bug.ecosystem == ecosystem)\n\n  bugs = []\n  if ecosystem:\n    if is_semver:\n      # Ecosystem supports semver only.\n      bugs.extend(_query_by_semver(query, project, ecosystem, purl, version))\n    else:\n      bugs.extend(\n          _query_by_generic_version(query, project, ecosystem, purl, version))\n  else:\n    # Unspecified ecosystem. Try both.\n    bugs.extend(_query_by_semver(query, project, ecosystem, purl, version))\n    bugs.extend(\n        _query_by_generic_version(query, project, ecosystem, purl, version))\n\n  return [to_response(bug) for bug in bugs]\n\n\ndef serve(port):\n  \"\"\"Configures and runs the bookstore API server.\"\"\"\n  server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))\n  osv_service_v1_pb2_grpc.add_OSVServicer_to_server(OSVServicer(), server)\n  server.add_insecure_port('[::]:{}'.format(port))\n  server.start()\n\n  print('Listening on port {}'.format(port))\n  try:\n    while True:\n      time.sleep(3600)\n  except KeyboardInterrupt:\n    server.stop(_SHUTDOWN_GRACE_DURATION)\n\n\ndef main():\n  \"\"\"Entrypoint.\"\"\"\n  logging.basicConfig(stream=sys.stderr)\n  logging.getLogger().setLevel(logging.INFO)\n\n  parser = argparse.ArgumentParser(\n      formatter_class=argparse.RawDescriptionHelpFormatter)\n  parser.add_argument(\n      '--port',\n      type=int,\n      default=None,\n      help='The port to listen on.'\n      'If arg is not set, will listen on the $PORT env var.'\n      'If env var is empty, defaults to 8000.')\n\n  args = parser.parse_args()\n  port = args.port\n  if not port:\n    port = os.environ.get('PORT')\n  if not port:\n    port = 8000\n\n  serve(port)\n\n\nif __name__ == '__main__':\n  main()\n", "path": "gcp/api/server.py"}]}