We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
many api json downloads fail with internal server errors
The following pages all give internal server errors; the problem appears to occur on collections whose documents contain attributes that are object ids (other than _id, which is handled separately), which causes errors like:

TypeError: ObjectId('4f7052dc5009fb18ef000811') is not JSON serializable

The following pages all have this problem:

http://www.lmfdb.org/api/MaassWaveForms/Coefficients?_format=json
http://www.lmfdb.org/api/MaassWaveForms/Coefficients.chunks?_format=json
http://www.lmfdb.org/api/MaassWaveForms/Coefficients.files?_format=json
http://www.lmfdb.org/api/MaassWaveForms/FS?_format=json
http://www.lmfdb.org/api/MaassWaveForms/Table.chunks?_format=json
http://www.lmfdb.org/api/MaassWaveForms/maassform_plots?_format=json
http://www.lmfdb.org/api/SL2Zsubgroups/groups?_format=json
http://www.lmfdb.org/api/modularforms2/webchar.chunks?_format=json
http://www.lmfdb.org/api/modularforms2/webeigenvalues.chunks?_format=json
http://www.lmfdb.org/api/modularforms2/webmodformspace.chunks?_format=json
http://www.lmfdb.org/api/modularforms2/webnewforms.chunks?_format=json
http://www.lmfdb.org/api/siegel_modular_forms/samples?_format=json


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `lmfdb/api/api.py`
Content:
```
1 # -*- coding: utf-8 -*-
2 
3 import pymongo
4 import urllib2
5 ASC = pymongo.ASCENDING
6 DESC = pymongo.DESCENDING
7 import flask
8 import yaml
9 import lmfdb.base as base
10 from lmfdb.utils import flash_error
11 from datetime import datetime
12 from flask import render_template, request, url_for
13 from lmfdb.api import api_page, api_logger
14 from bson.objectid import ObjectId
15 
16 # caches the database information
17 _databases = None
18 
19 def pluck(n, list):
20     return [_[n] for _ in list]
21 
22 def quote_string(value):
23     if isinstance(value,unicode) or isinstance(value,str):
24         return repr(value)
25     elif isinstance(value,ObjectId):
26         return "\"ObjectId('%s')\""%value
27     return value
28 
29 def pretty_document(rec,sep=", ",id=True):
30     # sort keys and remove _id for html display
31     attrs = sorted([(key,quote_string(rec[key])) for key in rec.keys() if (id or key != '_id')])
32     return "{"+sep.join(["'%s': %s"%attr for attr in attrs])+"}"
33 
34 
35 def censor(entries):
36     """
37     hide some of the databases and collection from the public
38     """
39     dontstart = ["system.", "test", "upload", "admin", "contrib"]
40     censor = ["local", "userdb"]
41     for entry in entries:
42         if any(entry == x for x in censor) or \
43            any(entry.startswith(x) for x in dontstart):
44             continue
45         yield entry
46 
47 def init_database_info():
48     global _databases
49     if _databases is None:
50         C = base.getDBConnection()
51         _databases = {}
52         for db in censor(C.database_names()):
53             colls = list(censor(C[db].collection_names()))
54             _databases[db] = sorted([(c, C[db][c].count()) for c in colls])
55 
56 @api_page.route("/")
57 def index():
58     init_database_info()
59     databases = _databases
60     title = "API"
61     return render_template("api.html", **locals())
62 
63 @api_page.route("/stats")
64 def stats():
65     def mb(x):
66         return int(round(x/1000000.0))
67     info={}
68     info['minsizes'] = ['0','1','10','100','1000','10000','100000']
69     info['minsize'] = request.args.get('minsize','1').strip()
70     if not info['minsize'] in info['minsizes']:
71         info['minsizes'] = '1'
72     info['groupby'] = 'db' if request.args.get('groupby','').strip().lower() == 'db' else ''
73     info['sortby'] = request.args.get('sortby','size').strip().lower()
74     if not info['sortby'] in ['size', 'objects']:
75         info['sortby'] = 'size'
76     init_database_info()
77     C = base.getDBConnection()
78     dbstats = {db:C[db].command("dbstats") for db in _databases}
79     info['dbs'] = len(dbstats.keys())
80     collections = objects = 0
81     size = dataSize = indexSize = 0
82     stats = {}
83     for db in dbstats:
84         dbsize = dbstats[db]['dataSize']+dbstats[db]['indexSize']
85         size += dbsize
86         dataSize += dbstats[db]['dataSize']
87         indexSize += dbstats[db]['indexSize']
88         dbsize = mb(dbsize)
89         dbobjects = dbstats[db]['objects']
90         for c in pluck(0,_databases[db]):
91             if C[db][c].count():
92                 collections += 1
93                 coll = '<a href = "' + url_for (".api_query", db=db, collection = c) + '">'+c+'</a>'
94                 cstats = C[db].command("collstats",c)
95                 objects += cstats['count']
96                 csize = mb(cstats['size']+cstats['totalIndexSize'])
97                 if csize >= int(info['minsize']):
98                     stats[cstats['ns']] = {'db':db, 'coll':coll, 'dbSize': dbsize, 'size':csize, 'dbObjects':dbobjects,
99                                           'dataSize':mb(cstats['size']), 'indexSize':mb(cstats['totalIndexSize']), 'avgObjSize':int(round(cstats['avgObjSize'])), 'objects':cstats['count'], 'indexes':cstats['nindexes']}
100     info['collections'] = collections
101     info['objects'] = objects
102     info['size'] = mb(size)
103     info['dataSize'] = mb(dataSize)
104     info['indexSize'] = mb(indexSize)
105     if info['sortby'] == 'objects' and info['groupby'] == 'db':
106         sortedkeys = sorted([db for db in stats],key=lambda x: (-stats[x]['dbObjects'],stats[x]['db'],-stats[x]['objects'],stats[x]['coll']))
107     elif info['sortby'] == 'objects':
108         sortedkeys = sorted([db for db in stats],key=lambda x: (-stats[x]['objects'],stats[x]['db'],stats[x]['coll']))
109     elif info['sortby'] == 'size' and info['groupby'] == 'db':
110         sortedkeys = sorted([db for db in stats],key=lambda x: (-stats[x]['dbSize'],stats[x]['db'],-stats[x]['size'],stats[x]['coll']))
111     else:
112         sortedkeys = sorted([db for db in stats],key=lambda x: (-stats[x]['size'],stats[x]['db'],stats[x]['coll']))
113     info['stats'] = [stats[key] for key in sortedkeys]
114     return render_template('api-stats.html', info=info)
115 
116 @api_page.route("/<db>/<collection>/<id>")
117 def api_query_id(db, collection, id):
118     return api_query(db, collection, id = id)
119 
120 
121 @api_page.route("/<db>/<collection>")
122 def api_query(db, collection, id = None):
123     init_database_info()
124 
125     # check what is queried for
126     if db not in _databases or collection not in pluck(0, _databases[db]):
127         return flask.abort(404)
128 
129     # parsing the meta parameters _format and _offset
130     format = request.args.get("_format", "html")
131     offset = int(request.args.get("_offset", 0))
132     DELIM = request.args.get("_delim", ",")
133     fields = request.args.get("_fields", None)
134     sortby = request.args.get("_sort", None)
135 
136     if fields:
137         fields = fields.split(DELIM)
138 
139     if sortby:
140         sortby = sortby.split(DELIM)
141 
142     if offset > 10000:
143         if format != "html":
144             flask.abort(404)
145         else:
146             flash_error("offset %s too large, please refine your query.", offset)
147             return flask.redirect(url_for(".api_query", db=db, collection=collection))
148 
149     # sort = [('fieldname1', ASC/DESC), ...]
150     if sortby is not None:
151         sort = []
152         for key in sortby:
153             if key.startswith("-"):
154                 sort.append((key[1:], DESC))
155             else:
156                 sort.append((key, ASC))
157     else:
158         sort = None
159 
160     # preparing the actual database query q
161     C = base.getDBConnection()
162     q = {}
163 
164     if id is not None:
165         if id.startswith('ObjectId('):
166             q["_id"] = ObjectId(id[10:-2])
167         else:
168             q["_id"] = id
169         single_object = True
170     else:
171         single_object = False
172 
173         for qkey, qval in request.args.iteritems():
174             from ast import literal_eval
175             try:
176                 if qkey.startswith("_"):
177                     continue
178                 if qval.startswith("s"):
179                     qval = qval[1:]
180                 if qval.startswith("i"):
181                     qval = int(qval[1:])
182                 elif qval.startswith("f"):
183                     qval = float(qval[1:])
184                 elif qval.startswith("ls"):      # indicator, that it might be a list of strings
185                     qval = qval[2:].split(DELIM)
186                 elif qval.startswith("li"):
187                     qval = [int(_) for _ in qval[2:].split(DELIM)]
188                 elif qval.startswith("lf"):
189                     qval = [float(_) for _ in qval[2:].split(DELIM)]
190                 elif qval.startswith("py"):     # literal evaluation
191                     qval = literal_eval(qval[2:])
192                 elif qval.startswith("cs"):     # containing string in list
193                     qval = { "$in" : [qval[2:]] }
194                 elif qval.startswith("ci"):
195                     qval = { "$in" : [int(qval[2:])] }
196                 elif qval.startswith("cf"):
197                     qval = { "$in" : [float(qval[2:])] }
198                 elif qval.startswith("cpy"):
199                     qval = { "$in" : [literal_eval(qval[3:])] }
200             except:
201                 # no suitable conversion for the value, keep it as string
202                 pass
203 
204             # update the query
205             q[qkey] = qval
206 
207     # executing the query "q" and replacing the _id in the result list
208     api_logger.info("API query: q = '%s', fields = '%s', sort = '%s', offset = %s" % (q, fields, sort, offset))
209     data = list(C[db][collection].find(q, projection = fields, sort=sort).skip(offset).limit(100))
210     
211     if single_object and not data:
212         if format != 'html':
213             flask.abort(404)
214         else:
215             flash_error("no document with id %s found in collection %s.%s.", id, db, collection)
216             return flask.redirect(url_for(".api_query", db=db, collection=collection))
217     
218     for document in data:
219         oid = document["_id"]
220         if type(oid) == ObjectId:
221             document["_id"] = "ObjectId('%s')" % oid
222         elif isinstance(oid, basestring):
223             document["_id"] = str(oid)
224 
225     # preparing the datastructure
226     start = offset
227     next_req = dict(request.args)
228     next_req["_offset"] = offset
229     url_args = next_req.copy()
230     query = url_for(".api_query", db=db, collection=collection, **next_req)
231     offset += len(data)
232     next_req["_offset"] = offset
233     next = url_for(".api_query", db=db, collection=collection, **next_req)
234 
235     # the collected result
236     data = {
237         "database": db,
238         "collection": collection,
239         "timestamp": datetime.utcnow().isoformat(),
240         "data": data,
241         "start": start,
242         "offset": offset,
243         "query": query,
244         "next": next
245     }
246 
247     # display of the result (default html)
248     if format.lower() == "json":
249         return flask.jsonify(**data)
250     elif format.lower() == "yaml":
251         y = yaml.dump(data,
252                       default_flow_style=False,
253                       canonical=False,
254                       allow_unicode=True)
255         return flask.Response(y, mimetype='text/plain')
256     else:
257         # sort displayed records by key (as json and yaml do)
258         data["pretty"] = pretty_document
259         location = "%s/%s" % (db, collection)
260         title = "API - " + location
261         bc = [("API", url_for(".index")), (location, query)]
262         query_unquote = urllib2.unquote(data["query"])
263         return render_template("collection.html",
264                                title=title,
265                                single_object=single_object,
266                                query_unquote = query_unquote,
267                                url_args = url_args,
268                                bread=bc,
269                                **data)
270 
271
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/lmfdb/api/api.py b/lmfdb/api/api.py
--- a/lmfdb/api/api.py
+++ b/lmfdb/api/api.py
@@ -4,13 +4,15 @@
 import urllib2
 ASC = pymongo.ASCENDING
 DESC = pymongo.DESCENDING
-import flask
 import yaml
+import json
+import flask
 import lmfdb.base as base
 from lmfdb.utils import flash_error
 from datetime import datetime
-from flask import render_template, request, url_for
+from flask import render_template, request, url_for, app, current_app
 from lmfdb.api import api_page, api_logger
+from bson import json_util
 from bson.objectid import ObjectId
 
 # caches the database information
@@ -26,6 +28,14 @@
         return "\"ObjectId('%s')\""%value
     return value
 
+def oids_to_strings(doc):
+    """ recursively replace all ObjectId values in dictionary doc with strings encoding the ObjectId values"""
+    for k,v in doc.items():
+        if isinstance(v,ObjectId):
+            doc[k] = "ObjectId('%s')" % v
+        elif isinstance(v,dict):
+            oids_to_strings(doc[k])
+
 def pretty_document(rec,sep=", ",id=True):
     # sort keys and remove _id for html display
     attrs = sorted([(key,quote_string(rec[key])) for key in rec.keys() if (id or key != '_id')])
@@ -215,12 +225,9 @@
             flash_error("no document with id %s found in collection %s.%s.", id, db, collection)
             return flask.redirect(url_for(".api_query", db=db, collection=collection))
     
+    # fixup object ids for display and json/yaml encoding
     for document in data:
-        oid = document["_id"]
-        if type(oid) == ObjectId:
-            document["_id"] = "ObjectId('%s')" % oid
-        elif isinstance(oid, basestring):
-            document["_id"] = str(oid)
+        oids_to_strings(document)
 
     # preparing the datastructure
     start = offset
@@ -244,9 +251,9 @@
         "next": next
     }
 
-    # display of the result (default html)
     if format.lower() == "json":
-        return flask.jsonify(**data)
+        #return flask.jsonify(**data) # can't handle binary data
+        return current_app.response_class(json.dumps(data, encoding='ISO-8859-1', indent=2, default=json_util.default), mimetype='application/json')
     elif format.lower() == "yaml":
         y = yaml.dump(data,
                       default_flow_style=False,
@@ -254,7 +261,7 @@
                       allow_unicode=True)
         return flask.Response(y, mimetype='text/plain')
     else:
-        # sort displayed records by key (as json and yaml do)
+        # sort displayed records by key (as jsonify and yaml_dump do)
         data["pretty"] = pretty_document
         location = "%s/%s" % (db, collection)
         title = "API - " + location

{"golden_diff": "diff --git a/lmfdb/api/api.py b/lmfdb/api/api.py\n--- a/lmfdb/api/api.py\n+++ b/lmfdb/api/api.py\n@@ -4,13 +4,15 @@\n import urllib2\n ASC = pymongo.ASCENDING\n DESC = pymongo.DESCENDING\n-import flask\n import yaml\n+import json\n+import flask\n import lmfdb.base as base\n from lmfdb.utils import flash_error\n from datetime import datetime\n-from flask import render_template, request, url_for\n+from flask import render_template, request, url_for, app, current_app\n from lmfdb.api import api_page, api_logger\n+from bson import json_util\n from bson.objectid import ObjectId\n \n # caches the database information\n@@ -26,6 +28,14 @@\n         return \"\\\"ObjectId('%s')\\\"\"%value\n     return value\n \n+def oids_to_strings(doc):\n+    \"\"\" recursively replace all ObjectId values in dictionary doc with strings encoding the ObjectId values\"\"\"\n+    for k,v in doc.items():\n+        if isinstance(v,ObjectId):\n+            doc[k] = \"ObjectId('%s')\" % v\n+        elif isinstance(v,dict):\n+            oids_to_strings(doc[k])\n+\n def pretty_document(rec,sep=\", \",id=True):\n     # sort keys and remove _id for html display\n     attrs = sorted([(key,quote_string(rec[key])) for key in rec.keys() if (id or key != '_id')])\n@@ -215,12 +225,9 @@\n             flash_error(\"no document with id %s found in collection %s.%s.\", id, db, collection)\n             return flask.redirect(url_for(\".api_query\", db=db, collection=collection))\n     \n+    # fixup object ids for display and json/yaml encoding\n     for document in data:\n-        oid = document[\"_id\"]\n-        if type(oid) == ObjectId:\n-            document[\"_id\"] = \"ObjectId('%s')\" % oid\n-        elif isinstance(oid, basestring):\n-            document[\"_id\"] = str(oid)\n+        oids_to_strings(document)\n \n     # preparing the datastructure\n     start = offset\n@@ -244,9 +251,9 @@\n         \"next\": next\n     }\n \n-    # display of the result (default html)\n     if format.lower() == \"json\":\n-        return flask.jsonify(**data)\n+        #return flask.jsonify(**data) # can't handle binary data\n+        return current_app.response_class(json.dumps(data, encoding='ISO-8859-1', indent=2, default=json_util.default), mimetype='application/json')\n     elif format.lower() == \"yaml\":\n         y = yaml.dump(data,\n                       default_flow_style=False,\n@@ -254,7 +261,7 @@\n                       allow_unicode=True)\n         return flask.Response(y, mimetype='text/plain')\n     else:\n-        # sort displayed records by key (as json and yaml do)\n+        # sort displayed records by key (as jsonify and yaml_dump do)\n         data[\"pretty\"] = pretty_document\n         location = \"%s/%s\" % (db, collection)\n         title = \"API - \" + location\n", "issue": "many api json downloads fail with internal server errors\nThe following pages all give internal server errors; the problem appears to occur on collections whose documents contain attributes that are object ids (other than _id, which is handled separately), which causes errors like:\n\nTypeError: ObjectId('4f7052dc5009fb18ef000811') is not JSON serializable\n\nThe following pages all have this problem:\n\nhttp://www.lmfdb.org/api/MaassWaveForms/Coefficients?_format=json\nhttp://www.lmfdb.org/api/MaassWaveForms/Coefficients.chunks?_format=json\nhttp://www.lmfdb.org/api/MaassWaveForms/Coefficients.files?_format=json\nhttp://www.lmfdb.org/api/MaassWaveForms/FS?_format=json\nhttp://www.lmfdb.org/api/MaassWaveForms/Table.chunks?_format=json\nhttp://www.lmfdb.org/api/MaassWaveForms/maassform_plots?_format=json\nhttp://www.lmfdb.org/api/SL2Zsubgroups/groups?_format=json\nhttp://www.lmfdb.org/api/modularforms2/webchar.chunks?_format=json\nhttp://www.lmfdb.org/api/modularforms2/webeigenvalues.chunks?_format=json\nhttp://www.lmfdb.org/api/modularforms2/webmodformspace.chunks?_format=json\nhttp://www.lmfdb.org/api/modularforms2/webnewforms.chunks?_format=json\nhttp://www.lmfdb.org/api/siegel_modular_forms/samples?_format=json\n\n", "before_files": [{"content": "# -*- coding: utf-8 -*-\n\nimport pymongo\nimport urllib2\nASC = pymongo.ASCENDING\nDESC = pymongo.DESCENDING\nimport flask\nimport yaml\nimport lmfdb.base as base\nfrom lmfdb.utils import flash_error\nfrom datetime import datetime\nfrom flask import render_template, request, url_for\nfrom lmfdb.api import api_page, api_logger\nfrom bson.objectid import ObjectId\n\n# caches the database information\n_databases = None\n\ndef pluck(n, list):\n    return [_[n] for _ in list]\n\ndef quote_string(value):\n    if isinstance(value,unicode) or isinstance(value,str):\n        return repr(value)\n    elif isinstance(value,ObjectId):\n        return \"\\\"ObjectId('%s')\\\"\"%value\n    return value\n\ndef pretty_document(rec,sep=\", \",id=True):\n    # sort keys and remove _id for html display\n    attrs = sorted([(key,quote_string(rec[key])) for key in rec.keys() if (id or key != '_id')])\n    return \"{\"+sep.join([\"'%s': %s\"%attr for attr in attrs])+\"}\"\n\n\ndef censor(entries):\n    \"\"\"\n    hide some of the databases and collection from the public\n    \"\"\"\n    dontstart = [\"system.\", \"test\", \"upload\", \"admin\", \"contrib\"]\n    censor = [\"local\", \"userdb\"]\n    for entry in entries:\n        if any(entry == x for x in censor) or \\\n           any(entry.startswith(x) for x in dontstart):\n            continue\n        yield entry\n\ndef init_database_info():\n    global _databases\n    if _databases is None:\n        C = base.getDBConnection()\n        _databases = {}\n        for db in censor(C.database_names()):\n            colls = list(censor(C[db].collection_names()))\n            _databases[db] = sorted([(c, C[db][c].count()) for c in colls])\n\n@api_page.route(\"/\")\ndef index():\n    init_database_info()\n    databases = _databases\n    title = \"API\"\n    return render_template(\"api.html\", **locals())\n\n@api_page.route(\"/stats\")\ndef stats():\n    def mb(x):\n        return int(round(x/1000000.0))\n    info={}\n    info['minsizes'] = ['0','1','10','100','1000','10000','100000']\n    info['minsize'] = request.args.get('minsize','1').strip()\n    if not info['minsize'] in info['minsizes']:\n        info['minsizes'] = '1'\n    info['groupby'] = 'db' if request.args.get('groupby','').strip().lower() == 'db' else ''\n    info['sortby'] = request.args.get('sortby','size').strip().lower()\n    if not info['sortby'] in ['size', 'objects']:\n        info['sortby'] = 'size'\n    init_database_info()\n    C = base.getDBConnection()\n    dbstats = {db:C[db].command(\"dbstats\") for db in _databases}\n    info['dbs'] = len(dbstats.keys())\n    collections = objects = 0\n    size = dataSize = indexSize = 0\n    stats = {}\n    for db in dbstats:\n        dbsize = dbstats[db]['dataSize']+dbstats[db]['indexSize']\n        size += dbsize\n        dataSize += dbstats[db]['dataSize']\n        indexSize += dbstats[db]['indexSize']\n        dbsize = mb(dbsize)\n        dbobjects = dbstats[db]['objects']\n        for c in pluck(0,_databases[db]):\n            if C[db][c].count():\n                collections += 1\n                coll = '<a href = \"' + url_for (\".api_query\", db=db, collection = c) + '\">'+c+'</a>'\n                cstats = C[db].command(\"collstats\",c)\n                objects += cstats['count']\n                csize = mb(cstats['size']+cstats['totalIndexSize'])\n                if csize >= int(info['minsize']):\n                    stats[cstats['ns']] = {'db':db, 'coll':coll, 'dbSize': dbsize, 'size':csize, 'dbObjects':dbobjects,\n                                          'dataSize':mb(cstats['size']), 'indexSize':mb(cstats['totalIndexSize']), 'avgObjSize':int(round(cstats['avgObjSize'])), 'objects':cstats['count'], 'indexes':cstats['nindexes']}\n    info['collections'] = collections\n    info['objects'] = objects\n    info['size'] = mb(size)\n    info['dataSize'] = mb(dataSize)\n    info['indexSize'] = mb(indexSize)\n    if info['sortby'] == 'objects' and info['groupby'] == 'db':\n        sortedkeys = sorted([db for db in stats],key=lambda x: (-stats[x]['dbObjects'],stats[x]['db'],-stats[x]['objects'],stats[x]['coll']))\n    elif info['sortby'] == 'objects':\n        sortedkeys = sorted([db for db in stats],key=lambda x: (-stats[x]['objects'],stats[x]['db'],stats[x]['coll']))\n    elif info['sortby'] == 'size' and info['groupby'] == 'db':\n        sortedkeys = sorted([db for db in stats],key=lambda x: (-stats[x]['dbSize'],stats[x]['db'],-stats[x]['size'],stats[x]['coll']))\n    else:\n        sortedkeys = sorted([db for db in stats],key=lambda x: (-stats[x]['size'],stats[x]['db'],stats[x]['coll']))\n    info['stats'] = [stats[key] for key in sortedkeys]\n    return render_template('api-stats.html', info=info)\n\n@api_page.route(\"/<db>/<collection>/<id>\")\ndef api_query_id(db, collection, id):\n    return api_query(db, collection, id = id)\n\n\n@api_page.route(\"/<db>/<collection>\")\ndef api_query(db, collection, id = None):\n    init_database_info()\n\n    # check what is queried for\n    if db not in _databases or collection not in pluck(0, _databases[db]):\n        return flask.abort(404)\n\n    # parsing the meta parameters _format and _offset\n    format = request.args.get(\"_format\", \"html\")\n    offset = int(request.args.get(\"_offset\", 0))\n    DELIM = request.args.get(\"_delim\", \",\")\n    fields = request.args.get(\"_fields\", None)\n    sortby = request.args.get(\"_sort\", None)\n\n    if fields:\n        fields = fields.split(DELIM)\n\n    if sortby:\n        sortby = sortby.split(DELIM)\n\n    if offset > 10000:\n        if format != \"html\":\n            flask.abort(404)\n        else:\n            flash_error(\"offset %s too large, please refine your query.\", offset)\n            return flask.redirect(url_for(\".api_query\", db=db, collection=collection))\n\n    # sort = [('fieldname1', ASC/DESC), ...]\n    if sortby is not None:\n        sort = []\n        for key in sortby:\n            if key.startswith(\"-\"):\n                sort.append((key[1:], DESC))\n            else:\n                sort.append((key, ASC))\n    else:\n        sort = None\n\n    # preparing the actual database query q\n    C = base.getDBConnection()\n    q = {}\n\n    if id is not None:\n        if id.startswith('ObjectId('):\n            q[\"_id\"] = ObjectId(id[10:-2])\n        else:\n            q[\"_id\"] = id\n        single_object = True\n    else:\n        single_object = False\n\n        for qkey, qval in request.args.iteritems():\n            from ast import literal_eval\n            try:\n                if qkey.startswith(\"_\"):\n                    continue\n                if qval.startswith(\"s\"):\n                    qval = qval[1:]\n                if qval.startswith(\"i\"):\n                    qval = int(qval[1:])\n                elif qval.startswith(\"f\"):\n                    qval = float(qval[1:])\n                elif qval.startswith(\"ls\"):      # indicator, that it might be a list of strings\n                    qval = qval[2:].split(DELIM)\n                elif qval.startswith(\"li\"):\n                    qval = [int(_) for _ in qval[2:].split(DELIM)]\n                elif qval.startswith(\"lf\"):\n                    qval = [float(_) for _ in qval[2:].split(DELIM)]\n                elif qval.startswith(\"py\"):     # literal evaluation\n                    qval = literal_eval(qval[2:])\n                elif qval.startswith(\"cs\"):     # containing string in list\n                    qval = { \"$in\" : [qval[2:]] }\n                elif qval.startswith(\"ci\"):\n                    qval = { \"$in\" : [int(qval[2:])] }\n                elif qval.startswith(\"cf\"):\n                    qval = { \"$in\" : [float(qval[2:])] }\n                elif qval.startswith(\"cpy\"):\n                    qval = { \"$in\" : [literal_eval(qval[3:])] }\n            except:\n                # no suitable conversion for the value, keep it as string\n                pass\n\n            # update the query\n            q[qkey] = qval\n\n    # executing the query \"q\" and replacing the _id in the result list\n    api_logger.info(\"API query: q = '%s', fields = '%s', sort = '%s', offset = %s\" % (q, fields, sort, offset))\n    data = list(C[db][collection].find(q, projection = fields, sort=sort).skip(offset).limit(100))\n    \n    if single_object and not data:\n        if format != 'html':\n            flask.abort(404)\n        else:\n            flash_error(\"no document with id %s found in collection %s.%s.\", id, db, collection)\n            return flask.redirect(url_for(\".api_query\", db=db, collection=collection))\n    \n    for document in data:\n        oid = document[\"_id\"]\n        if type(oid) == ObjectId:\n            document[\"_id\"] = \"ObjectId('%s')\" % oid\n        elif isinstance(oid, basestring):\n            document[\"_id\"] = str(oid)\n\n    # preparing the datastructure\n    start = offset\n    next_req = dict(request.args)\n    next_req[\"_offset\"] = offset\n    url_args = next_req.copy()\n    query = url_for(\".api_query\", db=db, collection=collection, **next_req)\n    offset += len(data)\n    next_req[\"_offset\"] = offset\n    next = url_for(\".api_query\", db=db, collection=collection, **next_req)\n\n    # the collected result\n    data = {\n        \"database\": db,\n        \"collection\": collection,\n        \"timestamp\": datetime.utcnow().isoformat(),\n        \"data\": data,\n        \"start\": start,\n        \"offset\": offset,\n        \"query\": query,\n        \"next\": next\n    }\n\n    # display of the result (default html)\n    if format.lower() == \"json\":\n        return flask.jsonify(**data)\n    elif format.lower() == \"yaml\":\n        y = yaml.dump(data,\n                      default_flow_style=False,\n                      canonical=False,\n                      allow_unicode=True)\n        return flask.Response(y, mimetype='text/plain')\n    else:\n        # sort displayed records by key (as json and yaml do)\n        data[\"pretty\"] = pretty_document\n        location = \"%s/%s\" % (db, collection)\n        title = \"API - \" + location\n        bc = [(\"API\", url_for(\".index\")), (location, query)]\n        query_unquote = urllib2.unquote(data[\"query\"])\n        return render_template(\"collection.html\",\n                               title=title,\n                               single_object=single_object,\n                               query_unquote = query_unquote,\n                               url_args = url_args,\n                               bread=bc,\n                               **data)\n\n", "path": "lmfdb/api/api.py"}], "after_files": [{"content": "# -*- coding: utf-8 -*-\n\nimport pymongo\nimport urllib2\nASC = pymongo.ASCENDING\nDESC = pymongo.DESCENDING\nimport yaml\nimport json\nimport flask\nimport lmfdb.base as base\nfrom lmfdb.utils import flash_error\nfrom datetime import datetime\nfrom flask import render_template, request, url_for, app, current_app\nfrom lmfdb.api import api_page, api_logger\nfrom bson import json_util\nfrom bson.objectid import ObjectId\n\n# caches the database information\n_databases = None\n\ndef pluck(n, list):\n    return [_[n] for _ in list]\n\ndef quote_string(value):\n    if isinstance(value,unicode) or isinstance(value,str):\n        return repr(value)\n    elif isinstance(value,ObjectId):\n        return \"\\\"ObjectId('%s')\\\"\"%value\n    return value\n\ndef oids_to_strings(doc):\n    \"\"\" recursively replace all ObjectId values in dictionary doc with strings encoding the ObjectId values\"\"\"\n    for k,v in doc.items():\n        if isinstance(v,ObjectId):\n            doc[k] = \"ObjectId('%s')\" % v\n        elif isinstance(v,dict):\n            oids_to_strings(doc[k])\n\ndef pretty_document(rec,sep=\", \",id=True):\n    # sort keys and remove _id for html display\n    attrs = sorted([(key,quote_string(rec[key])) for key in rec.keys() if (id or key != '_id')])\n    return \"{\"+sep.join([\"'%s': %s\"%attr for attr in attrs])+\"}\"\n\n\ndef censor(entries):\n    \"\"\"\n    hide some of the databases and collection from the public\n    \"\"\"\n    dontstart = [\"system.\", \"test\", \"upload\", \"admin\", \"contrib\"]\n    censor = [\"local\", \"userdb\"]\n    for entry in entries:\n        if any(entry == x for x in censor) or \\\n           any(entry.startswith(x) for x in dontstart):\n            continue\n        yield entry\n\ndef init_database_info():\n    global _databases\n    if _databases is None:\n        C = base.getDBConnection()\n        _databases = {}\n        for db in censor(C.database_names()):\n            colls = list(censor(C[db].collection_names()))\n            _databases[db] = sorted([(c, C[db][c].count()) for c in colls])\n\n@api_page.route(\"/\")\ndef index():\n    init_database_info()\n    databases = _databases\n    title = \"API\"\n    return render_template(\"api.html\", **locals())\n\n@api_page.route(\"/stats\")\ndef stats():\n    def mb(x):\n        return int(round(x/1000000.0))\n    info={}\n    info['minsizes'] = ['0','1','10','100','1000','10000','100000']\n    info['minsize'] = request.args.get('minsize','1').strip()\n    if not info['minsize'] in info['minsizes']:\n        info['minsizes'] = '1'\n    info['groupby'] = 'db' if request.args.get('groupby','').strip().lower() == 'db' else ''\n    info['sortby'] = request.args.get('sortby','size').strip().lower()\n    if not info['sortby'] in ['size', 'objects']:\n        info['sortby'] = 'size'\n    init_database_info()\n    C = base.getDBConnection()\n    dbstats = {db:C[db].command(\"dbstats\") for db in _databases}\n    info['dbs'] = len(dbstats.keys())\n    collections = objects = 0\n    size = dataSize = indexSize = 0\n    stats = {}\n    for db in dbstats:\n        dbsize = dbstats[db]['dataSize']+dbstats[db]['indexSize']\n        size += dbsize\n        dataSize += dbstats[db]['dataSize']\n        indexSize += dbstats[db]['indexSize']\n        dbsize = mb(dbsize)\n        dbobjects = dbstats[db]['objects']\n        for c in pluck(0,_databases[db]):\n            if C[db][c].count():\n                collections += 1\n                coll = '<a href = \"' + url_for (\".api_query\", db=db, collection = c) + '\">'+c+'</a>'\n                cstats = C[db].command(\"collstats\",c)\n                objects += cstats['count']\n                csize = mb(cstats['size']+cstats['totalIndexSize'])\n                if csize >= int(info['minsize']):\n                    stats[cstats['ns']] = {'db':db, 'coll':coll, 'dbSize': dbsize, 'size':csize, 'dbObjects':dbobjects,\n                                          'dataSize':mb(cstats['size']), 'indexSize':mb(cstats['totalIndexSize']), 'avgObjSize':int(round(cstats['avgObjSize'])), 'objects':cstats['count'], 'indexes':cstats['nindexes']}\n    info['collections'] = collections\n    info['objects'] = objects\n    info['size'] = mb(size)\n    info['dataSize'] = mb(dataSize)\n    info['indexSize'] = mb(indexSize)\n    if info['sortby'] == 'objects' and info['groupby'] == 'db':\n        sortedkeys = sorted([db for db in stats],key=lambda x: (-stats[x]['dbObjects'],stats[x]['db'],-stats[x]['objects'],stats[x]['coll']))\n    elif info['sortby'] == 'objects':\n        sortedkeys = sorted([db for db in stats],key=lambda x: (-stats[x]['objects'],stats[x]['db'],stats[x]['coll']))\n    elif info['sortby'] == 'size' and info['groupby'] == 'db':\n        sortedkeys = sorted([db for db in stats],key=lambda x: (-stats[x]['dbSize'],stats[x]['db'],-stats[x]['size'],stats[x]['coll']))\n    else:\n        sortedkeys = sorted([db for db in stats],key=lambda x: (-stats[x]['size'],stats[x]['db'],stats[x]['coll']))\n    info['stats'] = [stats[key] for key in sortedkeys]\n    return render_template('api-stats.html', info=info)\n\n@api_page.route(\"/<db>/<collection>/<id>\")\ndef api_query_id(db, collection, id):\n    return api_query(db, collection, id = id)\n\n\n@api_page.route(\"/<db>/<collection>\")\ndef api_query(db, collection, id = None):\n    init_database_info()\n\n    # check what is queried for\n    if db not in _databases or collection not in pluck(0, _databases[db]):\n        return flask.abort(404)\n\n    # parsing the meta parameters _format and _offset\n    format = request.args.get(\"_format\", \"html\")\n    offset = int(request.args.get(\"_offset\", 0))\n    DELIM = request.args.get(\"_delim\", \",\")\n    fields = request.args.get(\"_fields\", None)\n    sortby = request.args.get(\"_sort\", None)\n\n    if fields:\n        fields = fields.split(DELIM)\n\n    if sortby:\n        sortby = sortby.split(DELIM)\n\n    if offset > 10000:\n        if format != \"html\":\n            flask.abort(404)\n        else:\n            flash_error(\"offset %s too large, please refine your query.\", offset)\n            return flask.redirect(url_for(\".api_query\", db=db, collection=collection))\n\n    # sort = [('fieldname1', ASC/DESC), ...]\n    if sortby is not None:\n        sort = []\n        for key in sortby:\n            if key.startswith(\"-\"):\n                sort.append((key[1:], DESC))\n            else:\n                sort.append((key, ASC))\n    else:\n        sort = None\n\n    # preparing the actual database query q\n    C = base.getDBConnection()\n    q = {}\n\n    if id is not None:\n        if id.startswith('ObjectId('):\n            q[\"_id\"] = ObjectId(id[10:-2])\n        else:\n            q[\"_id\"] = id\n        single_object = True\n    else:\n        single_object = False\n\n        for qkey, qval in request.args.iteritems():\n            from ast import literal_eval\n            try:\n                if qkey.startswith(\"_\"):\n                    continue\n                if qval.startswith(\"s\"):\n                    qval = qval[1:]\n                if qval.startswith(\"i\"):\n                    qval = int(qval[1:])\n                elif qval.startswith(\"f\"):\n                    qval = float(qval[1:])\n                elif qval.startswith(\"ls\"):      # indicator, that it might be a list of strings\n                    qval = qval[2:].split(DELIM)\n                elif qval.startswith(\"li\"):\n                    qval = [int(_) for _ in qval[2:].split(DELIM)]\n                elif qval.startswith(\"lf\"):\n                    qval = [float(_) for _ in qval[2:].split(DELIM)]\n                elif qval.startswith(\"py\"):     # literal evaluation\n                    qval = literal_eval(qval[2:])\n                elif qval.startswith(\"cs\"):     # containing string in list\n                    qval = { \"$in\" : [qval[2:]] }\n                elif qval.startswith(\"ci\"):\n                    qval = { \"$in\" : [int(qval[2:])] }\n                elif qval.startswith(\"cf\"):\n                    qval = { \"$in\" : [float(qval[2:])] }\n                elif qval.startswith(\"cpy\"):\n                    qval = { \"$in\" : [literal_eval(qval[3:])] }\n            except:\n                # no suitable conversion for the value, keep it as string\n                pass\n\n            # update the query\n            q[qkey] = qval\n\n    # executing the query \"q\" and replacing the _id in the result list\n    api_logger.info(\"API query: q = '%s', fields = '%s', sort = '%s', offset = %s\" % (q, fields, sort, offset))\n    data = list(C[db][collection].find(q, projection = fields, sort=sort).skip(offset).limit(100))\n    \n    if single_object and not data:\n        if format != 'html':\n            flask.abort(404)\n        else:\n            flash_error(\"no document with id %s found in collection %s.%s.\", id, db, collection)\n            return flask.redirect(url_for(\".api_query\", db=db, collection=collection))\n    \n    # fixup object ids for display and json/yaml encoding\n    for document in data:\n        oids_to_strings(document)\n\n    # preparing the datastructure\n    start = offset\n    next_req = dict(request.args)\n    next_req[\"_offset\"] = offset\n    url_args = next_req.copy()\n    query = url_for(\".api_query\", db=db, collection=collection, **next_req)\n    offset += len(data)\n    next_req[\"_offset\"] = offset\n    next = url_for(\".api_query\", db=db, collection=collection, **next_req)\n\n    # the collected result\n    data = {\n        \"database\": db,\n        \"collection\": collection,\n        \"timestamp\": datetime.utcnow().isoformat(),\n        \"data\": data,\n        \"start\": start,\n        \"offset\": offset,\n        \"query\": query,\n        \"next\": next\n    }\n\n    if format.lower() == \"json\":\n        #return flask.jsonify(**data) # can't handle binary data\n        return current_app.response_class(json.dumps(data, encoding='ISO-8859-1', indent=2, default=json_util.default), mimetype='application/json')\n    elif format.lower() == \"yaml\":\n        y = yaml.dump(data,\n                      default_flow_style=False,\n                      canonical=False,\n                      allow_unicode=True)\n        return flask.Response(y, mimetype='text/plain')\n    else:\n        # sort displayed records by key (as jsonify and yaml_dump do)\n        data[\"pretty\"] = pretty_document\n        location = \"%s/%s\" % (db, collection)\n        title = \"API - \" + location\n        bc = [(\"API\", url_for(\".index\")), (location, query)]\n        query_unquote = urllib2.unquote(data[\"query\"])\n        return render_template(\"collection.html\",\n                               title=title,\n                               single_object=single_object,\n                               query_unquote = query_unquote,\n                               url_args = url_args,\n                               bread=bc,\n                               **data)\n\n", "path": "lmfdb/api/api.py"}]}