<jupyter_start><jupyter_text>Give me stuff by this identifier<jupyter_code>def get_by_id(id_: str):
    rv = requests.get(f"{BASE_URL}/nmdcschema/ids/{id_}")
    return rv.json()
doc = get_by_id("gold:Gs0135149")
print(doc["type"])
print(doc["name"])
print(
    [(ca["applies_to_person"]["name"],
      ca["applies_to_person"]["orcid"])
     for ca in doc["has_credit_associations"]
    ])
doc = get_by_id("igsn:IEWFS000Q")
print(doc["type"])
print(doc["name"])
print(doc["lat_lon"])
print(doc["part_of"])<jupyter_output>nmdc:Biosample
Bulk soil microbial communities from the East River watershed near Crested Butte, Colorado, United States - ER_153
{'has_raw_value': '38.920576363 -106.948748019', 'latitude': 38.920576363, 'longitude': -106.948748019}
['gold:Gs0135149']<jupyter_text>Give me all the MetaG/MAGS/metaT from Study ABC<jupyter_code>import json

def get(
    collection: str,
    filter_: dict,
    max_page_size=20,
    page_token=None,
    limit=0
):
    params = {
        "filter": json.dumps(filter_),
        "max_page_size": max_page_size,
    }    
    getmore = True
    results = []
    while getmore:
        response = requests.get(
            f"{BASE_URL}/nmdcschema/{collection}",
            params=params
        ).json()
        if "resources" not in response:
            return response
        results.extend(response["resources"])
        if limit and len(results) > limit:
            getmore = False
        elif response.get("next_page_token"):
            print(f"Got {len(results)} so far. Getting more...")
            params["page_token"] = response["next_page_token"]
        else:
            getmore = False
    print(f"Got {len(results)}")
    return results
get("lollipops", {})
biosamples = get(
    "biosample_set", {"part_of": "gold:Gs0135149"}, max_page_size=100
)
def ids_of(documents):
    return [d["id"] for d in documents]
processings = get(
    "omics_processing_set",
    {"has_input": {"$in": ids_of(biosamples)}},
    max_page_size=100
)
"""
Request is too large with 831 IDs, so breaking it up here.
"""

docs = []

for i in range(0, 1000, 100):
    ids = ids_of(processings)[i:i+100]
    docs.extend(get(
    "metatranscriptome_activity_set",
    {"was_informed_by": {"$in": ids}},
    max_page_size=100))
len(docs)
def get_turbo(
    collection: str,
    filter_: dict,
    max_page_size=1000,
    limit=0,
    authorization=None
):
    json_in = {
        "find": collection,
        "filter": filter_,
        "limit": limit or max_page_size,
        "sort": {"id": 1}
    }    
    getmore = True
    results = []
    headers = {"Authorization": authorization} if authorization else None
    while getmore:
        response = requests.post(
            f"{BASE_URL}/queries:run",
            json=json_in,
            headers=headers
        ).json()
        if not response.get("ok"):
            return response
        batch = response["cursor"]["firstBatch"]
        for d in batch:
            d.pop("_id")
        results.extend(batch)
        if limit and len(results) > limit:
            getmore = False
        elif len(batch) == max_page_size:
            print(f"Got {len(results)} so far. Getting more...")
            json_in["find"]["_id"] = {"$gt": str(batch[-1]["_id"])}
        else:
            getmore = False
    print(f"Got {len(results)}")
    return results
def get_authorization(username, password):
    rv = requests.post(
        f"{BASE_URL}/token",
        data={
            "grant_type": "password",
            "username": username,
            "password": password,
        })
    if not str(rv.status_code).startswith("2"):
        raise Exception(rv.text)
    response = rv.json()
    print("Expires", response["expires"])
    return f'Bearer {response["access_token"]}'
authorization = get_authorization("<USERNAME>", "<PASSWORD>")
docs = get_turbo(
    "metatranscriptome_activity_set",
    {"was_informed_by": {"$in": ids_of(processings)}},
    authorization=authorization
)
docs = get_turbo(
    "metagenome_assembly_set",
    {"was_informed_by": {"$in": ids_of(processings)}},
    authorization=authorization
)
docs = get_turbo(
    "metagenome_annotation_activity_set",
    {"was_informed_by": {"$in": ids_of(processings)}},
    authorization=authorization
)
docs = get_turbo(
    "nom_analysis_activity_set",
    {"was_informed_by": {"$in": ids_of(processings)}},
    authorization=authorization
)<jupyter_output>Got 101
StarcoderdataJupyterStructuredCleanDedup