<jupyter_start><jupyter_text>Read in similarity matrix and format as floats<jupyter_code>def read_similarity_matrix(filepath):
    # Read in data correctly into columns
    similarities = (
        read_with_nulls(filepath)
        .reset_index(drop=True).T
        .reset_index().T
        .reset_index(drop=True)
    )
    # Get column headers correctly and format PERSID according to apps
    similarities.columns = ["PERSID", *similarities.columns[1:]]
    similarities["PERSID"] = similarities["PERSID"].str.replace("nr:", "")
    similarities.columns = ["PERSID", *similarities["PERSID"]]
    # Change all similarity column types to float
    similarity_matrix = similarities.loc[:, similarities.columns != 'PERSID'].astype(float)
    # Reconstruct similarity matrix with correct formatting and datatypes
    final_similarities = pd.concat((similarities["PERSID"], similarity_matrix), axis=1)
    return final_similarities
similarities_strict = read_similarity_matrix(f"../graph_data/{files[2]}")
similarities_strict.head()
(
    similarities_strict.loc[similarities_strict["PERSID"].isin(persid_list)].iloc[:, :6]
).to_csv(f"../graph_data_small/{files[2]}", index=False, header=True)

similarities_strict["PERSID"][0] == apps["PERSID"][0]
print(apps["PERSID"][0])<jupyter_output>E4FFAE5299FA01F7ADC0E4115B9A6514<jupyter_text>Obtain AHD data with App PERSIDs<jupyter_code>ahd_hits = read_with_nulls(f"../graph_data/{files[3]}")
ahd_hits["PERSID"] = ahd_hits["PERSID"].str.replace("nr:", "")
ahd_hits.loc[apps["PERSID"].isin(persid_list)].head()
ahd_hits.loc[apps["PERSID"].isin(persid_list)].to_csv(f"../graph_data_small/{files[3]}", index=False, header=True)<jupyter_output><empty_output><jupyter_text>Obtain OS instances data<jupyter_code>os_instances = read_with_nulls(f"../graph_data/{files[4]}")
# Rename first column as PERSID
os_instances.rename({u"os_\ufeffPersID": "PERSID"}, axis=1, inplace=True)
os_instances.head()
os_instances.iloc[:5, :].to_csv(f"../graph_data_small/{files[4]}", index=False, header=True)<jupyter_output><empty_output><jupyter_text>Process similarity data for connected components<jupyter_code>similarities_connected_components = read_with_nulls(f"../graph_data/{files[5]}")
similarities_connected_components.rename(
    {"PersID-1": "PERSID_1", "PersID-2": "PERSID_2"}, axis=1, inplace=True
)
similarities_connected_components["PERSID_2"] = similarities_connected_components[
    "PERSID_2"
].str.replace("nr:", "")
similarities_connected_components.head()

similarities_connected_components.iloc[:5, :].to_csv(f"../graph_data_small/{files[5]}", index=False, header=True)
similarities_all = read_similarity_matrix(f"../graph_data/{files[6]}")
similarities_all.head()
(
    similarities_all.loc[similarities_all["PERSID"].isin(persid_list)].iloc[:, :6]
).to_csv(f"../graph_data_small/{files[6]}", index=False, header=True)
similarities_max = read_with_nulls(f"../graph_data/{files[7]}")
similarities_max.rename(
    {"PersID-1": "PERSID_1", "PersID-2": "PERSID_2"}, axis=1, inplace=True
)
similarities_max["PERSID_2"] = similarities_max[
    "PERSID_2"
].str.replace("nr:", "")
similarities_max.head()
similarities_max.loc[similarities_max["PERSID_1"].isin(persid_list)].to_csv(f"../graph_data_small/{files[7]}", index=False, header=True)
db_all = read_with_nulls(f"../graph_data/{files[8]}")
len(db_all)
db_all.columns
def clean_strings(df, colnames=[]):
    for col in colnames:
        df[col] = df[col].str.replace("nr:", "")
    return df
db_all_clean = clean_strings(db_all, colnames=["EAR_MAPPED_CMDB_ID", "CMDB_CI_PERSID", "CMDB_CI_ID_OF_PARENT_CI"])
(
    db_all_clean.loc[db_all_clean["EAR_MAPPED_CMDB_ID"].isin(persid_list)]
).to_csv(f"../graph_data_small/{files[8]}", index=False, header=True)
# cond1 = os_instances.loc[os_instances["PERSID"].isin(db_all_clean["CMDB_CI_PERSID"])]
# os_instances.loc[cond1.index]
# cond2 = apps.loc[apps["PERSID"].isin(db_all_clean["CMDB_CI_PERSID"])]
# apps.loc[cond2.index]
db_all_clean.head()<jupyter_output><empty_output><jupyter_text>Check for presence of IDs in the applications and organizations CSVs<jupyter_code>any(db_all_clean["CMDB_CI_PERSID"].isin(orgs["PERSID"]))
any(db_all_clean["CMDB_CI_ID_OF_PARENT_CI"].isin(orgs["PERSID"]))
any(db_all_clean["CMDB_CI_PERSID"].isin(apps["PERSID"]))
any(db_all_clean["CMDB_CI_ID_OF_PARENT_CI"].isin(apps["PERSID"]))
any(db_all_clean["CMDB_CI_PERSID"].isin(os_instances["PERSID"]))
any(db_all_clean["CMDB_CI_ID_OF_PARENT_CI"].isin(os_instances["PERSID"]))<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup