# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
# TITLE

# +
# %matplotlib inline
import matplotlib.pyplot as plt

# magics and warnings
# %load_ext autoreload
# %autoreload 2
import warnings; warnings.simplefilter('ignore')

import os, random, codecs, json, time
from tqdm.notebook import tqdm
import pandas as pd
import numpy as np

seed = 99
random.seed(seed)
np.random.seed(seed)

import nltk, sklearn
import matplotlib.pyplot as plt
import seaborn as sns
sns.set(style="white")
sns.set_context("notebook", font_scale=1.2, rc={"lines.linewidth": 2.5})

# +
# datasets

who_latest = "datasets/WHO_18_03_2020.csv"
dimensions_latest = "datasets/Dimensions_18_03_2020.csv"

df_who = pd.read_csv(who_latest)
df_dimensions = pd.read_csv(dimensions_latest)


# +
# clean DOIs

def clean_doi(d):
    if isinstance(d,str):
        d = d.replace("https://doi.org/","")
        d = d.replace("doi:","")
        return d
    return d


# -

df_who["DOI"] = df_who["DOI"].apply(clean_doi)
df_dimensions["DOI"] = df_dimensions["DOI"].apply(clean_doi)

df_who.head()

df_dimensions.head()

# +
# check DOIs
print("WHO")
print(df_who.shape)
print(df_who[pd.notna(df_who["DOI"])].shape)

print("Dimensions")
print(df_dimensions.shape)
print(df_dimensions[pd.notna(df_dimensions["DOI"])].shape)
# -

df_join = df_dimensions.join(df_who.set_index("DOI"), how='inner', on="DOI", lsuffix='dimensions', rsuffix='who')

df_join = df_join[pd.notna(df_join["DOI"])]

df_join.shape

df_join.head()

who_dois = df_who[pd.notnull(df_who["DOI"])]["DOI"].tolist()
dimensions_dois = df_dimensions[pd.notnull(df_dimensions["DOI"])]["DOI"].tolist()
dimensions_pmids = df_dimensions[(pd.notnull(df_dimensions["PMID"])) & ~(pd.notnull(df_dimensions["DOI"]))]["PMID"].tolist()

len(set(dimensions_dois).intersection(set(who_dois)))

all_dois = list(set(dimensions_dois).union(set(who_dois)))

print(len(all_dois))

extra_pmids = list(set(dimensions_pmids))

print(len(extra_pmids))

# +
# load Scite data

# +
scite_folder = "datasets/Scite_COVID_Dimensions"

df_covid = pd.read_csv(os.path.join(scite_folder,"covid.csv"))
df_sources = pd.read_csv(os.path.join(scite_folder,"covid-source-tallies.csv"))
df_targets = pd.read_csv(os.path.join(scite_folder,"covid-target-tallies.csv"))
print(df_sources.shape)
print(df_targets.shape)
# -

df_sources.drop_duplicates(subset="doi", keep = False, inplace = True)
df_targets.drop_duplicates(subset="doi", keep = False, inplace = True)
print(df_sources.shape)
print(df_targets.shape)

df_covid.head()

df_sources.head()

df_targets.head()

df_sources.describe()

df_targets.describe()

df_sources.sort_values("supporting", ascending=False).head(20)

df_targets.sort_values("supporting", ascending=False).head(20)

sns.scatterplot("supporting","total",data=df_targets)

sns.scatterplot("contradicting","total",data=df_targets)

sns.scatterplot("mentioning","total",data=df_targets)

sns.scatterplot("mentioning","total",data=df_sources)

# ### Index of support

# +
# For each source, calculate a vector of supporting - contradicting - mentioning - total cited publications
# -

len(set(df_sources.doi.tolist()).intersection(set(df_covid.source_doi.tolist())))/len(set(df_sources.doi.tolist()))

for source in df_sources.doi.tolist():
    # get cited
    targets = list(set(df_covid[df_covid.source_doi == source].target_doi.tolist()))
    print(len(targets))

df_sources.doi.tolist()



StarcoderdataJupyterScriptsDedupFiltered