# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/Gaskell-1206/AI_Autoimmune_disease/blob/main/Data_Explore_GSM69372.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + id="MZZ3Bu4XW2zM"
import numpy as np
import pandas as pd
import os

# + [markdown] id="jKltdexQkzhk"
# # DataTable

# + [markdown] id="zpBoDbjbXv2J"
# # + ID = 
# # + SPOT_ID = (Microaaray measurements) Alternative spot identifier - use only when no identifier or sequence tracking information is available. This column is useful for designating control and empty features. 
# # + Full_Name = Full name of the printed material
# # + Printed conc. (mg/ml) = Concentration of printed solution in mg/ml
# # + Supplier = Company that delivered the material
# # + Cat.No. = 
# # + Lot.No. = 
# # + UniProt entry = 

# + id="wrNJSSXBnWOY"
folder_path = '/content/drive/MyDrive/Autoimmune_Disease/Datasets/Autoantibodies_and_nucleic_acids/GSE69372_RAW'

# + colab={"base_uri": "https://localhost:8080/", "height": 423} id="ZGlRjSzfXxZP" outputId="ae68d55d-954d-4e85-cf11-30a0d833d457"
df0 = pd.read_csv(os.path.join(folder_path,'DataTable.csv'))
df0

# + [markdown] id="DAHV1Lymh25T"
# # Raw Data (one patient)
# Microaaray slides \
# IgM, IgG, C3 and C4 signals
# Bkg: local background \
# Spot: Three parallel signal intensities
#
# # + Normalization:
# 1. slide to slide normalization was applied separately to the two print-batches of microarray slides
# 2. compensate the possible differences in the two printing batches. Values in a given batch were divided by geometric means of values derived from the given antigen in control serum samples

# + colab={"base_uri": "https://localhost:8080/", "height": 505} id="qn2guAkOW_d1" outputId="37960c30-93f6-41d1-c325-a597756f93fa"
df1 = pd.read_csv(os.path.join(folder_path,"GSE69365-C3/GSM1698601_C3-1.txt"), sep="\t")
df1.columns = df1.columns.str.replace(' ','')
df1

# + [markdown] id="pxcU2evDMZqM"
# ## Merge DataTable and Raw Data

# + colab={"base_uri": "https://localhost:8080/", "height": 748} id="lTvVZQU3MZOU" outputId="c0ef01f3-f88a-4fb2-fb88-3191ce4fc563"
df_all_1 = df0.merge(df1, on='ID', how='inner')
df_all_1.head(-20)

# + colab={"base_uri": "https://localhost:8080/", "height": 676} id="AsevsDN_Nfhe" outputId="731c36e6-4350-4d49-8390-dbc2a95bfb71"
df_all_1_sub = df_all_1[['SPOT_ID','Full_Name','Bkg.Median','Spot.Median']]
df_all_1_sub.head(20)

# + [markdown] id="fe8Ez3jsi4TP"
# # Processed Data (all patients)
# # + Fluorescent signal intensity
# # + Data Processing: 
# Local background of spot ('Bkg.Median') was substracted from the value of signal ('Spot.Median'). Values were multiplied by 65535 and rounded to be integers. Negative signals values were clamped to arbitrary value 1.

# + colab={"base_uri": "https://localhost:8080/", "height": 487} id="F2FYnydkXaxn" outputId="57835a1e-e48a-4172-bc51-c9d427e0e255"
df2 = pd.read_csv(os.path.join(folder_path,"expreData.csv"))
df2

# + [markdown] id="IpPUoOUdOZYO"
# ## Merge Processed Data

# + colab={"base_uri": "https://localhost:8080/", "height": 713} id="QxH3BbWEOX-b" outputId="cb080afd-8d9c-420d-b519-c2a0d170fcf6"
df_all_2 = pd.merge(df_all_1_sub, df2, left_index=True, right_index=True)
df_all_2

# + [markdown] id="-5AjY48Gk2Zr"
# # Demography

# + colab={"base_uri": "https://localhost:8080/", "height": 1000} id="eZrKyghgjBCb" outputId="e22ad9ff-96b2-4529-f0fa-85755f880b38"
df3 = pd.read_csv("/content/drive/MyDrive/Autoimmune Disease/Datasets/Autoantibodies and nucleic acids skew complement consumption in systemic lupus erythematosus [C4]/GSE69372_RAW/phenoData.csv")
df3

# + [markdown] id="lvXKEdCiQGuk"
# ## Merge Patients with Demography

# + id="MaPKlVG6lCsx" colab={"base_uri": "https://localhost:8080/", "height": 523} outputId="edbdd08c-26f3-4e21-e908-199556ae10c0"
df4 = df2.T
df4.index = df4.index.str.replace('exprs.','')
df4

# + colab={"base_uri": "https://localhost:8080/", "height": 1000} id="fZY29IjOQx_s" outputId="a74b91e1-06ad-4c05-8252-c874814ee39c"
df5 = df3.set_index('geo_accession')
df5

# + colab={"base_uri": "https://localhost:8080/"} id="n08UIdXSSVZP" outputId="cdf804a8-6669-4199-e14a-544ad11bdfdc"
df5['gender:ch1'].value_counts()

# + colab={"base_uri": "https://localhost:8080/"} id="e9VN1vGFTRCy" outputId="068a5fcd-4fd1-451e-8789-97671196d158"
df5['age in years:ch1'].value_counts()

# + [markdown] id="_hPNBQAQVOQW"
# # + NHS: normal human serum
# # + SLE: systemic lupus erythematosus
# # + SSc: systemic sclerosis
# # + SjS: sjörgen's syndrome
# # + UCTD: undifferentiated connective tissue disease
# # + PsA: psoriatic arthritis

# + colab={"base_uri": "https://localhost:8080/"} id="CcgpdIlPTWp5" outputId="c5949614-1835-41e4-81ce-57dc4f545b4a"
df5["disease group (nhs-normal human serum; sle-systemic lupus erythematosus ;uctd-undifferentiated connective tissue disease; sjs-sjörgen's syndrome;ssc-systemic sclerosis; psa-psoriatic arthritis):ch1"].value_counts()

# + colab={"base_uri": "https://localhost:8080/"} id="mcPqREfNSMQ_" outputId="bcbd7b37-923d-431a-d55f-1dfb9f39ebbe"
df5.columns

# + colab={"base_uri": "https://localhost:8080/", "height": 455} id="q88vYYSoSNeU" outputId="2e8f38e8-1080-403a-ad30-ad32b660c7f0"
df5_sub = df5[["age in years:ch1",
               "gender:ch1",
               "disease group (nhs-normal human serum; sle-systemic lupus erythematosus ;uctd-undifferentiated connective tissue disease; sjs-sjörgen's syndrome;ssc-systemic sclerosis; psa-psoriatic arthritis):ch1",]]
df5_sub = df5_sub.rename(columns = {"disease group (nhs-normal human serum; sle-systemic lupus erythematosus ;uctd-undifferentiated connective tissue disease; sjs-sjörgen's syndrome;ssc-systemic sclerosis; psa-psoriatic arthritis):ch1":'Label'})        
df5_sub = df5_sub.rename(columns = {"age in years:ch1":'Age'})        
df5_sub = df5_sub.rename(columns = {"gender:ch1":'Gender'})

df5_sub

# + colab={"base_uri": "https://localhost:8080/", "height": 467} id="jFCqUi7oRQ1f" outputId="3fee20c6-fd3c-4bfd-ef30-99be96030f95"
df_all_3 = pd.merge(df4,df5_sub, left_index=True, right_index=True)
df_all_3

# + colab={"base_uri": "https://localhost:8080/", "height": 487} id="LlsX6VYHSDL3" outputId="0a5b5a86-cc55-4178-b2c3-4cac89c67a1c"
df_all_3_IgM = df_all_3.iloc[0:400,:]
df_all_3_IgM

# + colab={"base_uri": "https://localhost:8080/"} id="nTTZp1b3XM65" outputId="eab83dde-bf87-4591-fd3d-6b72eb2b5f6c"
df_all_3.iloc[0,:]

# + colab={"base_uri": "https://localhost:8080/"} id="NZLgEH5yXWIN" outputId="99fe55f5-88de-46db-b2ba-6f5fde0d663a"
df_all_3.iloc[425,:]

# + id="qVu-lE2fYjpS"
# df_all_3[(df_all_3['Age']==35) & (df_all_3['Gender']=='Female') & (df_all_3['Label']=='SLE')]

# + [markdown] id="9zWRKvmVafJ_"
# # Groupby patients

# + [markdown] id="CishH2cI7iot"
#

# + colab={"base_uri": "https://localhost:8080/", "height": 1000} id="lUnnzIDNZo2j" outputId="ea652d78-52e6-4bbc-dc08-9c94c7c37c2f"
df3

# + colab={"base_uri": "https://localhost:8080/", "height": 467} id="I77PatGzaeDi" outputId="62715b49-81b8-44cc-f96d-2d6807a3d8d6"
df4

# + colab={"base_uri": "https://localhost:8080/", "height": 423} id="ueY1OqHVbugh" outputId="964423d0-da56-49d4-b3be-2bf88558e69b"
df3_sub = df3[["title",
               "age in years:ch1",
               "gender:ch1",
               "disease group (nhs-normal human serum; sle-systemic lupus erythematosus ;uctd-undifferentiated connective tissue disease; sjs-sjörgen's syndrome;ssc-systemic sclerosis; psa-psoriatic arthritis):ch1",]]
df3_sub = df3_sub.rename(columns = {"disease group (nhs-normal human serum; sle-systemic lupus erythematosus ;uctd-undifferentiated connective tissue disease; sjs-sjörgen's syndrome;ssc-systemic sclerosis; psa-psoriatic arthritis):ch1":'Label'})        
df3_sub = df3_sub.rename(columns = {"age in years:ch1":'Age'})        
df3_sub = df3_sub.rename(columns = {"gender:ch1":'Gender'})
df3_sub

# + colab={"base_uri": "https://localhost:8080/", "height": 423} id="t3AtmKFmcUL5" outputId="12f15107-6f2d-40a2-fb08-56e687a0d5a9"
df3_C3 = df3_sub.iloc[0:425,:]
df3_C4 = df3_sub.iloc[425:850,:]
df3_IgG = df3_sub.iloc[850:1275,:]
df3_IgM = df3_sub.iloc[1275:1700,:]
df3_C3

# + colab={"base_uri": "https://localhost:8080/", "height": 487} id="DTmACuSMeHKw" outputId="0bffd0c4-4b15-47ed-cd79-3051689711f6"
df4_C3 = df4.iloc[0:425,:].reset_index().drop(columns="index").add_prefix('C3_')
df4_C4 = df4.iloc[425:850,:].reset_index().drop(columns="index").add_prefix('C4_')
df4_IgG = df4.iloc[850:1275,:].reset_index().drop(columns="index").add_prefix('IgG_')
df4_IgM = df4.iloc[1275:1700,:].reset_index().drop(columns="index").add_prefix('IgM_')
df4_C3

# + colab={"base_uri": "https://localhost:8080/", "height": 487} id="bI5Ylo6ogM2F" outputId="1e0eeafc-b5eb-4667-9bfd-0a121007caa0"
df_final_all = pd.merge(df4_C3, df4_C4, left_index=True, right_index=True)
df_final_all = pd.merge(df_final_all, df4_IgG, left_index=True, right_index=True)
df_final_all = pd.merge(df_final_all, df4_IgM, left_index=True, right_index=True)
df_final_all = pd.merge(df_final_all, df3_C3, left_index=True, right_index=True)
df_final_all = df_final_all.drop(columns="title")
df_final_all

# + colab={"base_uri": "https://localhost:8080/", "height": 142} id="KmocDcWJjkpi" outputId="374ab619-7a04-49de-d6f8-311fc57c4b72"
df_final_all[df_final_all.isnull().T.any()]

# + id="CSilMS6TWPCP"
df_final_all.to_csv('/content/drive/MyDrive/Autoimmune Disease/Datasets/Autoantibodies and nucleic acids skew complement consumption in systemic lupus erythematosus [C4]/GSE69372_RAW/AllMergedData.csv',index=False)

# + [markdown] id="WG17oLV4oYa0"
# # New Processed Data

# + id="XPVG9UYkobYZ" colab={"base_uri": "https://localhost:8080/", "height": 438} outputId="8cba72c9-fae7-45fa-b0b1-29e20e42112a"
df_all_1.head()

# + colab={"base_uri": "https://localhost:8080/", "height": 487} id="Wp4XOHGwR1wR" outputId="f4b9e149-fcf5-4460-984c-ad<PASSWORD>"
df_all_2_new = df_all_2.drop(columns=['Bkg.Median','Spot.Median','Full_Name','SPOT_ID'])
# df_all_2_new = df_all_2_new.transform(data_transform)
df_all_2_new

# + id="UII--OMNUGnk"


StarcoderdataJupyterScriptsDedupFiltered