<jupyter_start><jupyter_text>data process5897列: 第一列是数据编号,其余数据Hyb值22284行: 第一行是表头,其余为数据(5896, 22283)最终的数据即5896个instance,22283个feature<jupyter_code>with open('./../Gene_Chip_Data/microarray.original.txt','r') as fin:
    lines = fin.readlines()

    lines[0]
print lines[0]<jupyter_output>"ProbeSet ID"	"RMA_Signal (Hyb_1)"	"RMA_Signal (Hyb_2)"	"RMA_Signal (Hyb_3)"	"RMA_Signal (Hyb_4)"	"RMA_Signal (Hyb_5)"	"RMA_Signal (Hyb_6)"	"RMA_Signal (Hyb_7)"	"RMA_Signal (Hyb_8)"	"RMA_Signal (Hyb_9)"	"RMA_Signal (Hyb_10)"	"RMA_Signal (Hyb_11)"	"RMA_Signal (Hyb_12)"	"RMA_Signal (Hyb_13)"	"RMA_Signal (Hyb_14)"	"RMA_Signal (Hyb_15)"	"RMA_Signal (Hyb_16)"	"RMA_Signal (Hyb_17)"	"RMA_Signal (Hyb_18)"	"RMA_Signal (Hyb_19)"	"RMA_Signal (Hyb_20)"	"RMA_Signal (Hyb_21)"	"RMA_Signal (Hyb_22)"	"RMA_Signal (Hyb_23)"	"RMA_Signal (Hyb_24)"	"RMA_Signal (Hyb_25)"	"RMA_Signal (Hyb_26)"	"RMA_Signal (Hyb_27)"	"RMA_Signal (Hyb_28)"	"RMA_Signal (Hyb_29)"	"RMA_Signal (Hyb_30)"	"RMA_Signal (Hyb_31)"	"RMA_Signal (Hyb_32)"	"RMA_Signal (Hyb_33)"	"RMA_Signal (Hyb_34)"	"RMA_Signal (Hyb_35)"	"RMA_Signal (Hyb_36)"	"RMA_Signal (Hyb_37)"	"RMA_Signal (Hyb_38)"	"RMA_Signal (Hyb_39)"	"RMA_Signal (Hyb_40)"	"RMA_Signal (Hyb_41)"	"RMA_Signal (Hyb_42)"	"RMA_Signal (Hyb_43)"	"RMA_Signal (Hyb_44)"	"RMA_Signal (Hyb_45)"	"RMA_[...]<jupyter_text>确认是这5896列是从小到大排列的<jupyter_code>import numpy as np
# 需要一个5896*22283的array
# 但是先用22283*5896的
data=np.zeros([22283,5896],dtype='float64')
print lines[1][0:10]
print lines[2][0:10]
print lines[3][0:10]
print lines[4][0:10]
import re
# print lines[1]
data_1=lines[1].split('	')[1:]
len(data_1)
data_1
data[0]=np.array(data_1)
data
for i in range(1,22284):
    data[i-1]=np.array(lines[i].split('	')[1:])
data.transpose()
data_out=data.T
print data_out
data_out.shape
# output the raw data
data_out.tofile("./../data/all_raw_data.bin")<jupyter_output><empty_output><jupyter_text>label data<jupyter_code>import pandas as pd
df=pd.read_csv('./../data/label.csv')
df
colums=df.columns
colums
for name in colums:
    print "=================================="
    uni=df[name].unique()
    print name,len(uni)
    print df[name].value_counts()
"""
==================================
Characteristics [BioSourceType] 13
frozen_sample       794
blood               636
frozen              585
biopsy              395
fresh_sample        252
frozen tissue       233
frozen, biopsy       79
Name: Characteristics [BioSourceType], dtype: int64
==================================
Characteristics [DiseaseState] 194
breast tumor                                                   535
acute myeloid leukemia                                         293
B-cell lymphoma, dlbcl                                         166
breast cancer                                                  139
acute lymphoblastic leukemia, chemotherapy response            129
Huntington's disease                                           113
bone marrow relapse                                            113
germ cell tumor                                                101
brain tumor                                                    100
precursor T lymphoblastic leukemia                              99
control brain                                                   87
acute lymphoblastic leukemia                                    81
prostate cancer                                                 71
normal                                                          67
colorectal adenocarcinoma                                       65
breast tumor, luminal                                           62
Name: Characteristics [DiseaseState], Length: 194, dtype: int64
======================================================
Characteristics [Sex] 8
                 4221
male             1036
female            500
femalemale         58
unknown_sex        38
mixed_sex          24
males              14
hermaphrodite       5
Name: Characteristics [Sex], dtype: int64
"""
ndf=df[['Hybridization Name','Material Type','Characteristics [Sex]','Characteristics [DiseaseState]','Characteristics [BioSourceType]']]
ndf.columns = ['Hyb', 'MaterialType', 'Sex', 'DiseaseState', 'BioSourceType']
ndf
for i in range(5896):
    print ndf.iloc[i]['Hyb'][4:],i
    if int(ndf.iloc[i]['Hyb'][4:])!=i+1:
        print "1"
        
# 是按照顺序来的
"""
==================================
Characteristics [BioSourceType] 13
frozen_sample       794
blood               636
frozen              585
biopsy              395
fresh_sample        252
frozen tissue       233
frozen, biopsy       79
Name: Characteristics [BioSourceType], dtype: int64

"""
for i in range(5896):
    # 7分类
    label=0
    name = ndf.iloc[i]['BioSourceType'] 
    if name == 'frozen_sample':
        label= 1
    elif name == 'blood':
        label= 2
    elif name == 'frozen':
        label= 3
    elif name == 'biopsy':
        label= 4  
    elif name == 'fresh_sample':
        label= 5
    elif name == 'frozen tissue':
        label= 6
    elif name == 'frozen, biopsy':
        label= 7
    else:
        label=0
    ndf.iloc[i]['BioSourceType']=label
"""
Characteristics [Sex] 8
                 4221
male             1036
female            500
Name: Characteristics [Sex], dtype: int64
"""
for i in range(5896):
    # 2分类
    label=0
    name = ndf.iloc[i]['Sex'] 
    if name == 'male':
        label= 1
    elif name == 'female':
        label= 2
    else:
        label=0
    ndf.iloc[i]['Sex']=label
"""
==================================
Characteristics [DiseaseState] 194
breast tumor                                                   535
acute myeloid leukemia                                         293
B-cell lymphoma, dlbcl                                         166
breast cancer                                                  139
acute lymphoblastic leukemia, chemotherapy response            129
Huntington's disease                                           113
bone marrow relapse                                            113
germ cell tumor                                                101
brain tumor                                                    100
precursor T lymphoblastic leukemia                              99
control brain                                                   87
acute lymphoblastic leukemia                                    81
prostate cancer                                                 71
normal                                                          67
colorectal adenocarcinoma                                       65
breast tumor, luminal                                           62
Name: Characteristics [DiseaseState], Length: 194, dtype: int64

"""
for i in range(5896):
    # 16分类
    label=0
    name = ndf.iloc[i]['DiseaseState'] 
    if name == 'breast tumor':
        label= 1
    elif name == 'acute myeloid leukemia':
        label= 2
    elif name == 'B-cell lymphoma, dlbcl':
        label= 3
    elif name == 'breast cancer':
        label= 4  
    elif name == 'acute lymphoblastic leukemia, chemotherapy response':
        label= 5
    elif name == "Huntington's disease":
        label= 6
    elif name == 'bone marrow relapse':
        label= 7
    elif name == 'germ cell tumor':
        label= 8
    elif name == 'brain tumor':
        label= 9
    elif name == 'precursor T lymphoblastic leukemia':
        label= 10
    elif name == 'control brain':
        label= 11
    elif name == 'acute lymphoblastic leukemia':
        label= 12
    elif name == 'prostate cancer':
        label= 13
    elif name == 'normal':
        label= 14
    elif name == 'colorectal adenocarcinoma':
        label= 15
    elif name == 'breast tumor, luminal':
        label= 16
    else:
        label=0
    ndf.iloc[i]['DiseaseState']=label
"""
Material Type 2
organism_part    4754
cell_line        1142
Name: Material Type, dtype: int64
"""
for i in range(5896):
    # 16分类
    label=0
    name = ndf.iloc[i]['MaterialType'] 
    if name == 'organism_part':
        label= 1
    elif name == 'cell_line':
        label= 2
    else:
        label=0
    ndf.iloc[i]['MaterialType']=label
for i in range(5896):
    Hyb = ndf.iloc[i]['Hyb']
    ndf.iloc[i]['Hyb']=int(Hyb[4:])
ndf<jupyter_output><empty_output><jupyter_text>这个还只是最初步的数据处理,还需要划分训练集<jupyter_code>ndf.to_csv('./../data/all_label.csv',index=False)<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup