<jupyter_start><jupyter_text>sklearn非常慢<jupyter_code>kmeans = KMeans(n_clusters=135,random_state=0).fit(s)
center = kmeans.cluster_centers_
np.save('./data/bert_center.npy',center)<jupyter_output><empty_output><jupyter_text>faiss 又快得害怕<jupyter_code>n_center = 135
niter = 100
d=1024
fkmeans = faiss.Kmeans(d,n_center,niter=niter,verbose = True)
%%time
fkmeans.train(s)
kaggle = pd.read_json('./kaggle_data/arxiv-metadata-oai-snapshot.json',lines=True)
#kaggle.head()
label = kaggle['categories'].apply(lambda x : set(x.split()))
label = label.to_list()
#找据数据最近的一个中心，也就是属于那个聚类
D, I = fkmeans.search(s,1)
index = faiss.IndexFlatL2 (1024)
index.add(s)
D, I = index.search (fkmeans.centroids, 100)
#只找一个的话，应该是它本身了吧
D, I = index.search (fkmeans.centroids, 1)
#np.save('./kaggle_data/tmp.npy',I)
I = np.load('./kaggle_data/tmp.npy')
I = list(I)
I[0]
label[int(I[i])]
dic = defaultdict(list)
for i in range(len(I)):
    dic[','.join(list(label[int(I[i])]))].append(int(I[i]))
len(dic)
kaggle.head()
'https://arxiv.org/abs/'+kaggle.loc[1676474,'id']
'cs.AI' in dic
dic<jupyter_output><empty_output><jupyter_text>实验结果下一步作什么实验，需要把数据集分成每条一个样本，再跑一次吗?<jupyter_code>ans = [0] * 135
for i in range(len(fcenters)):
    center = I[i][0]
    for j in range(1,100):
        index = I[i][j]
        if len(label[index] & label[center]) > 0:
            ans[i] += 1
dic = {}
for i in range(len(fcenters)):
    center = I[i][0]
    for j in range(1,100):
        index = I[i][j]
        if len(label[index] & label[center]) > 0:
            dic[','.join(list(label[center]))] =dic.get(','.join(list(label[center])),0)+ 1
#小于135因为有些中心的标签是一样的，也许说明了分类不好
len(dic)
dic
label[1520554]
data = kaggle.join(kaggle['categories'].str.split(' ', expand=True).stack().reset_index(level=1, drop=True).rename('label'))
from collections import Counter
num = Counter(data['label'])
num['cond-mat.dis-nn']
num<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup