<jupyter_start><jupyter_text>Use USPTO-2M dataset<jupyter_code>df = pd.read_csv('../USPTO-2M/uspto_2m.tsv', sep='\t')
df.head()
#only keep digital patent numbers
#df['No'] = df['No'].map(lambda x: x[3:])
df = df.dropna().reset_index(drop=True)
len(df)
df.columns
abst_list = list(df['Abstract'])
title_list = list(df['Title'])
all_contents = abst_list + title_list
len(all_contents)
all_contents[0:2]<jupyter_output><empty_output><jupyter_text>Preprocessing<jupyter_code>import re
import time
from nltk.tokenize import word_tokenize
from nltk.tokenize import sent_tokenize
#remove non-alphabetic characters
temp_all_contents = [re.sub('[^a-zA-Z]', ' ', str(item)).lower() for item in all_contents]
len(temp_all_contents)
#use Spacy for tokenization
start_time = time.time()

test_sentence = temp_all_contents[0:10000]
all_words_list = []
for sent in nlp.pipe(test_sentence, batch_size=50, n_threads=4):
    tmp_word_list = [token.text for token in sent]
    all_words_list.append(tmp_word_list)

print("--- %s seconds ---" % (time.time() - start_time))
#use NLTK for tokenization - faster
start_time = time.time()
test_sentence = temp_all_contents[0:100000]
all_words = [word_tokenize(sent) for sent in test_sentence]
print("--- %s seconds ---" % (time.time() - start_time))
start_time = time.time()
all_words_list = [word_tokenize(sent) for sent in temp_all_contents]
print("--- %s seconds ---" % (time.time() - start_time))<jupyter_output>--- 1561.3763830661774 seconds ---<jupyter_text>Training the model - gensim word2vec<jupyter_code>train_set = all_words_list[0:1998373]
from gensim.models import Word2Vec
import logging
import multiprocessing

cores = multiprocessing.cpu_count()
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

start_time = time.time()
model = Word2Vec(train_set, 
                 size = 100,
                 sg = 1,
                 window = 5,
                 min_alpha = 0.001,
                 alpha  = 0.05,
                 workers=cores-1,
                 iter = 5
                )
print("--- %s seconds ---" % (time.time() - start_time))
#save the model
model.wv.save_word2vec_format('uspto_2m_abstract_word2vec.bin', binary=True)
model.wv.similarity('computer','program')
#analogy
model.wv.most_similar(positive=['bottle', 'chip'], negative=['computer'], topn=1)<jupyter_output><empty_output><jupyter_text>Visualization  1. PCA<jupyter_code>from sklearn.decomposition import PCA
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline
len(model.wv.vocab)
selected_words = ['physics', 'chemistry', 'vehicle', 'plastics', 'hygiene', 'photography', 'electric']
words = selected_words.copy()
similar_words = {}
for key in selected_words:
    similar_words[key] = [item[0] for item in model.wv.most_similar(key, topn=5)]
similar_words['physics']
for key,value in similar_words.items():
    words = words + value
    
#get vectors for all the words
sample_wv = model.wv[words]
fig = plt.figure(1, figsize=(20, 15))
pca = PCA(n_components=2)
result = pca.fit_transform(sample_wv)
plt.scatter(result[:, 0], result[:, 1])
for i, word in enumerate(words):
    plt.annotate(word, xy=(result[i, 0], result[i, 1]))<jupyter_output><empty_output><jupyter_text>2. t-SNE<jupyter_code>from sklearn.manifold import TSNE
import numpy as np
tsne = TSNE(n_components=2, random_state=0, n_iter=10000, perplexity=15)
np.set_printoptions(suppress=True)
result = tsne.fit_transform(sample_wv)
labels = words

plt.figure(figsize=(15, 15))
plt.scatter(result[:, 0], result[:, 1], c='orange', edgecolors='r')
for label, x, y in zip(labels, result[:, 0], result[:, 1]):
    plt.annotate(label, xy=(x+1, y+1), xytext=(0, 0), textcoords='offset points')<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup