<jupyter_start><jupyter_text>TF-IDF Vectorization<jupyter_code>from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import CountVectorizer
#here applying making question bank of all q1 and q2 for calculating tfidf

questions = df['question1'].to_list() + df['question2'].to_list()
questions[:5]
questions_splitted=[]

for sent in questions:
  questions_splitted.append(sent.split())
#training tfidf
tfidf = TfidfVectorizer(lowercase=False,)
tfidf.fit(questions)

#creating dictionary of tfidf_features and their idf values
word2tfidf = dict(zip(tfidf.get_feature_names(), tfidf.idf_))
len(word2tfidf)
nlp=spacy.load('en_core_web_sm')
doc1=nlp(df['question1'][0])
mean_vec1 = np.zeros([len(doc1), len(doc1[0].vector)])
mean_vec1.shape
# en_vectors_web_lg, which includes over 1 million unique vectors.
nlp = spacy.load('en_core_web_sm')

vecs1 = []
# https://github.com/noamraph/tqdm
# tqdm is used to print the progress bar
for qu1 in tqdm(list(df['question1'])):
    doc1 = nlp(qu1) 
    # 384 is the number of dimensions of vectors 
    mean_vec1 = np.zeros([len(doc1), len(doc1[0].vector)])
    for word1 in doc1:
        # word2vec
        vec1 = word1.vector
        # fetch df score
        try:
            idf = word2tfidf[str(word1)]
        except:
            idf = 0
        # compute final vec
        mean_vec1 += vec1 * idf
    mean_vec1 = mean_vec1.mean(axis=0)
    vecs1.append(mean_vec1)
df['q1_feats_m'] = list(vecs1)

len(vecs1)
from sklearn.preprocessing import MinMaxScaler
df=pd.read_csv(r'/gdrive/MyDrive/Colab Notebooks/Datasets/nlp_features_train.csv',encoding='latin-1')
dfp_subsampled = df[0:5000]
dfp_subsampled.head(2)
X = MinMaxScaler().fit_transform(dfp_subsampled[['cwc_min', 'cwc_max', 'csc_min', 'csc_max' , 'ctc_min' , 'ctc_max' , 'last_word_eq', 'first_word_eq' , 'abs_len_diff' , 'mean_len' , 'token_set_ratio' , 'token_sort_ratio' ,  'fuzz_ratio' , 'fuzz_partial_ratio' , 'longest_substr_ratio']])
y = dfp_subsampled['is_duplicate'].values
from sklearn.manifold import TSNE
tsne2d = TSNE(
    n_components=2,
    init='random', # pca
    random_state=101,
    method='barnes_hut',
    n_iter=1000,
    verbose=2,
    angle=0.5,perplexity=40
).fit_transform(X)
df = pd.DataFrame({'x':tsne2d[:,0], 'y':tsne2d[:,1] ,'label':y})
df.head(5)
import seaborn as sns

sns.lmplot(data=df, x='x', y='y', hue='label', fit_reg=False, size=8,palette="Set1",markers=['s','o'])
plt.title("perplexity : {} and max_iter : {}".format(30, 1000))
plt.show()<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup