# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import pandas as pd
import numpy as np
from  keras.models import Sequential
from keras.layers import Dense, Activation
import keras
from keras.models import load_model
import pickle
import os
import time
from sklearn.utils import shuffle

dirs = os.listdir('./aclImdb/train')

text = []
labels = []
for i in dirs:
    if i == 'neg':
        for j in os.listdir('./aclImdb/train/neg'):
            with open('./aclImdb/train/neg/'+j, 'rb') as f:
                text.append(f.read())
                labels.append([1,0])
    if i == 'pos':
        for j in os.listdir('./aclImdb/train/pos'):
            with open('./aclImdb/train/pos/'+j, 'rb') as f:
                text.append(f.read())
                labels.append([0,1])

my_set = set()
for x in text:
    for i in str(x).split(' '):
        my_set.add(i)

len(my_set)

#key-value pairs for dictionary
word_dict = {}
for index, word in enumerate(my_set):
    word_dict[word] = index

with open('word_dict.pkl', 'wb') as f:
    pickle.dump(word_dict, f)


# +
# train_vec = []
# for sentence in text[:1000]:
#     word_vec = np.zeros(len(my_set))
#     for word in str(sentence).split(' '):
#         if not word == '':
#             word_vec[word_dict[word]] = 1
#     train_vec.append(word_vec)
# train_vec = np.array(train_vec)
# -

def encode_percep(text, word_dict):
    train_vec = []
    for sentence in text:
        word_vec = np.zeros(len(my_set))
        for word in str(sentence).split(' '):
            if not word == '':
                word_vec[word_dict[word]] = 1
        train_vec.append(word_vec)
    return np.array(train_vec)


reviews1 = encode_percep(text[:10000], word_dict)
del text

labels = np.array(labels)

model = Sequential([
    Dense(120, input_shape=(len(my_set),)),
    Activation('relu'),
    Dense(240),
    Activation('relu'),
    Dense(240),
    Activation('relu'),
    Dense(120),
    Activation('relu'),
    Dense(2),
    Activation('softmax')
])

model.compile(optimizer='rmsprop',
              loss='categorical_crossentropy',
              metrics=['accuracy'])

model.fit(reviews1, labels[:10000], epochs=10, batch_size=32)

del reviews1
model.save('percept_1.h5')

model = load_model('percept_1.h5')

reviews2 = encode_percep(text[10000:20000], word_dict)

model.fit(reviews2, labels[10000:20000], epochs=10, batch_size=32)

del reviews2
model.save('percept_1.h5')
model = load_model('percept_1.h5')

reviews3 = encode_percep(text[20000:], word_dict)

model.fit(reviews2, labels[20000:], epochs=5, batch_size=32)





StarcoderdataJupyterScriptsDedupFiltered