<jupyter_start><jupyter_text>Pre-processing<jupyter_code>def preprocess(text):
    stop_words = stopwords.words('english')
    lemmatizer=nltk.stem.WordNetLemmatizer()
    
    # tokenazation
    tokens_list = []
    for sent in nltk.sent_tokenize(text, language='english'):
        for word in nltk.word_tokenize(sent, language='english'):
            tokens_list.append(word)
    
    output = []
    for Token in tokens_list:
        # to lowercase
        token=Token.lower()
        
        # punctuation removal
        for punc in string.punctuation:
            token=token.replace(punc,'')
            
        # number digits removal
        for digit in string.digits:
            token=token.replace(digit,'')
            
        # lemmatzation
        token = lemmatizer.lemmatize(token)
        
        # stop words removal  
        if (token != "") and (token not in stop_words):
            output.append(token)
            
    return output
%%time
test_x=test_x.apply(preprocess)
%%time
train_x=train_x.apply(preprocess)
# list to str
train_x=train_x.apply(lambda x: ''.join(i+' ' for i in x))
test_x=test_x.apply(lambda x: ''.join(i+' '  for i in x))<jupyter_output><empty_output><jupyter_text>Save pre-processed data<jupyter_code>train_set_pp = pd.DataFrame(columns=["Class Index", "Description"])
test_set_pp = pd.DataFrame(columns=["Class Index", "Description"])
train_set_pp["Class Index"] = train_y
train_set_pp["Description"] = train_x
test_set_pp["Class Index"] = test_y
test_set_pp["Description"] = test_x
train_set_pp.to_csv("dataset_train_pp.csv")
test_set_pp.to_csv("dataset_test_pp.csv")
train_set_pp.head()<jupyter_output><empty_output>