# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/R4HUL-ROY/Multimodal_feature_extraction/blob/main/tobaco3482_text.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + id="ejoGZZRKFArL"
import numpy as np
import pandas as pd
import os
import pathlib

# + colab={"base_uri": "https://localhost:8080/"} id="Ulmix5uzJ6wL" outputId="cac19930-b9d9-4364-8320-47126ea42bba"
data_root = pathlib.Path('/content/drive/MyDrive/tobaco_OCR/')

print(data_root)
for item in data_root.iterdir():
  print(item)


# + id="yFa0ssSKFgEK" colab={"base_uri": "https://localhost:8080/"} outputId="1ff8daba-71e3-4dc8-f951-d84de6409cae"
def get_file_paths_and_labels(data_root):
     text_paths = [str(path) for path in data_root.glob('*/*.txt')]
     return text_paths

text_paths = get_file_paths_and_labels(data_root)
print(text_paths)

# + colab={"base_uri": "https://localhost:8080/"} id="jLdJ6h4VK3oA" outputId="a728267e-b295-4fad-ae19-1121fea3b63c"
texts = []
for this_text in text_paths:
    with open(this_text) as f:
        lines = f.readlines()
        lines  = ' '.join(lines)
        texts.append(lines)

print(len(texts))

# + colab={"base_uri": "https://localhost:8080/", "height": 206} id="5h4AM27PRWI2" outputId="572f828e-62e6-434b-944c-418be52029b7"
df = pd.DataFrame(list(zip(text_paths, texts)),
               columns =['Path', 'text'])
df.head()

# + colab={"base_uri": "https://localhost:8080/", "height": 423} id="O90R-3vvCrY6" outputId="7577d34f-b04d-4e17-cefb-23b84d39852c"
import re
df['text'] = [re.sub(r'[^\w\s]','',s) for s in df['text']]
df['text']  = [s.replace('\n','') for s in df['text']]
df

# + colab={"base_uri": "https://localhost:8080/"} id="vR30wSq68Di8" outputId="939996d7-5222-41ae-fdb3-2bc9c7709bf7"
## Tokenize, Lemmatize, stopwords removal
import spacy 
import nltk
nlp = spacy.load("en", disable=['parser', 'tagger', 'ner'])
from nltk.corpus import stopwords
nltk.download('stopwords')
stops = stopwords.words("english")

def normalize(comment, lowercase, remove_stopwords):
    if lowercase:
        comment = comment.lower()
    comment = nlp(comment)
    lemmatized = list()
    for word in comment:
        lemma = word.lemma_.strip()
        if lemma:
            if not remove_stopwords or (remove_stopwords and lemma not in stops):
                lemmatized.append(lemma)
    return " ".join(lemmatized)


df['text'] = df['text'].apply(normalize, lowercase=True, remove_stopwords=True)

# + colab={"base_uri": "https://localhost:8080/", "height": 423} id="QNV0grylE14s" outputId="f6d25272-6afd-4011-af95-0838beb17704"
df

# + colab={"base_uri": "https://localhost:8080/", "height": 497} id="D9euiWtsFH9s" outputId="78f71d5f-05c5-430a-e7ef-a8ab8c77f110"
#Tokenize
import nltk
nltk.download('punkt')
nltk.download('wordnet')
df['text'] = [nltk.word_tokenize(s) for s in df['text']]
df

# + id="i11N0u1EIsNT"
from gensim.models.fasttext import FastText
word_tokens = df['text']

# Defining values for parameters
embedding_size = 300
window_size = 5
min_word = 5
down_sampling = 1e-2
 
# # %%time 
fast_Text_model = FastText(word_tokens,
                      size=embedding_size,
                      window=window_size,
                      min_count=min_word,
                      sample=down_sampling,
                      workers = 4,
                      sg=1,
                      iter=100)

# + id="8RoqxjW6NXgF"
# !mkdir -p saved_model
fast_Text_model.save('saved_model/my_model')

# + colab={"base_uri": "https://localhost:8080/"} id="5P_SqxIN9Q77" outputId="1ab82c36-3579-4956-94d8-7fe2db63ef48"
# !zip -r '/content/saved_model.zip' '/content/saved_model'

# + colab={"base_uri": "https://localhost:8080/", "height": 16} id="1lVoXxje_vy8" outputId="d13ed10a-53b6-4501-d8e3-2e5f70ab5d2f"
from google.colab import files
files.download('/content/saved_model.zip')

# + id="_pxQTmt7JJlJ"
# from gensim.models import Word2Vec
# # Save fastText gensim model
# fast_Text_model.save("model/ft_model_yelp")
# # Load saved gensim fastText model
# fast_Text_model = Word2Vec.load("model/ft_model_yelp")

# + colab={"base_uri": "https://localhost:8080/", "height": 677} id="fwNFNhv15Ng7" outputId="fb16e36c-e91b-403a-943f-e86a119e8f87"
# !pip install fasttext
import fasttext
import fasttext.util
ft = fasttext.load_model('cc.en.300.bin')



# + id="RS3LlDHtKQgP"
	
# Check word embedding for a perticular word
l = fast_Text_model.wv['chicken']
print(len(l))
print(l)


# + colab={"base_uri": "https://localhost:8080/", "height": 206} id="TXZ15kZbMtkA" outputId="afd21c64-b83c-4cc6-adad-e093540a7342"
def vectorize_str(token_list):
    embedd = []
    for word in token_list:
        embedd.append(fast_Text_model.wv['word'])

df['text'] = [fast_Text_model.wv['chicken'] for word df['text']]

# + colab={"base_uri": "https://localhost:8080/"} id="6vesJp81MzCH" outputId="fd3a718b-8422-4328-d947-b51a5e01a3da"
biglist = []
for this_list in df['text']:
    for i in this_list:
        biglist.append(i)

print(len(biglist))
print(len(set(biglist)))

# + id="r_uQP3d8NwG8"


StarcoderdataJupyterScriptsDedupFiltered