# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/Shantanu9326/Data-Science-Portfolio/blob/master/Latent_Dirichlet_Allocation_on_Articles.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + [markdown] id="I6AJRZnqoi9i" colab_type="text"
# # Latent Dirichlet Allocation

# + [markdown] id="M6FGtZf6oi9i" colab_type="text"
# ## Data
#
# We will be using articles from NPR (National Public Radio), obtained from their website [www.npr.org](http://www.npr.org)

# + id="bCBAW567oi9j" colab_type="code" colab={}
import pandas as pd

# + id="gA3YsT-ao8df" colab_type="code" outputId="6e0b10a6-36aa-45bc-cdaa-18068d0401c6" colab={"base_uri": "https://localhost:8080/", "height": 34}
#Running or Importing .py Files with Google Colab
from google.colab import drive
drive.mount('/content/drive/')

# + id="UQpIxqV6oi9k" colab_type="code" colab={}
npr = pd.read_csv("/content/drive/My Drive/app/npr.csv")

# + id="AgbIMBtcoi9m" colab_type="code" outputId="caed0316-38f0-436f-fc2d-f7df780575a1" colab={"base_uri": "https://localhost:8080/", "height": 204}
npr.head()

# + [markdown] id="MDyoxoZ-oi9p" colab_type="text"
# Notice how we don't have the topic of the articles! Let's use LDA to attempt to figure out clusters of the articles.

# + [markdown] id="2eGnw3G1oi9p" colab_type="text"
# ## Preprocessing

# + id="6skz31_loi9q" colab_type="code" colab={}
from sklearn.feature_extraction.text import CountVectorizer

# + [markdown] id="un0zCGajoi9r" colab_type="text"
# **`max_df`**` : float in range [0.0, 1.0] or int, default=1.0`<br>
# When building the vocabulary ignore terms that have a document frequency strictly higher than the given threshold (corpus-specific stop words). If float, the parameter represents a proportion of documents, integer absolute counts. This parameter is ignored if vocabulary is not None.
#
# **`min_df`**` : float in range [0.0, 1.0] or int, default=1`<br>
# When building the vocabulary ignore terms that have a document frequency strictly lower than the given threshold. This value is also called cut-off in the literature. If float, the parameter represents a proportion of documents, integer absolute counts. This parameter is ignored if vocabulary is not None.

# + id="SkhHsnteoi9s" colab_type="code" colab={}
cv = CountVectorizer(max_df=0.95, min_df=2, stop_words='english')

# + id="j5yonZ8roi9t" colab_type="code" colab={}
dtm = cv.fit_transform(npr['Article'])

# + id="Dbn3ejluoi9v" colab_type="code" outputId="986356d4-8b16-47b8-83fa-3bb6bb6feeba" colab={"base_uri": "https://localhost:8080/", "height": 51}
dtm

# + [markdown] id="1luFG-zooi9y" colab_type="text"
# ## LDA

# + id="s5zdudFToi9z" colab_type="code" colab={}
from sklearn.decomposition import LatentDirichletAllocation

# + id="VNX7DDpQoi90" colab_type="code" colab={}
LDA = LatentDirichletAllocation(n_components=7,random_state=42)

# + id="Ygjy_01uoi92" colab_type="code" outputId="95b122ae-c649-48bf-8fe5-aca97da36d5c" colab={"base_uri": "https://localhost:8080/", "height": 136}
# This can take a while, we're dealing with a large amount of documents!
LDA.fit(dtm)

# + [markdown] id="yNf4usl2oi95" colab_type="text"
# ## Showing Stored Words

# + id="qAT7nAfBoi96" colab_type="code" outputId="f5e7099a-76c6-4f41-9357-6fa1c21fd6aa" colab={"base_uri": "https://localhost:8080/", "height": 34}
len(cv.get_feature_names())

# + id="-01QBp41oi97" colab_type="code" colab={}
import random

# + id="ksmN1zJPoi99" colab_type="code" outputId="6a83c287-0076-4341-e9d6-92dfc997bfae" colab={"base_uri": "https://localhost:8080/", "height": 187}
for i in range(10):
    random_word_id = random.randint(0,54776)
    print(cv.get_feature_names()[random_word_id])

# + id="K7Ywf1Luoi9_" colab_type="code" outputId="6f6a1370-edee-44dc-b862-843901f9338d" colab={"base_uri": "https://localhost:8080/", "height": 187}
for i in range(10):
    random_word_id = random.randint(0,54776)
    print(cv.get_feature_names()[random_word_id])

# + [markdown] id="B2NZt_KSoi-B" colab_type="text"
# ### Showing Top Words Per Topic

# + id="H6eyvhGCoi-C" colab_type="code" outputId="e7b59918-4054-42b7-cd84-f581d2e57f67" colab={"base_uri": "https://localhost:8080/", "height": 34}
len(LDA.components_)

# + id="zl9ICtaMoi-E" colab_type="code" outputId="36849a8a-173e-434e-f344-a70e3428840e" colab={"base_uri": "https://localhost:8080/", "height": 238}
LDA.components_

# + id="nXDZnGMboi-G" colab_type="code" outputId="930c490c-7c8f-42a9-c51a-6430d8cbfa38" colab={"base_uri": "https://localhost:8080/", "height": 34}
len(LDA.components_[0])

# + id="H1CpNUGWoi-J" colab_type="code" outputId="a2fdd07f-21ae-48d3-d8c1-5d8b56379ee0" colab={"base_uri": "https://localhost:8080/", "height": 51}
single_topic = LDA.components_[0]
single_topic[0:7]

# + id="80VSJiRcoi-L" colab_type="code" outputId="93cc7971-c1f7-4bdc-f576-17b97db701cd" colab={"base_uri": "https://localhost:8080/", "height": 34}
# Returns the indices that would sort this array.
single_topic.argsort()

# + id="Wdft5sEboi-O" colab_type="code" outputId="f4febbf1-560f-4910-cc67-b1c2412f670a" colab={"base_uri": "https://localhost:8080/", "height": 34}
# Word least representative of this topic
single_topic[18302]

# + id="W_bJFb4soi-Q" colab_type="code" outputId="56d609b9-66dc-4891-96e9-823f0e8af12c" colab={"base_uri": "https://localhost:8080/", "height": 34}
# Word most representative of this topic
single_topic[42993]

# + id="cHhR7zGpoi-S" colab_type="code" outputId="b8549707-17ed-472b-cc05-cd8c155eb79d" colab={"base_uri": "https://localhost:8080/", "height": 51}
# Top 10 words for this topic:
single_topic.argsort()[-10:]

# + id="QXHaduMgoi-T" colab_type="code" colab={}
top_word_indices = single_topic.argsort()[-10:]

# + id="7pCdxYMyoi-V" colab_type="code" outputId="9e66d247-87a7-48aa-a09a-aac1bc52c208" colab={"base_uri": "https://localhost:8080/", "height": 187}
for index in top_word_indices:
    print(cv.get_feature_names()[index])

# + [markdown] id="D8DT6Zs9oi-Y" colab_type="text"
# These look like business articles perhaps... Let's confirm by using .transform() on our vectorized articles to attach a label number. But first, let's view all the 10 topics found.

# + id="ooQnxjINoi-Y" colab_type="code" outputId="d6f4b031-d4d9-4638-cc4d-d05b6a4972b2" colab={"base_uri": "https://localhost:8080/", "height": 493}
for index,topic in enumerate(LDA.components_):
    print(f'THE TOP 15 WORDS FOR TOPIC #{index}')
    print([cv.get_feature_names()[i] for i in topic.argsort()[-15:]])
    print('\n')

# + [markdown] id="azxes_46oi-b" colab_type="text"
# ### Attaching Discovered Topic Labels to Original Articles

# + id="5Ly4YPAroi-b" colab_type="code" outputId="71d498b6-87c4-4a40-b8c9-d168fd11839d" colab={"base_uri": "https://localhost:8080/", "height": 51}
dtm

# + id="vzfZMMWUoi-d" colab_type="code" outputId="7244816c-30fd-4d85-9160-60d6375e3fe1" colab={"base_uri": "https://localhost:8080/", "height": 34}
dtm.shape

# + id="iCUeS5Bboi-e" colab_type="code" outputId="acf8b909-3a85-4f48-ec22-8fb9e04172e4" colab={"base_uri": "https://localhost:8080/", "height": 34}
len(npr)

# + id="LmFcxGihoi-g" colab_type="code" colab={}
topic_results = LDA.transform(dtm)

# + id="cldzVYGloi-i" colab_type="code" outputId="21ae5a16-5777-4a87-a8c7-ab36a6d99acc" colab={"base_uri": "https://localhost:8080/", "height": 34}
topic_results.shape

# + id="uRzKffZ-oi-k" colab_type="code" outputId="f9af84b8-188b-4c45-a5f2-dcb8c788bc08" colab={"base_uri": "https://localhost:8080/", "height": 51}
topic_results[0]

# + id="WW7cJLJvoi-m" colab_type="code" outputId="88b9f630-8e87-4c36-de19-1d37861e1056" colab={"base_uri": "https://localhost:8080/", "height": 34}
topic_results[0].round(2)

# + id="YKbSfxMCoi-o" colab_type="code" outputId="95cd2182-794b-401d-d591-ea93f4d67073" colab={"base_uri": "https://localhost:8080/", "height": 34}
topic_results[0].argmax()

# + [markdown] id="WdI-CQ3xoi-q" colab_type="text"
# This means that our model thinks that the first article belongs to topic #1.

# + [markdown] id="AyaJESXSoi-q" colab_type="text"
# ### Combining with Original Data

# + id="bGQSKRw2oi-r" colab_type="code" outputId="824a4f76-08ee-4994-bb13-5da034d99dd3" colab={"base_uri": "https://localhost:8080/", "height": 204}
npr.head()

# + id="81shC1cFoi-s" colab_type="code" outputId="73b611ec-b9c8-41f6-ad7e-05828bc80eee" colab={"base_uri": "https://localhost:8080/", "height": 34}
topic_results.argmax(axis=1)

# + id="PB-v1ydeoi-v" colab_type="code" colab={}
npr['Topic'] = topic_results.argmax(axis=1)

# + id="mhTqevA_oi-x" colab_type="code" outputId="55ef7dea-ff53-4f8f-d252-1fda29cf5013" colab={"base_uri": "https://localhost:8080/", "height": 359}
npr.head(10)

# + [markdown] id="VCpb1qzKoi-z" colab_type="text"
# ## Great work!

StarcoderdataJupyterScriptsDedupFiltered