# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + colab={"base_uri": "https://localhost:8080/"} id="VnHGTkrunq6T" executionInfo={"status": "ok", "timestamp": 1630500974932, "user_tz": -330, "elapsed": 642, "user": {"displayName": "CE137 - <NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjZC2bbr7T78zgxxjNKGVdv8bW0Fu-H40Pm5weg=s64", "userId": "07444974201945981837"}} outputId="4c3c039e-4525-442e-ecbf-d08b6041adab"
from google.colab import drive
drive.mount('/content/drive')

# + id="qfZ9zKYXn7mt" executionInfo={"status": "ok", "timestamp": 1630500976601, "user_tz": -330, "elapsed": 1671, "user": {"displayName": "CE137 - <NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjZC2bbr7T78zgxxjNKGVdv8bW0Fu-H40Pm5weg=s64", "userId": "07444974201945981837"}}
import nltk
import matplotlib.pyplot as plt
import pandas as pd

# + colab={"base_uri": "https://localhost:8080/"} id="efz-9nBPN-Lw" executionInfo={"status": "ok", "timestamp": 1630500976602, "user_tz": -330, "elapsed": 11, "user": {"displayName": "CE137 - Jeet Trivedi", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjZC2bbr7T78zgxxjNKGVdv8bW0Fu-H40Pm5weg=s64", "userId": "07444974201945981837"}} outputId="73cc84cc-135e-47d3-f1bc-51d87812e7d6"
# text analysis
text = """Discussing climate, sustainability, and preserving the natural world with President @EmmanuelMacron today in Paris. #BezosEarthFund #ClimatePledge"""

import re
import string
from nltk.corpus import stopwords
from nltk.stem import PorterStemmer
from nltk.tokenize import TweetTokenizer

remove_link = re.sub(f'https?:\/\/.*[\r\n]*', '', text)
remove_link = re.sub(r'#', '', remove_link)
print(remove_link)

# + colab={"base_uri": "https://localhost:8080/"} id="-A7WllK8O2BD" executionInfo={"status": "ok", "timestamp": 1630500976602, "user_tz": -330, "elapsed": 9, "user": {"displayName": "CE137 - <NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjZC2bbr7T78zgxxjNKGVdv8bW0Fu-H40Pm5weg=s64", "userId": "07444974201945981837"}} outputId="fc6d760b-9e96-469a-fb57-d09098ab1405"
print('\033[92m' + text)
print('\033[92m' + remove_link)

# + colab={"base_uri": "https://localhost:8080/"} id="naw66Y2BO8hg" executionInfo={"status": "ok", "timestamp": 1630500977370, "user_tz": -330, "elapsed": 774, "user": {"displayName": "CE137 - <NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjZC2bbr7T78zgxxjNKGVdv8bW0Fu-H40Pm5weg=s64", "userId": "07444974201945981837"}} outputId="b2eacdbf-15c2-4493-fd99-b03d5d0bd573"
from nltk.tokenize import sent_tokenize

text = """Hello Mr. steve, how you doing? whats up? The weather is great, and city is awesome. how you doing? The sky is pinkish-blue. 
You shouldn't eat cardboard, how you doing?"""
nltk.download('punkt')
tokenized_text = sent_tokenize(text)
print(tokenized_text)

# + colab={"base_uri": "https://localhost:8080/"} id="8UubfzEhPaZl" executionInfo={"status": "ok", "timestamp": 1630500977371, "user_tz": -330, "elapsed": 5, "user": {"displayName": "CE137 - <NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjZC2bbr7T78zgxxjNKGVdv8bW0Fu-H40Pm5weg=s64", "userId": "07444974201945981837"}} outputId="7a5080a6-7da9-4c04-a804-c614112cfb80"
from nltk.tokenize import word_tokenize
tokenized_word = word_tokenize(text)

print(tokenized_word)

# + colab={"base_uri": "https://localhost:8080/", "height": 330} id="hV1elgAaPn7E" executionInfo={"status": "ok", "timestamp": 1630500978092, "user_tz": -330, "elapsed": 723, "user": {"displayName": "CE137 - <NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjZC2bbr7T78zgxxjNKGVdv8bW0Fu-H40Pm5weg=s64", "userId": "07444974201945981837"}} outputId="d6457241-b80b-471b-bb1b-dd6d8f98ef6c"
# frequency distribution
from nltk.probability import FreqDist
fredist = FreqDist(tokenized_word)
fredist.most_common(4)

#plotting frequency distribution
import matplotlib.pyplot as plt
fredist.plot(30, cumulative = False, color = 'blue')
plt.show()

# + colab={"base_uri": "https://localhost:8080/"} id="FhB3jZWXQdKd" executionInfo={"status": "ok", "timestamp": 1630500978092, "user_tz": -330, "elapsed": 11, "user": {"displayName": "CE137 - <NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjZC2bbr7T78zgxxjNKGVdv8bW0Fu-H40Pm5weg=s64", "userId": "07444974201945981837"}} outputId="d057000f-c6e5-4d9b-cddf-48f62f7f8e75"
from nltk.corpus import stopwords
nltk.download('stopwords')
stop_words = set(stopwords.words('english'))
print(stop_words)

# + colab={"base_uri": "https://localhost:8080/"} id="ooeyACgUQtpK" executionInfo={"status": "ok", "timestamp": 1630500978093, "user_tz": -330, "elapsed": 10, "user": {"displayName": "CE137 - <NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjZC2bbr7T78zgxxjNKGVdv8bW0Fu-H40Pm5weg=s64", "userId": "07444974201945981837"}} outputId="e61d7fa8-a6dd-4399-b60e-c2973a2da7d3"
filtered_sentence = []
for word in tokenized_word:
  if word not in stop_words:
    filtered_sentence.append(word)
print('Tokenized Sentence : \n', tokenized_word)
print('\nFiltered Sentence : \n', filtered_sentence)

# + colab={"base_uri": "https://localhost:8080/"} id="aopr6n44RKrM" executionInfo={"status": "ok", "timestamp": 1630500978094, "user_tz": -330, "elapsed": 9, "user": {"displayName": "CE137 - <NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjZC2bbr7T78zgxxjNKGVdv8bW0Fu-H40Pm5weg=s64", "userId": "07444974201945981837"}} outputId="dbd3f337-7bb2-411e-93cc-307d5719521e"
from nltk.stem import PorterStemmer
from nltk.tokenize import sent_tokenize, word_tokenize

ps = PorterStemmer()
stemmed_sentence = []
for word in filtered_sentence:
  stemmed_sentence.append(ps.stem(word))

print('Filtered Sentence : \n', filtered_sentence)
print('\nStemmed Sentence : \n', stemmed_sentence)

# + colab={"base_uri": "https://localhost:8080/"} id="FDsRBcF8TNGv" executionInfo={"status": "ok", "timestamp": 1630500980503, "user_tz": -330, "elapsed": 2416, "user": {"displayName": "CE137 - <NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjZC2bbr7T78zgxxjNKGVdv8bW0Fu-H40Pm5weg=s64", "userId": "07444974201945981837"}} outputId="9685b30e-bfb3-4533-d5d8-1af5bb939ee5"
# stemming and lemmatization

from nltk.stem.wordnet import WordNetLemmatizer
from nltk.stem.porter import PorterStemmer

nltk.download('wordnet')
lemmatizer = WordNetLemmatizer()

ps = PorterStemmer()
word = 'crying'
print('Lemmatized Word  :  ', lemmatizer.lemmatize(word, 'v'))
print('Stemmed word  :  ', ps.stem(word))

StarcoderdataJupyterScriptsDedupFiltered