# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
import pandas as pd
import numpy as np
from collections import Counter

import laws_ai.db as db

db = db.base()

# +
texts = db.published_laws.aggregate(
    [
        {
            '$lookup':
            {
                'from': "texts",
                'localField': "text",
                'foreignField': "_id",
                'as': "texts"
            }
        },
        {
            '$lookup':
            {
                'from': "laws",
                'localField': "law",
                'foreignField': "_id",
                'as': "laws"
            }
        },
        {
           '$unwind': "$laws"
        },
        {
           '$unwind': "$texts"
        },
          {
            "$project": {
              "_id": 1,
              "date": 1,  
              "texts.text": 1, 
              "laws.authors": 1, 
              "laws.law_code": 1, 
            }
          }
    ]
)
dados = []
for i in texts:
    text =  i["texts"]["text"]
    author = i["laws"]["authors"][0]
    law_code = i["laws"]["law_code"]
    date = pd.to_datetime( i["date"],infer_datetime_format=True)
    dados.append({'date': date, 'law_code': law_code, 'text': text, 'author': author})

dataset = pd.DataFrame(list(dados))
df = dataset
# -

# ## Análise Exploratória

# +
df = dataset
size1 = len(df)
print("Size before clean: ", size1) 

df = df[~df['text'].str.contains("O PRESIDENTE DA REPÚBLICA")]
df = df[~df['text'].str.contains("O VICE–PRESIDENTE DA REPÚBLICA")]

size2 = len(df)
print("Size after clean: ", len(df), ". Redução de ", (100 - int((size2 * 100) / size1)), "% dos dados")

# -

print("Sample (3):")
print(df.sample(n=3, random_state=1))

# ## Prepara o texto

import nltk
from string import punctuation
import re
nltk.download('stopwords')
stopwords = nltk.corpus.stopwords.words('portuguese')
stop_words = set(stopwords + list(punctuation))


def df_clean_text(df):
    text = df['clean_text']
    text = " ".join(text.split())
    text = re.sub('<[^>]+>', '', text)
    text = text.split("(function")[0]
    text = text.lower()
    text = [palavra for palavra in text.split() if palavra not in stop_words]
    text = " ".join(text)
    return text



def clean_text(text):
    text = " ".join(text.split())
    text = re.sub('<[^>]+>', '', text)
    text = text.split("(function")[0]
    text = text.lower()
    text = [palavra for palavra in text.split() if palavra not in stop_words]
    text = " ".join(text)
    return text



def count_text(text):
    text_splited = text.split(' ')
    fdist1 = nltk.FreqDist(text_splited) 
    return fdist1



def categorizar(text):
    text_array = text.split()
    if "icms" in text_array:
        return "icms"
    if "is" in text_array:
        return "is"
    if "cofins" in text_array:
        return "pis cofins"
    if "pasep" in text_array:
        return "pis pasep"
    if "ipi" in text_array:
        return "ipi"
    if "iss" in text_array:
        return "iss"
    if "itr" in text_array:
        return "itr" 
    if "ie" in text_array:
        return "ie" 
    if "iof" in text_array:
        return "iof"
    if "irpj" in text_array:
        return "irpj"
    if "irpf" in text_array:
        return "irpf"
    if "cofins" in text_array:
        return "cofins"
    if "cide" in text_array:
        return "cide"
    if "csll" in text_array:
        return "csll" 
    if "fgts" in text_array:
        return "fgts"


df['clean_text'] = df['text'].apply(clean_text)
df['cat'] = df['clean_text'].apply(categorizar)
df['n_words'] = df['clean_text'].apply(lambda x: len(x.split(' ')))

# +
texts = df['clean_text'].str.split()
vocab = Counter()

for i in texts:
    for w in i:
        vocab[w]+=1

print("Total words:", len(vocab))
# -

df_categorized = df
df_categorized = df_categorized.mask(df["cat"].eq(None)).dropna()
df_categorized

import matplotlib.pyplot as plt
import pandas as pd
fig = plt.figure()
fig.suptitle('Textos contendo ciglas de impostos', fontsize=20)
cat_values = df_categorized['cat'].value_counts()
cat_values.plot(kind='bar', figsize=(15, 5)) 


# Esta é uma simples aplicação, o objetivo aqui foi apena identificar referências aos impostos nos textos.



StarcoderdataJupyterScriptsDedupFiltered