# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] colab_type="text" id="VfGzS7lzmZr3"
# # Text Pre Processor - Experimento
#
# Este é um componente que utiliza a biblioteca [nltk](https://www.nltk.org/) e [ftfy](https://pypi.org/project/ftfy/) e [regex](https://docs.python.org/3/library/re.html) para pré processar textos que entrrão em outros componentes.
#
# Este notebook apresenta:
# - como usar o [SDK](https://platiagro.github.io/sdk/) para carregar datasets, salvar modelos e outros artefatos.
# - como declarar parâmetros e usá-los para criar componentes reutilizáveis.

# + [markdown] colab_type="text" id="tD7cOYh6mZr5"
# ## Declaração de parâmetros e hiperparâmetros
#
# Declare parâmetros com o botão <img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAUCAYAAACNiR0NAAABhWlDQ1BJQ0MgcHJvZmlsZQAAKJF9kT1Iw0AcxV9TtaIVBzuIOASpThb8QhylikWwUNoKrTqYXPohNGlIUlwcBdeCgx+LVQcXZ10dXAVB8APEydFJ0UVK/F9SaBHjwXE/3t173L0DhFqJqWbbGKBqlpGMRcVMdkUMvKID3QhiCOMSM/V4aiENz/F1Dx9f7yI8y/vcn6NHyZkM8InEs0w3LOJ14ulNS+e8TxxiRUkhPiceNeiCxI9cl11+41xwWOCZISOdnCMOEYuFFpZbmBUNlXiKOKyoGuULGZcVzluc1VKFNe7JXxjMacsprtMcRAyLiCMBETIq2EAJFiK0aqSYSNJ+1MM/4PgT5JLJtQFGjnmUoUJy/OB/8LtbMz854SYFo0D7i21/DAOBXaBete3vY9uunwD+Z+BKa/rLNWDmk/RqUwsfAb3bwMV1U5P3gMsdoP9JlwzJkfw0hXweeD+jb8oCfbdA16rbW2Mfpw9AmrpaugEODoGRAmWveby7s7W3f880+vsBocZyukMJsmwAAAAGYktHRAD/AP8A/6C9p5MAAAAJcEhZcwAADdcAAA3XAUIom3gAAAAHdElNRQfkBgsMIwnXL7c0AAACDUlEQVQ4y92UP4gTQRTGf29zJxhJZ2NxbMBKziYWlmJ/ile44Nlkd+dIYWFzItiNgoIEtFaTzF5Ac/inE/urtLWxsMqmUOwCEpt1Zmw2xxKi53XitPO9H9978+aDf/3IUQvSNG0450Yi0jXG7C/eB0cFeu9viciGiDyNoqh2KFBrHSilWstgnU7nFLBTgl+ur6/7PwK11kGe5z3n3Hul1MaiuCgKDZwALHA7z/Oe1jpYCtRaB+PxuA8kQM1aW68Kt7e3zwBp6a5b1ibj8bhfhQYVZwMRiQHrvW9nWfaqCrTWPgRWvPdvsiy7IyLXgEJE4slk8nw+T5nDgDbwE9gyxryuwpRSF5xz+0BhrT07HA4/AyRJchUYASvAbhiGaRVWLIMBYq3tAojIszkMoNRulbXtPM8HwV/sXSQi54HvQRDcO0wfhGGYArvAKjAq2wAgiqJj3vsHpbtur9f7Vi2utLx60LLW2hljEuBJOYu9OI6vAzQajRvAaeBLURSPlsBelA+VhWGYaq3dwaZvbm6+m06noYicE5ErrVbrK3AXqHvvd4bD4Ye5No7jSERGwKr3Pms2m0pr7Rb30DWbTQWYcnFvAieBT7PZbFB1V6vVfpQaU4UtDQetdTCZTC557/eA48BlY8zbRZ1SqrW2tvaxCvtt2iRJ0i9/xb4x5uJRwmNlaaaJ3AfqIvKY/+78Av++6uiSZhYMAAAAAElFTkSuQmCC" /> na barra de ferramentas.<br>
# O parâmetro `dataset` identifica os conjuntos de dados. Você pode importar arquivos de dataset com o botão <img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAUCAYAAACNiR0NAAABhWlDQ1BJQ0MgcHJvZmlsZQAAKJF9kT1Iw0AcxV9TtaIVBzuIOASpThb8QhylikWwUNoKrTqYXPohNGlIUlwcBdeCgx+<KEY>dXAVB8APEydFJ0UVK/F9SaBHjwXE/3t173L0DhFqJqWbbGKBqlpGMRcVMdkUMvKID3QhiCOMSM/V4aiENz/F1Dx9f7yI8y/vcn6NHyZkM8InEs0w3LOJ14ulNS+e8TxxiRUkhPiceNeiCxI9cl11+41xwWOCZISOdnCMOEYuFFpZbmBUNlXiKOKyoGuULGZcVzluc1VKFNe7JXxjMacsprtMcRAyLiCMBETIq2EAJFiK0aqSYSNJ+1MM/4PgT5JLJtQFGjnmUoUJy/OB/8LtbMz854SYFo0D7i21/DAOBXaBete3vY9uunwD+Z+BKa/rLNWDmk/RqUwsfAb3bwMV1U5P3gMsdoP9JlwzJkfw0hXweeD+jb8oCfbdA16rbW2Mfpw9AmrpaugEODoGRAmWveby7s7W3f880+vsBocZyukMJsmwAAAAGYktHRAD/AP8A/6C9p5MAAAAJcEhZcwAADdcAAA3XAUIom3gAAAAHdElNRQfkBgsOBy6ASTeXAAAC/0lEQVQ4y5WUT2gcdRTHP29m99B23Uiq6dZisgoWCxVJW0oL9dqLfyhCvGWY2YUBI95MsXgwFISirQcLhS5hfgk5CF3wJIhFI7aHNsL2VFZFik1jS1qkiZKdTTKZ3/MyDWuz0fQLc/m99/vMvDfv+4RMlUrlkKqeAAaBAWAP8DSgwJ/AXRG5rao/WWsvTU5O3qKLBMD3fSMiPluXFZEPoyj67PGAMzw83PeEMABHVT/oGpiamnoAmCcEWhH5tFsgF4bh9oWFhfeKxeJ5a+0JVT0oImWgBPQCKfAQuAvcBq67rltX1b+6ApMkKRcKhe9V9QLwbavV+qRer692Sx4ZGSnEcXw0TdP3gSrQswGYz+d/S5IkVtXTwOlCoZAGQXAfmAdagAvsAErtdnuXiDy6+023l7qNRsMODg5+CawBzwB9wFPA7mx8ns/KL2Tl3xCRz5eWlkabzebahrHxPG+v4zgnc7ncufHx8Z+Hhoa29fT0lNM03Q30ikiqqg+ttX/EcTy3WTvWgdVqtddaOw/kgXvADHBHROZVNRaRvKruUNU+EdkPfGWM+WJTYOaSt1T1LPDS/4zLWWPMaLVaPWytrYvIaBRFl/4F9H2/JCKvGmMu+76/X0QOqGoZKDmOs1NV28AicMsYc97zvFdc1/0hG6kEeNsY83UnsCwivwM3VfU7YEZE7lhr74tIK8tbnJiYWPY8b6/ruleAXR0ftQy8boyZXi85CIIICDYpc2ZgYODY3NzcHmvt1eyvP64lETkeRdE1yZyixWLx5U2c8q4x5mIQBE1g33/0d3FlZeXFR06ZttZesNZejuO4q1NE5CPgWVV9E3ij47wB1IDlJEn+ljAM86urq7+KyAtZTgqsO0VV247jnOnv7/9xbGzMViqVMVX9uANYj6LonfVtU6vVkjRNj6jqGeCXzGrPAQeA10TkuKpOz87ONrayhnIA2Qo7BZwKw3B7kiRloKSqO13Xja21C47jPNgysFO1Wi0GmtmzQap6DWgD24A1Vb3SGf8Hfstmz1CuXEIAAAAASUVORK5CYII=" /> na barra de ferramentas.

# + colab={} colab_type="code" id="jK9Bd6X7mZr6" tags=["parameters"]
# parâmetros
dataset = "/tmp/data/imdb-2.csv" #@param {type:"string"}
target = "sentiment" #@param {type:"string", label:"Atributo alvo", description:"Seu modelo será treinado para prever os valores do alvo."}
language = "english" #@param ["portuguese", "english"] {type:"string", label:"Linguagem", description:"Linguagem da qual os stopwords pertencem. Deve ser a mesma utilizada no dataset."}

# selected features to perform the model
filter_type = "incluir" #@param ["incluir","remover"] {type:"string",label:"Modo de seleção das features", description:"Se deseja informar quais features deseja incluir no modelo, selecione a opção [incluir]. Caso deseje informar as features que não devem ser utilizadas, selecione [remover]."}
#model_features = "review" #@param {type:"string",multiple:true,label:"Features para incluir/remover no modelo",description:"Seu modelo será feito considerando apenas as features selecionadas. Caso nada seja especificado, todas as features serão utilizadas"}
model_features = "review" #@param {type:"string"}

# preprocessamento
case = "Lower" #@param ["Lower","Upper","NotApply"] {type:"string",label:"Aplicação de casing", description:"Caixa baixa, caixa alta ou não aplicação de caixa"}
remove_stop_words = True #@param {type:"boolean",label:"Remoção de Stop Words", description:"Remoção de palavras, conjunções, artigos e outros"}
remove_top_words = True #@param {type:"boolean",label:"Remoção de Top Words", description:"Remoção dea porcentagem palavras mais frequentes no texto"}
top_words_percentage = 0.01 #@param {type:"number",label:"Porcentagem de Top Words", description:"Porcentagem das palavras mais frequentes no texto"}
stemming = False #@param {type:"boolean",label:"Stemming"}
lemmatization = True #@param {type:"boolean",label:"Lemmatization"}
remove_punctuation = True #@param {type:"boolean",label:"Remoção de pontuação"}
remove_line_braks = True #@param {type:"boolean",label:"Remoção de quebras de lina",description:"Remoção de quebras de linha por \n e \r"}
remove_accents = True #@param {type:"boolean",label:"Remoção de acentos"}
remove_html = True #@param {type:"boolean",label:"Remoção de HTML"}
remove_css = True #@param {type:"boolean",label:"Remoção de CSS"}

# + [markdown] colab_type="text" id="nsJMgpBRmZr_"
# ## Acesso ao conjunto de dados
#
# O conjunto de dados utilizado nesta etapa será o mesmo carregado através da plataforma.<br>
# O tipo da variável retornada depende do arquivo de origem:
# - [pandas.DataFrame](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html) para CSV e compressed CSV: .csv .csv.zip .csv.gz .csv.bz2 .csv.xz
# - [Binary IO stream](https://docs.python.org/3/library/io.html#binary-i-o) para outros tipos de arquivo: .jpg .wav .zip .h5 .parquet etc

# + colab={} colab_type="code" id="dzan1wl1mZr_" outputId="b893eabd-dbe5-46a8-bc26-81a96418dc8d"
import pandas as pd

df = pd.read_csv(dataset)
print(df)

# + [markdown] colab_type="text" id="duCQT-2YmZsF"
# ## Acesso aos metadados do conjunto de dados
#
# Utiliza a função `stat_dataset` do [SDK da PlatIAgro](https://platiagro.github.io/sdk/) para carregar metadados. <br>
# Por exemplo, arquivos CSV possuem `metadata['featuretypes']` para cada coluna no conjunto de dados (ex: categorical, numerical, or datetime).

# + colab={} colab_type="code" id="qyNVXA7CmZsG"
import numpy as np
#from platiagro import stat_dataset

#metadata = stat_dataset(name=dataset)
#featuretypes = metadata["featuretypes"]

columns = df.columns.to_numpy()
#featuretypes = np.array(featuretypes)
target_index = np.argwhere(columns == target)
columns = np.delete(columns, target_index)
#featuretypes = np.delete(featuretypes, target_index)

# + [markdown] colab_type="text" id="qtyfRV_0mZsJ"
# ## Remoção de linhas com valores faltantes no atributo alvo
#
# Caso haja linhas em que o atributo alvo contenha valores faltantes, é feita a remoção dos casos faltantes.

# + colab={} colab_type="code" id="Vh8OXGLImZsK"
df.dropna(subset = [target],inplace=True)
df.dropna(subset = [model_features],inplace=True)
y = df[target].to_numpy()

# + [markdown] colab_type="text" id="bRlxsXqwmZsO"
# ## Filtragem das features 
#
# Seleciona apenas as features que foram declaradas no parâmetro model_features. Se nenhuma feature for especificada, todo o conjunto de dados será utilizado para a modelagem.

# + colab={} colab_type="code" id="B2viBFvUmZsP"
if filter_type == 'incluir':
    if len(model_features) >= 1:
        columns_index = (np.where(np.isin(columns,model_features)))[0]
        columns_index.sort()
        columns_to_filter = columns[columns_index]
        #featuretypes = featuretypes[columns_index]
    else:
        columns_to_filter = columns
else:
    if len(model_features) >= 1:
        columns_index = (np.where(np.isin(columns,model_features)))[0]
        columns_index.sort()
        columns_to_filter = np.delete(columns,columns_index)
        #featuretypes = np.delete(featuretypes,columns_index)
    else:
        columns_to_filter = columns

# keep the features selected
df_model = df[columns_to_filter]
X = df_model.to_numpy()

# + [markdown] colab_type="text" id="dUySX0csmZsT"
# ## Codifica labels do atributo alvo
#
# As labels do atributo alvo são convertidos em números inteiros ordinais com valor entre 0 e n_classes-1.

# + colab={} colab_type="code" id="OBbq--yEmZsU"
from sklearn.preprocessing import LabelEncoder

label_encoder = LabelEncoder()
y = label_encoder.fit_transform(y)

# + [markdown] colab_type="text" id="tqtHW62PmZsc"
# ## Definição dos filtros

# + colab={"base_uri": "https://localhost:8080/", "height": 373} colab_type="code" id="Ic4xH_S3mZsg" outputId="37afeb30-6cb0-43d3-c272-e9ed67a8fbb9"
import string
from collections import defaultdict
from functools import reduce
from re import sub

import unidecode
from ftfy import fix_text
from nltk.stem import PorterStemmer, WordNetLemmatizer
from nltk.tokenize import word_tokenize


def tokenize_text(text_list: list = None):
    """Tokenize Text without the hyperparâmeters defined.

    Args:
        text_list (list): a list of texts to be used.

    Returns:
        A list of tokenized text without punctuation.
    """

    tokenize_list = list()
    for text in text_list:
        text = text[0]
        text = fix_text(text)
        text = sub("<.*?>", " ", text) if remove_html else text
        text = sub("{.*?}", " ", text) if remove_css else text
        text = unidecode.unidecode(text) if remove_accents else text
        text = sub("/\r\n|\n|\r|", "", text) if remove_line_braks else text
        text = (
            sub("[" + string.punctuation + "]", "", text)
            if remove_punctuation
            else text
        )
        text = sub(" +", " ", text)  # only to avoid multiple spaces
        text = text.split(" ")

        tokenize_list.append(text)
    return tokenize_list


def top_tokens_stopwords(sentence_list: list, percentage: float = 0.01):
    """Selects the most relevant stops words of the tokerized texts.

    Args:
        sentence_list (list): list of tokens.
        percentage (float): percentage threshold.
    """
    percentage = top_words_percentage
    vocabulary = defaultdict(int)

    for sample in sentence_list:
        for token in sample:
            vocabulary[token] += 1

    all_tokens = sorted(vocabulary.items(), key=lambda token: token[1], reverse=True)
    top_tokens = all_tokens[: int(len(all_tokens) * percentage)]

    return [token[0] for token in top_tokens]


def remove_specific_tokens(sentence_list: list, tokens_to_be_removed: list = None):
    """Removes specific tokens from a token list.

    Args:
        sentence_list (list): list of tokens from which other tokens will be removed.
        tokens_to_be_removed (list): list of tokens that need to be removed.
    """
    sentence_list_ = list()
    sentence_list_ = [x for x in sentence_list if x not in tokens_to_be_removed]

    return sentence_list_


def apply_stemming(sentence_list: list):
    ps = PorterStemmer()
    sentence_list = [
        [ps.stem(word) for word in token_list] for token_list in sentence_list
    ]
    return sentence_list


def apply_lemmatization(sentence_list: list):
    lemmatizer = WordNetLemmatizer()
    sentence_list = [
        [lemmatizer.lemmatize(word) for word in token_list]
        for token_list in sentence_list
    ]
    return sentence_list


def apply_casing(sentence_list: list, case: str):
    if case == "Lower":
        sentence_list = [
            [word.lower() for word in token_list] for token_list in sentence_list
        ]
    elif case == "Upper":
        sentence_list = [
            [word.upper() for word in token_list] for token_list in sentence_list
        ]
    else:
        pass
    return sentence_list


def token_restructuring(sentence_list: list):
    """Reduce a nested list of tokens to a single list (1D).

    Args:
        sentence_list (list): list to be work on.
    """
    return reduce(lambda x, y: x + y, sentence_list)


# + [markdown] colab_type="text" id="miiSextFmZso"
# ## Aplicação dos filtros

# + [markdown] colab_type="text" id="QUZAOGE8mZso"
# Baixando stop words do idioma especificado e a wordnet para lemmatizaton

# + colab={} colab_type="code" id="oWSBNLFamZsp" outputId="13c97fdc-8719-4ba0-a457-e51dc2ee196c"
import nltk
if remove_stop_words:
    # Download stopwords from nltk
    nltk.download('stopwords')

    # Get a list of stopwords for the defined language
    stopwords = nltk.corpus.stopwords.words(language)

if lemmatization:
    nltk.download('wordnet')

# + [markdown] colab_type="text" id="nB4iSRIOmZst"
# Aplicando pré processamento

# + colab={} colab_type="code" id="fmu-I2rTmZsw"
vocab = tokenize_text(X)
top_tokens = top_tokens_stopwords(vocab) if remove_top_words else None
vocab = remove_specific_tokens(vocab,top_tokens) if remove_top_words else vocab
vocab = remove_specific_tokens(vocab,stopwords) if remove_stop_words else vocab
vocab = apply_stemming(vocab) if stemming else vocab
vocab = apply_lemmatization(vocab) if lemmatization else vocab
vocab = apply_casing(vocab,case)
text = [' '.join(tokens) for tokens in vocab]

# + [markdown] colab_type="text" id="l0K6F2vSmZsz"
# ## Salva alterações no conjunto de dados
#
# O conjunto de dados será salvo (e sobrescrito com as respectivas mudanças) localmente, no container da experimentação, utilizando a função `pandas.DataFrame.to_csv`.<br>

# + colab={} colab_type="code" id="HnP9SI5wmZs0" outputId="df0b124e-41cd-49ca-cf11-6459746b7fa1"
y_dec = list(label_encoder.inverse_transform(y))
new_columns = ['review', 'sentiment']
df_result = pd.DataFrame(list(zip(text, y_dec)),columns = new_columns) 
#save dataset changes
df_result.to_csv(dataset, index=False)
df_result

StarcoderdataJupyterScriptsDedupFiltered