# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import numpy as np
import pandas as pd
import re
import json
from konlpy.tag import Okt
from keras.preprocessing.sequence import pad_sequences
from keras.preprocessing.text import Tokenizer

f = open('./data_in/ChatBotData.csv','r', encoding='utf-8')
chatBotData = [line.strip().split(',') for line in f.readlines()]

chatBotData[1:]

okt = Okt()
voca = []
for line in chatBotData[1:]:
    for text in line[:2]:
        voca.extend(okt.morphs(text))

voca[0:10]



StarcoderdataJupyterScriptsDedupFiltered