# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# Tested single data split only, not cross validation applied.
#
# And results are not very consistent, vary with each run...
#
# ### Polarity classification
#
#     INFO:tensorflow:***** Eval results *****
#     INFO:tensorflow:  eval_accuracy = 0.8264151
#     INFO:tensorflow:  eval_loss = 0.4085315
#     INFO:tensorflow:  global_step = 744
#     INFO:tensorflow:  loss = 0.41608095
#
# ### Category classification
#
#     INFO:tensorflow:***** Eval results *****
#     INFO:tensorflow:  eval_accuracy = 0.49811321
#     INFO:tensorflow:  eval_loss = 1.4121249
#     INFO:tensorflow:  global_step = 223
#     INFO:tensorflow:  loss = 1.3853042
#

# ! git clone https://github.com/daisukelab/dl-cliche.git
# ! cd dl-cliche && pip install .
# ! rm -fr dl-cliche

from dlcliche.utils import *
from dlcliche.nlp_mecab import *

# Download dataset & stop_words_ja.txt
# ! wget https://s3-ap-northeast-1.amazonaws.com/dev.tech-sketch.jp/chakki/public/chABSA-dataset.zip
# ! unzip -q chABSA-dataset.zip && rm chABSA-dataset.zip && rm -r __MACOSX
# ! ls chABSA-dataset
# ! cd chABSA-dataset && wget https://raw.githubusercontent.com/chakki-works/chABSA-dataset/master/notebooks/resource/stop_words_ja.txt

# ! wget https://storage.googleapis.com/bert_models/2018_11_03/multilingual_L-12_H-768_A-12.zip
# ! unzip multilingual_L-12_H-768_A-12.zip 
# ! rm multilingual_L-12_H-768_A-12.zip

# +
DATA = Path('chABSA-dataset')

def check_data_existence(folder):
    file_count = len(list(folder.glob("e*_ann.json")))
    if  file_count == 0:
        raise Exception("Processed Data does not exist.")
    else:
        print("{} files ready.".format(file_count))

check_data_existence(DATA)

stop_words = []
with (DATA/"stop_words_ja.txt").open(encoding="utf-8") as f:
    stop_words = f.readlines()
    stop_words = [w.strip() for w in stop_words]

print("{} stop words ready.".format(len(stop_words)))
# -

labels = []
# make labels (exclude NULL and OOD)
for e in ["market", "company", "business", "product"]:
    for a in ["general", "sales", "profit", "amount", "price", "cost"]:
        labels.append(e + "#" + a)
        if e in ["market"]:
            break;
print(labels)

# +
import json
import numpy as np
import pandas as pd
from collections import Counter

sentences = []
dataset = []
tokenizer = get_mecab_tokenizer()

for f in DATA.glob("e*_ann.json"):
    with f.open(encoding="utf-8") as j:
        d = json.load(j)
        for s in d["sentences"]:
            tokenized = tokenizer.tokenize(s["sentence"].upper())
            for o in s["opinions"]:
                if o["category"] in labels:
                    # sentence index + category
                    dataset.append((len(sentences), o["category"], o["polarity"]))
            sentences.append(tokenized)
# -

# ## Polarity classification

# +
from sklearn.model_selection import train_test_split
Y = 2
dataset = np.array(dataset)
Xtrn, Xval, ytrn, yval = train_test_split(dataset[:, 0], dataset[:, Y], test_size=0.1, random_state=0)

def write_dataset(filename, X, y):
    with open(filename, 'w') as f:
        for _x, _y in zip(X, y):
            w = list(sentences[int(_x)])
            f.write(_y+'\t'+' '.join(w)+'\n')
write_dataset(DATA/'train.tsv', Xtrn, ytrn)
write_dataset(DATA/'valid.tsv', Xval, yval)
len(Xtrn), len(Xval), list(set(dataset[:, Y]))
# -

# ! export BERT_BASE_DIR=./multilingual_L-12_H-768_A-12 && export ChABSA_DIR=./chABSA-dataset && python run_classifier.py   --task_name=generic   --do_train=true   --do_eval=true   --data_dir=$ChABSA_DIR   --vocab_file=$BERT_BASE_DIR/vocab.txt  --bert_config_file=$BERT_BASE_DIR/bert_config.json   --init_checkpoint=$BERT_BASE_DIR/bert_model.ckpt   --max_seq_length=128   --train_batch_size=32   --learning_rate=2e-5   --num_train_epochs=3.0   --output_dir=/tmp/chabsa_output/

# ## Category classification

# +
from sklearn.model_selection import train_test_split
Y = 1
dataset = np.array(dataset)
Xtrn, Xval, ytrn, yval = train_test_split(dataset[:, 0], dataset[:, Y], test_size=0.1, random_state=0)

def write_dataset(filename, X, y):
    with open(filename, 'w') as f:
        for _x, _y in zip(X, y):
            w = list(sentences[int(_x)])
            f.write(_y+'\t'+' '.join(w)+'\n')
write_dataset(DATA/'train.tsv', Xtrn, ytrn)
write_dataset(DATA/'valid.tsv', Xval, yval)
len(Xtrn), len(Xval), list(set(dataset[:, Y]))
# -

# ! export BERT_BASE_DIR=./multilingual_L-12_H-768_A-12 && export ChABSA_DIR=./chABSA-dataset && python run_classifier.py   --task_name=generic   --do_train=true   --do_eval=true   --data_dir=$ChABSA_DIR   --vocab_file=$BERT_BASE_DIR/vocab.txt  --bert_config_file=$BERT_BASE_DIR/bert_config.json   --init_checkpoint=$BERT_BASE_DIR/bert_model.ckpt   --max_seq_length=128   --train_batch_size=32   --learning_rate=2e-5   --num_train_epochs=3.0   --output_dir=/tmp/chabsa2_output/



StarcoderdataJupyterScriptsDedupFiltered