# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Anago36
#     language: python
#     name: anago36
# ---

# +
import sys
sys.executable

# %load_ext autoreload
# %autoreload 2
# %reload_ext autoreload
'''
TRAINING_DATA_PATH = "/Users/slouvan/sandbox/cross-domain/data/frame.train.conll"
DEV_DATA_PATH = "/Users/slouvan/sandbox/cross-domain/data/frame.dev.conll"
TEST_DATA_PATH = "/Users/slouvan/sandbox/cross-domain/data/frame.test.conll"
'''

'''
TRAINING_DATA_PATH = "/Users/slouvan/sandbox/cross-domain/data/ner_conll2003_en.train.conll"
DEV_DATA_PATH = "/Users/slouvan/sandbox/cross-domain/data/ner_conll2003_en.dev.conll"
TEST_DATA_PATH = "/Users/slouvan/sandbox/cross-domain/data/ner_conll2003_en.test.conll"
'''

'''
TRAINING_DATA_PATH = "/Users/slouvan/sandbox/anago/data/conll2003/en/ner/train.lower.txt"
DEV_DATA_PATH = "/Users/slouvan/sandbox/anago/data/conll2003/en/ner/valid.lower.txt"
TEST_DATA_PATH = "/Users/slouvan/sandbox/anago/data/conll2003/en/ner/test.lower.txt"
'''

'''
TRAINING_DATA_PATH = "/Users/slouvan/sandbox/cross-domain/data/restaurant.train.conll"
DEV_DATA_PATH = "/Users/slouvan/sandbox/cross-domain/data/restaurant.dev.conll"
TEST_DATA_PATH = "/Users/slouvan/sandbox/cross-domain/data/restaurant.test.conll"
'''


'''
TRAINING_DATA_PATH = "/Users/slouvan/sandbox/cross-domain/data/movie_MIT.train.conll"
DEV_DATA_PATH = "/Users/slouvan/sandbox/cross-domain/data/movie_MIT.dev.conll"
TEST_DATA_PATH = "/Users/slouvan/sandbox/cross-domain/data/movie_MIT.test.conll"
'''


TRAINING_DATA_PATH = "/Users/slouvan/sandbox/cross-domain/data/atis-2.train.conll"
DEV_DATA_PATH = "/Users/slouvan/sandbox/cross-domain/data/atis-2.dev.conll"
TEST_DATA_PATH = "/Users/slouvan/sandbox/cross-domain/data/atis.test.iob.conll"


# -

import anago
from anago.reader import load_data_and_labels, load_glove

x_train, y_train = load_data_and_labels(TRAINING_DATA_PATH)
x_dev, y_dev = load_data_and_labels(DEV_DATA_PATH)
x_test, y_test = load_data_and_labels(TEST_DATA_PATH)

# +
print(len(x_test))
print(len(y_train))
print(len(y_dev))
print(y_test[0])

cnt = 0
for x in x_train:
    if len(x) == 1:
        cnt +=1
print(cnt)
# -

embeddings = load_glove("/Users/slouvan/sandbox/cross-domain/data/glove.6B.100d.txt")


model = anago.Sequence(embeddings=embeddings,char_feature=True)

model.train(x_train, y_train, x_dev, y_dev)

model.eval(x_test, y_test)

model.save("/Users/slouvan/sandbox/anago/mymodel")

# +
from anago import config

model = anago.Sequence(char_feature=True)
model = model.load(config.ANAGO_CHECKPOINT_DIR)
model.eval(x_test, y_test)

# +
'''
 soccer  o
 -   o   
 japan   b-loc
 get o
 lucky   o   
 win o
 ,   o   
 china   b-per
 in  o
 surprise    o   
 defeat  o
 .   o 
'''

model.analyze("i want to fly from milan to boston .".split())

# -

model.eval(x_test, y_test)





StarcoderdataJupyterScriptsDedupFiltered