# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 2
#     language: python
#     name: python2
# ---

# +
from __future__ import division
import pandas as pd
import numpy as np
import gzip
import cPickle as pickle
import json

import scipy.stats

from time import strftime

import sys

from collections import Counter
from operator import itemgetter

from tqdm import tqdm

import matplotlib
import matplotlib.pyplot as plt
# %matplotlib inline

sys.path.append('../TrainModels')
sys.path.append('../ApplyModels')
sys.path.append('../Utils')
from utils import filter_by_filelist, plot_labelled_bb
from utils import saiapr_image_filename, mscoco_image_filename, separation_plot
from train_model import STOPWORDS, is_relational
from train_model import create_word2den, make_train
from apply_model import make_test
# -

from sklearn.metrics import average_precision_score

from tqdm import tqdm

# +
with gzip.open('../Preproc/PreProcOut/saiapr_refdf.pklz', 'r') as f:
    srefdf = pickle.load(f)


## Splits

with open('../Preproc/PreProcOut/saiapr_90-10_splits.json', 'r') as f:
    ssplit90 = json.load(f)

# -

X = np.load('/Volumes/USB_128GB/Models/2016-v3-image-wac/saiapr.npz')
X = X['arr_0']

with gzip.open('../TrainModels/TrainedModels/model05_sr5r.pklz', 'r') as f:
    wac_05 = pickle.load(f)

wac = wac_05

trefdf = filter_by_filelist(srefdf, ssplit90['train'])


# +
def run_wac_on_data(refdf, X, wac, word, nneg=3, nsrc='random'):
    w2d = create_word2den(refdf, [word])
    if word not in w2d:
        return False, False
    X_t, y_t = make_train(X, w2d, word, nneg, nsrc)
    res = wac[word]['clsf'].predict_proba(X_t)[:,1]
    return np.array(y_t), res

def test_vocab(testdf, X, wac, vocab):
    out = []
    for word in tqdm(vocab):
        y_true, y_score = run_wac_on_data(testdf, X, wac, word)
        if y_score is False:
            continue
        auc = (average_precision_score(y_true, y_score))
        pos = wac[word]['npos']
        out.append((word, auc, pos))
    return out


# -

average_precision_score(*run_wac_on_data(trefdf, X, wac, 'water'))

average_precision_score(*run_wac_on_data(trefdf, X, wac, 'white'))

average_precision_score(*run_wac_on_data(trefdf, X, wac, 'yellow'))

test_vocab(trefdf, X, wac, ['boy', 'girl'])

len(wac.keys())

all_aucs = test_vocab(trefdf, X, wac, wac.keys())

# +
#with gzip.open('all_aucs.pklz', 'w') as f:
#    pickle.dump(all_aucs, f)
# -

all_aucs[:10]

auc_dic = {w:s for w,s,_ in all_aucs}

auc_dic['spiders']

auc_dic['white'], auc_dic['black'], auc_dic['red'], auc_dic['yellow'], auc_dic['green']

auc_sorted = np.array(sorted([(w,s,p) for w,s,p in all_aucs], key=itemgetter(1), reverse=True))

plt.figure(figsize=(20,5))
plt.plot(range(len(auc_sorted)), auc_sorted[:,1])
xs_thinned = range(0,len(auc_sorted), 10)
xtick_labels = auc_sorted[:,0][xs_thinned]
_ = plt.xticks(xs_thinned, xtick_labels, rotation='vertical')

auc_sorted[-50:,]

auc_sorted[:50,]

y_t, y_s = run_wac_on_data(trefdf, X, wac, 'dessert')

y_t, y_s, average_precision_score(y_t, y_s)

w2d_full = create_word2den(trefdf, wac.keys())

len([w for w, frids in w2d_full.items() if len(frids) > 20])

reliable_words = [w for w, frids in w2d_full.items() if len(frids) > 20]

r_auc_sorted = np.array(sorted([(w,s,p) for w,s,p in all_aucs if w in reliable_words],
                               key=itemgetter(1), reverse=True))

r_auc_sorted

plt.figure(figsize=(20,5))
plt.plot(range(len(r_auc_sorted)), r_auc_sorted[:,1])
xs_thinned = range(0,len(r_auc_sorted), 8)
xtick_labels = r_auc_sorted[:,0][xs_thinned]
_ = plt.xticks(xs_thinned, xtick_labels, rotation='vertical')

np.mean(r_auc_sorted[:,1].astype('f')), np.std(r_auc_sorted[:,1].astype('f'))

r_auc_sorted[:30]

r_auc_sorted[-30:]

top = [w for w,s,_ in r_auc_sorted if float(s) > 0.85]
len(top)
print top

bottom = [w for w,s,_ in r_auc_sorted if float(s) < 0.60]
len(bottom)
print bottom

colours = ['white', 'black', 'red', 'green', 'blue', 'yellow', 'purple', 'pink']
c_s = [float(s) for w,s,_ in r_auc_sorted if w in colours]
np.mean(c_s), np.std(c_s)

positions = 'left right middle center front back'.split()
p_s = [float(s) for w,s,_ in r_auc_sorted if w in positions]
np.mean(p_s), np.std(p_s)



StarcoderdataJupyterScriptsDedupFiltered