# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 2
#     language: python
#     name: python2
# ---

# %matplotlib inline
# %config InlineBackend.figure_format = 'svg'
import matplotlib
import numpy as np
import matplotlib.pyplot as plt

# +
acc_results = {}
loss_results = {}
# List files in data:
from os import listdir
for folder in listdir("data"):
    if "expr2" not in folder:
        continue
    acc_results[folder] = []
    loss_results[folder] = []
    for filename in listdir("data/"+folder):
        if filename != 'ps.out':
            with open("data/"+folder+"/"+filename, "r") as f:
                for line in f.readlines():
                    if line.startswith("INFO:tensorflow:Step:"):
                        parts = line.split(" ")
                        step = int(parts[1][:-1])
                        if step > 5000:
                            break
                        accuracy = float(parts[3][:-1])
                        loss = float(parts[5])
                        acc_results[folder].append((step, accuracy))
                        loss_results[folder].append((step, loss))

for key, val in acc_results.items():
    val.sort()
    acc_results[key] = [y for x,y in val]
for key, val in loss_results.items():
    val.sort()
    loss_results[key] = [y for x,y in val]


# -

def moving_average(a, n=100) :
    ret = np.cumsum(a, dtype=float)
    ret[n:] = ret[n:] - ret[:-n]
    return ret[n - 1:] / n




# +
best_async = None
best_sync = None
async_keeper = None
sync_keeper = None
for folder, loss in loss_results.items():
    if "True" in folder:
        avg = moving_average(loss)
        if len(avg) < 25:
            continue
        if avg[-1] < sync_keeper or sync_keeper is None:
            sync_keeper = avg[-1]
            best_sync = avg
            print folder
for folder, loss in loss_results.items():
    if "False" in folder:
        avg = moving_average(loss)
        if len(avg) < 25:
            continue
        if avg[-1] < async_keeper or async_keeper is None:
            async_keeper = avg[-1]
            best_async = avg
            print folder

plt.plot(best_sync, label="Sync")
plt.plot(best_async, label="Async")
plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)


# +
best_async = None
best_sync = None
async_keeper = None
sync_keeper = None
for folder, acc in acc_results.items():
    if "True" in folder:
        avg = moving_average(acc)
        if len(avg) < 25:
            continue
        if avg[-1] > sync_keeper or sync_keeper is None:
            sync_keeper = avg[-1]
            best_sync = avg
            print folder
for folder, acc in acc_results.items():
    if "False" in folder:
        avg = moving_average(acc)
        if len(avg) < 25:
            continue
        if avg[-1] > async_keeper or async_keeper is None:
            async_keeper = avg[-1]
            best_async = avg
            print folder

plt.plot(best_sync, label="Sync")
plt.plot(best_async, label="Async")
plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)

# +
best_sync = "expr2-0.01-0.75-True"# "expr2-0.01-0.6-True"
best_async = "expr2-0.0025-0.9-False"# "expr2-0.01-0.6-False"
s = moving_average(loss_results[best_sync])
a = moving_average(loss_results[best_async])

min_s = min(s)
min_a = min(a)
target = max(min_s, min_a)
async_result = np.argmax(a <= target)
sync_result = np.argmax(s <= target)

plt.plot([1,4],np.array([sync_result,async_result])/float(sync_result), '-s')
plt.ylabel('Statistical Efficiency')
plt.xlabel('# Compute groups')
plt.grid()


# -



np.argmax(a<target)

for k, v in acc_results.items():
    if len(v) < 40:
        print k









# +

momentums = [0.9, 0.75, 0.6, 0.45, 0.3, 0.15, 0.0]
lrs = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1]

f, axes = plt.subplots(1, len(lrs), sharey=True, figsize=(12,3))

for ind in range(len(lrs)):
    lr=lrs[ind]
    x_sync = []
    y_sync = []
    x_async = []
    y_async = []
    for m in momentums:
        sync_key = "expr2-%s-%s-%s" % (str(lr), str(m), "True")
        async_key = "expr2-%s-%s-%s" % (str(lr), str(m), "False")
        x_sync.append(m)
        x_async.append(m)
        if sync_key in acc_results and len(acc_results[sync_key]) > 40:                
            y_sync.append(moving_average(acc_results[sync_key])[-1])
        else:
            y_sync.append(np.nan)
        if async_key in acc_results and len(acc_results[async_key]) > 40:
            y_async.append(moving_average(acc_results[async_key])[-1])
        else:
            y_async.append(np.nan)


    ax = axes[ind]
    ax.plot(x_sync, y_sync, '-*', markersize=12)
    ax.plot(x_async, y_async, '-v', markersize=12)
    ax.set_xlabel('Explicit momentum')
    ax.set_ylabel('Training Accuracy')
    ax.set_title('LR = ' + str(lr))
    ax.grid()
    ax.axis([0,1.0, 0.56, 0.8])
    plt.legend(['Synchronous', 'Asynchronous'], loc='best')

    
f.savefig('exp2-accuracy-4graphs-full.png', bbox_inches='tight', pad_inches=0.1)



# +

momentums = [0.9, 0.75, 0.6, 0.45, 0.3, 0.15, 0.0]
lrs = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1]

f, axes = plt.subplots(1, len(lrs), sharey=True, figsize=(12,3))

for ind in range(len(lrs)):
    lr=lrs[ind]
    x_sync = []
    y_sync = []
    x_async = []
    y_async = []
    for m in momentums:
        sync_key = "expr2-%s-%s-%s" % (str(lr), str(m), "True")
        async_key = "expr2-%s-%s-%s" % (str(lr), str(m), "False")
        x_sync.append(m)
        x_async.append(m)
        if sync_key in loss_results:
            if len(acc_results[sync_key]) < 100:
                y_sync.append(np.nan)
            else:
                y_sync.append(moving_average(loss_results[sync_key])[-1])
        else:
            y_sync.append(np.nan)
        if async_key in loss_results:
            if len(acc_results[async_key]) < 100:
                y_async.append(np.nan)
            else:
                y_async.append(moving_average(loss_results[async_key])[-1])
        else:
            y_async.append(np.nan)


    ax = axes[ind]
    ax.plot(x_sync, y_sync, '-*', markersize=12)
    ax.plot(x_async, y_async, '-v', markersize=12)
    ax.set_xlabel('Explicit momentum')
    ax.set_ylabel('Training Accuracy')
    ax.set_title('LR = ' + str(lr))
    ax.grid()
    ax.axis([0,1.0, 4.5, 5.25])
    plt.legend(['Synchronous', 'Asynchronous'], loc='best')
    
f.savefig('exp2-loss-4graphs-full.png', bbox_inches='tight', pad_inches=0.1)

# +
acc_results = {}
loss_results = {}
# List files in data:
from os import listdir
for folder in listdir("data"):
    acc_results[folder] = []
    loss_results[folder] = []
    for filename in listdir("data/"+folder):
        if filename != 'ps.out':
            with open("data/"+folder+"/"+filename, "r") as f:
                for line in f.readlines():
                    if line.startswith("INFO:tensorflow:Step:"):
                        parts = line.split(" ")
                        step = int(parts[1][:-1])
                        if step > 5000:
                            break
                        accuracy = float(parts[3][:-1])
                        loss = float(parts[5])
                        acc_results[folder].append((step, accuracy))
                        loss_results[folder].append((step, loss))

for key, val in acc_results.items():
    val.sort()
    acc_results[key] = [y for x,y in val]
for key, val in loss_results.items():
    val.sort()
    loss_results[key] = [y for x,y in val]

# -



StarcoderdataJupyterScriptsDedupFiltered