<jupyter_start><jupyter_text>Performance vs Learning Rate<jupyter_code>def get_vs_lr_scores(learning_rates_str, top_k):
    scores = {'loss': list(), 'Recall@1': list(), 'Recall@5': list(), 'Recall@20': list()}
    lr_list = list()
    for lr_str in learning_rates_str:
        dir_path = root / '1' / f'top{top_k}' / lr_str
        file_name = [x for x in os.listdir(dir_path) if x.startswith('best_checkpoint_')][0]
        checkpoints = torch.load(dir_path / file_name, 'cpu')
        for m in scores.keys():
            scores[m].append(checkpoints['metrics_scores'][m])
        lr_list.append(checkpoints['solver_construct_params_dict']['learning_rate'])
    scores['learning_rate'] = lr_list
    return scores
def plot_vs_lr(scores, logscale):
    fig, ax = plt.subplots(2, 2, figsize=(10, 10))
    ax[0,0].plot(scores['learning_rate'], scores['loss'])
    ax[0,0].set_title('loss vs learning rate')
    
    ax[0,1].plot(scores['learning_rate'], scores['Recall@1'])
    ax[0,1].set_title('Recall@1 vs learning rate')

    ax[1,0].plot(scores['learning_rate'], scores['Recall@5'])
    ax[1,0].set_title('Recall@5 vs learning rate')
   
    ax[1,1].plot(scores['learning_rate'], scores['Recall@20'])
    ax[1,1].set_title('Recall@20 vs learning rate')
    
    if logscale:
        ax[0,0].set_xscale('log')
        ax[0,1].set_xscale('log')
        ax[1,0].set_xscale('log')
        ax[1,1].set_xscale('log')
    
    for a in ax:
        for b in a:
            b.set_xlabel('learning rate')
    ax[0,0].set_ylabel('loss')
    ax[0,1].set_ylabel('Recall@1')
    ax[1,0].set_ylabel('Recall@5')
    ax[1,1].set_ylabel('Recall@20')
    fig.tight_layout()
learning_rates_str = ['1e-3', '1e-4', '1e-5', '1e-6','1e-7']
top_k=20
scores = get_vs_lr_scores(learning_rates_str, top_k)
plot_vs_lr(scores, True)
learning_rates_str = ['1e-7', '2e-7', '3e-7', '4e-7', '5e-7', '6e-7', '7e-7', '8e-7', '9e-7']
top_k=20
scores = get_vs_lr_scores(learning_rates_str, top_k)
plot_vs_lr(scores, False)<jupyter_output><empty_output><jupyter_text>A Random Point vs Nearest Neighbor<jupyter_code>def get_vs_arch_scores(model_arch, top_k):
    scores = {'loss': list(), 'Recall@1': list(), 'Recall@5': list(), 'Recall@20': list()}
    lr_str = '5e-7'
    for arch in model_arch:
        dir_path = root / arch / f'top{top_k}' / lr_str
        file_name = [x for x in os.listdir(dir_path) if x.startswith('best_checkpoint_')][0]
        checkpoints = torch.load(dir_path / file_name, 'cpu')
        for m in scores.keys():
            scores[m].append(checkpoints['metrics_scores'][m])
    scores['model_arch'] = model_arch
    return scores
top_k = 20
model_arch = ['500-100-20', '500-100-20-20', '5000-1000-200-200']
scores = get_vs_arch_scores(model_arch, top_k)
scores<jupyter_output><empty_output><jupyter_text>retrieve a model checkpoint<jupyter_code>s = '200-200-100-100-100-50-50-50-25-25-25-20-20'
lr = '1e-2'
dir_path = Path('../checkpoints') / s / 'top20' / lr
file_name = [x for x in os.listdir(dir_path) if x.startswith('best_checkpoint_')][0]

print(torch.load(dir_path / file_name, 'cpu')['dev_metrics_scores'])<jupyter_output>{'loss': 2751.59326171875, 'Recall@1': 0.01701701701701702, 'Recall@5': 0.06306306306306306, 'Recall@10': 0.12612612612612611, 'Recall@20': 0.22922922922922923}<jupyter_text>performance vs number of top_k neighbors considered<jupyter_code>def get_vs_arch_scores(top_k_list):
    scores = {'loss': list(), 'Recall@1': list(), 'Recall@5': list(), 'Recall@20': list()}
    lr_str = '5e-7'
    for top_k in top_k_list:
        dir_path = root / '200-100-50-25-20' / f'top{top_k}' / lr_str
        file_name = [x for x in os.listdir(dir_path) if x.startswith('best_checkpoint_')][0]
        checkpoints = torch.load(dir_path / file_name, 'cpu')
        for m in scores.keys():
            scores[m].append(checkpoints['dev_metrics_scores'][m])
    scores['top_k'] = top_k_list
    return scores
scores = get_vs_arch_scores([1,3,5,7,9,11,13,15,17,19])
scores
plot_subgraph(scores, False, 'top_k')<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup