# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

# + colab={"base_uri": "https://localhost:8080/"} id="8VKebFUhOTuq" outputId="66042085-fa89-48d5-a6af-3bd84d5d413c"
#clone repository until final version of package is released
# !git clone https://github.com/oskarfernlund/niteshade.git
# %cd 'niteshade/'

#necessary installs
# !pip install fpdf
# #!pip install niteshade

# + [markdown] id="Na2Sn5owOzFa"
# # Import Necessary Libraries

# + id="7251Nyj6OvCV"
#necessary imports
import numpy as np
import torch

from niteshade.attack import AddLabeledPointsAttacker, LabelFlipperAttacker, BrewPoison
from niteshade.defence import FeasibleSetDefender, KNN_Defender, SoftmaxDefender
from niteshade.models import MNISTClassifier, BaseModel
from niteshade.postprocessing import PostProcessor, PDF
from niteshade.simulation import Simulator, wrap_results
from niteshade.utils import train_test_MNIST, save_plot, get_time_stamp_as_string
from niteshade.data import DataLoader

import matplotlib.pyplot as plt

# + [markdown] id="DPaAPC5bNjsO"
# # Using the **niteshade** Pipeline to Test Attack and Defense Strategies on an Off The Shelf MNIST Classifier

# + colab={"base_uri": "https://localhost:8080/", "height": 103, "referenced_widgets": ["09f247c707114db4af6d98abc1213945", "<KEY>", "<KEY>", "<KEY>", "<KEY>", "<KEY>", "<KEY>", "80ed724924f34862bf9042a841dc9c21", "<KEY>", "<KEY>", "7c0538299e51421fa62142824908c18b"]} id="c-DJ_1EH8jxE" outputId="335f1a3d-70f5-429a-ef97-305f0b8a4549"
X_train, y_train, X_test, y_test = train_test_MNIST()

# + colab={"base_uri": "https://localhost:8080/"} id="gmdansNXPPEz" outputId="66253165-82f6-4e9f-c468-2541f43e7343"
torch.manual_seed(0) #seed for reproducibility

BATCH_SIZE = 128 # --> mini-batches on which model is trained on
NUM_EPISODES = 30 # --> number of times the attacker/defender intervene
lrs = [0.001, 0.005, 0.01, 0.05, 1]

#define attacker and defender
label_flips_dict = {1:9, 9:1}
attacker = LabelFlipperAttacker(1, label_flips_dict)
defender = FeasibleSetDefender(X_train, y_train, 2000)

#define model and simulator
baseline_model = MNISTClassifier(lr=0.01)
simulator_regular = Simulator(X_train, y_train, baseline_model, attacker=attacker,
                    defender=None, batch_size=BATCH_SIZE, num_episodes=NUM_EPISODES)

simulator_regular.run()
simulators = {'regular_0.01': simulator_regular}

for lr in lrs:
    model = MNISTClassifier(lr=lr)
    simulator = Simulator(X_train, y_train, model, attacker=attacker,
                defender=defender, batch_size=BATCH_SIZE, num_episodes=NUM_EPISODES)
    simulator.run()
    simulators[f'lr_{lr}'] = simulator
    

postprocessor = PostProcessor(simulators)
metrics = postprocessor.evaluate_simulators_metrics(X_test, y_test)

# +
fig, ax = plt.subplots(1, figsize=(15,10))
for key, value in metrics.items():
    x = float(key.split('_')[-1])
    y = value
    ax.scatter(x, y, label=key, marker='D', alpha=0.7, s=150)

ax.set_xlabel('Learning Rate')
ax.set_ylabel('Accuracy')
plt.legend()
plt.show()
save_plot(fig, plotname='Learning Rate vs Accuracy')
# -

# !ls output

# ## PDF demo

simulation_metrics = postprocessor.compute_online_learning_metrics(X_test, y_test)
postprocessor.plot_online_learning_metrics(simulation_metrics, plotname='Accuracy by Episode')

# +
# Generate a PDF
time_stamp = get_time_stamp_as_string()
header_title = f'Example Simulation Report as of {time_stamp}'
pdf = PDF()

pdf.set_title(header_title)
pdf.add_all_charts_from_directory('output')
pdf.output('summary_report.pdf', 'F')
# -

from IPython.display import Image
Image("summary_pdf_demo.png")



StarcoderdataJupyterScriptsDedupFiltered