<jupyter_start><jupyter_text>Anomaly Detection Model - FIRST  Modellbeschreibung:- Training auf den Jahren 2020 und 2021- Adam als Opzimizer- Komplexerer AE- Mehr als 6 Epochen- Normalisierung der Daten- Modell wie aus der Vorstudie  Imports<jupyter_code>%run -i ./scripts/PythonImports.py<jupyter_output><empty_output><jupyter_text>Set Configs and load data<jupyter_code>%run -i ./scripts/TrainPreperations.py<jupyter_output>Shape of normal data: (105216, 17)
Shape of anormal data: (35040, 18)<jupyter_text>Skalieren der Daten & "Column name zu Zahl"-Mapping speichern & Daten für PyTorch vorbereiten<jupyter_code>%run -i ./scripts/ScaleAndPrepare.py<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>from models.SimpleAutoEncoder import SimpleAutoEncoder
torch.manual_seed(42)
num_inputs = len(df_data.columns)
print('Num Inputs: {}'.format(num_inputs))

val_lambda = 0.5
model = SimpleAutoEncoder(num_inputs=num_inputs, val_lambda=val_lambda)
critereon = mse_loss
optimizer = Adam(model.parameters())

print(model)
losses = []
EPOCHS = 32
for epoch in range(EPOCHS):
    model.train()
    for batch_idx, (data,target) in enumerate(trn_dataloader):
        data = torch.autograd.Variable(data)
        optimizer.zero_grad()
        pred = model(data)
        loss = critereon(pred, data)
        losses.append(loss.cpu().data.item())
        
        # Backpropagation
        loss.backward()
        optimizer.step()
        
        # Display
        if batch_idx % 50 == 1:
            print('\n Train Epoch: {}/{} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch+1,
                EPOCHS,
                batch_idx * len(data), 
                len(trn_dataloader.dataset),
                100. * batch_idx / len(trn_dataloader), 
                loss.cpu().data.item()), 
                end='')
fig = plt.figure(figsize=(16,8))
plt.plot(np.arange(len(losses)),losses)
plt.title('Loss (Training) Einfacher AE',fontsize=20)
loss_fn = '{}_AE_model_loss.pdf'.format(arrow.now().format('YYYYMMDD'))
fn = os.path.join(fig_path, loss_fn)
fig.savefig(fn, bbox_inches = 'tight', pad_inches = 0)<jupyter_output><empty_output><jupyter_text>Save Modell<jupyter_code>model_fn = '{}_firstAE_model.pt'.format(arrow.now().format('YYYYMMDD'))
print('Model Name: {}'.format(model_fn))
torch.save(model.state_dict(), os.path.join(model_bib_path, model_fn))<jupyter_output>Model Name: 20200303_firstAE_model.pt<jupyter_text>Logistic Regression zum Lernen des Thresholds<jupyter_code>from sklearn.linear_model import LogisticRegression
from utils.evalUtils import calc_cm_metrics

from models.SimpleAutoEncoder import SimpleAutoEncoder
torch.manual_seed(42)
s_labels = df_data_anormal['label']
df_data_anormal.drop('label', axis=1, inplace=True)
num_inpus = len(df_data_anormal.columns)
val_lambda = 42 * 0.01
fn_now = os.path.join(model_path, 'model_bib' ,'20200303_firstAE_model.pt')
model = SimpleAutoEncoder(num_inputs=num_inpus, val_lambda=val_lambda)
model.load_state_dict(torch.load(fn_now))
model.eval()
data_fn_anormal = os.path.join(data_path, 'anomalous_data_y_2022_reduced.h5')
df_data_anormal = pd.read_hdf(data_fn_anormal, key='df')
print('Shape of anormal data: {}'.format(df_data_anormal.shape))
scaler_train = MinMaxScaler((-1,1))
scaler_train = scaler_train.fit(df_data)
df_data_anormal.shape
scaled_anormal = scaler_train.transform(df_data_anormal.to_numpy())
# build tensor from numpy
anormal_torch_tensor = torch.from_numpy(scaled_anormal).type(torch.FloatTensor)
# build TensorDataset from Tensor
anormal_dataset = TensorDataset(anormal_torch_tensor, anormal_torch_tensor)
# build DataLoader from TensorDataset
anormal_dataloader = torch.utils.data.DataLoader(anormal_dataset,batch_size=128,shuffle=False, num_workers=0)
losses_anormal = []
for val in anormal_torch_tensor:
    loss = model.calc_reconstruction_error(val)
    losses_anormal.append(loss.item())
    
s_losses_anormal = pd.Series(losses_anormal)
X = s_losses_anormal.to_numpy()
X = X.reshape(-1, 1)
y = [1 if x > 0 else 0 for x in s_labels]
clf = LogisticRegression(random_state=42, fit_intercept=True, solver='liblinear', class_weight={1:2.0})
clf.fit(X, y)
predictions = []
for val in X:
    val = val.reshape(1,-1)
    pred = clf.predict(val)
    predictions.append(pred[0])
from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score
from sklearn.metrics import precision_score
from mlxtend.plotting import plot_confusion_matrix
import joblib
cm = confusion_matrix(y, predictions)
tn, fp, fn, tp  = confusion_matrix(y, predictions).ravel()
accuracy, precision, specifity, sensitivity, f1_score = calc_cm_metrics(tp, tn, fp, fn)
print('Accuracy: {}'.format(accuracy))
print('Precision: {}'.format(precision))
print('Specifity: {}'.format(specifity))
print('Sensitivity: {}'.format(sensitivity))
print('F1-Score: {}'.format(f1_score))<jupyter_output>Accuracy: 99.70034246572497
Precision: 99.99999999958195
Specifity: 93.15299842562096
Sensitivity: 95.79495394435004
F1-Score: 97.85232153794672<jupyter_text>Save Model<jupyter_code>model_fn = '{}_LogRegModell.save'.format(arrow.now().format('YYYYMMDD'))
filename = os.path.join(model_bib_path, model_fn)
joblib.dump(clf, filename)<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup