# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
# Real life data

import logging
import json
import threading
import itertools
import pandas as pd
import numpy as np  
import matplotlib.pyplot as plt
import ibm_db
import shap

from pandas_profiling import ProfileReport
from matplotlib import cm
from mpl_toolkits.mplot3d import axes3d
import seaborn as seabornInstance
from sqlalchemy import Column, Integer, String, Float, DateTime, Boolean, func
from iotfunctions import base
from iotfunctions import bif
from iotfunctions.db import Database
from iotfunctions import entity
from iotfunctions import metadata
from iotfunctions.metadata import EntityType
from iotfunctions.enginelog import EngineLogging
from iotfunctions.dbtables import FileModelStore
from iotfunctions import estimator
from iotfunctions.ui import (UISingle, UIMultiItem, UIFunctionOutSingle,
                 UISingleItem, UIFunctionOutMulti, UIMulti, UIExpression,
                 UIText, UIStatusFlag, UIParameters)
from iotfunctions.dbtables import FileModelStore, DBModelStore
from mmfunctions.anomaly import (SaliencybasedGeneralizedAnomalyScore, SpectralAnomalyScore,
                                 FFTbasedGeneralizedAnomalyScore, KMeansAnomalyScore, GBMRegressor,
                                 Standard_Scaler, Robust_Scaler, MinMax_Scaler)
import datetime as dt
from sklearn.model_selection import train_test_split 
from sklearn.linear_model import LinearRegression
#from sklearn.preprocessing import StandardScaler
from sklearn.covariance import MinCovDet
from sklearn import metrics
import scipy as sp
import scipy.fftpack
import skimage as ski  
from skimage import util as skiutil # for nifty windowing
import pyod as pyod
from pyod.utils.data import generate_data
from pyod.utils.data import evaluate_print
from pyod.utils.example import visualize
from pyod.models.knn import KNN
from pyod.models.iforest import IForest
# %matplotlib inline
from pandas.plotting import register_matplotlib_converters
register_matplotlib_converters()

EngineLogging.configure_console_logging(logging.INFO)

# +
with open('credentials_as_monitor_demo.json', encoding='utf-8') as F:
    credentials = json.loads(F.read())
    
db_schema=None
fm = FileModelStore()
db = Database(credentials=credentials, model_store=fm)
print (db)

# +
#con = db.connection.connect()
DB2ConnString = 'DATABASE=' + credentials['db2']['databaseName'] + \
                ';HOSTNAME=' + credentials['db2']['host'] + \
                ';PORT=' + str(credentials['db2']['port']) + \
                ';PROTOCOL=TCPIP;UID=' + credentials['db2']['username'] + \
                ';PWD=' + credentials['db2']['password']
            
db_connection = ibm_db.connect(DB2ConnString, '', '')


# db.model = DBModelStore(credentials['tenantId'], None, credentials['db2']['username'], db.connection.connect(), 'db2'  )
# -

model_store = DBModelStore(credentials['tenantId'], "TESTENTITYID", credentials['db2']['username'], db_connection, 'db2')
db.model_store = model_store

# +
# need a helper function to convert array columns to something easier
from scipy import linalg
def l2norm(df, tcol, col1, col2 = None, col3 = None):
    def str_norm(cols_str):
        '''norm for one string element (encodes an array of value)
           in one column of a data point'''
        return linalg.norm(np.fromstring(cols_str.replace('[',' ').replace(']','').replace('\"', ''), sep = ','))**2

    def column_norm(df, tcol, col1, col2=None, col3=None):
        '''norm of all columns specified in parameters for all datapoints'''
        df_temp = pd.DataFrame()
        df_temp['col1_np'] = df[col1].apply(str_norm)
        df_temp['col2_np'] = 0 
        df_temp['col3_np'] = 0
        if col2 is not None:
            df_temp['col2_np'] = df[col2].apply(str_norm)
        if col3 is not None:
            df_temp['col3_np'] = df[col3].apply(str_norm)

        return (df_temp['col1_np'] + df_temp['col2_np'] + df_temp['col3_np'])**(1/2)
    
    df[tcol] = column_norm(df, tcol, col1, col2, col3)

def unrollAccel(df):
    l0,l1,l2,l3,l4=[],[],[],[],[]
    for i in df['ACCEL_POWER'].values:
        l0.append(eval(eval(i)[0]))
        l1.append(eval(eval(i)[1]))
        l2.append(eval(eval(i)[2]))
        l3.append(eval(eval(i)[3]))
        l4.append(eval(eval(i)[4]))
    df['accel_power_0'] = np.asarray(l0)
    df['accel_power_1'] = np.asarray(l1)
    df['accel_power_2'] = np.asarray(l2)
    df['accel_power_3'] = np.asarray(l3)
    df['accel_power_4'] = np.asarray(l4)
    
listAttr = ['timestamp','entity','vibrations','rms','accel_speed','accel_power_0','accel_power_1',
            'accel_power_2','accel_power_3','accel_power_4']

# +
# Now we proceed to customer data - GOOD CASE

# Get stuff in
df_input_raw = pd.read_csv('./Armstark04714B6046D5.csv', index_col=False, parse_dates=['RCV_TIMESTAMP_UTC'])
df_input_raw['entity']=df_input_raw['DEVICE_ID']
df_input_raw['timestamp']=df_input_raw['RCV_TIMESTAMP_UTC']

# and sort it by timestamp
df_input_raw = df_input_raw.sort_values(by='timestamp')
df_input_raw = df_input_raw.set_index(['entity','timestamp']).dropna()

l2norm(df_input_raw, 'vibrations', 'VIBRATIONS_XAXIS', 'VIBRATIONS_YAXIS', 'VIBRATIONS_ZAXIS')
l2norm(df_input_raw, 'rms', 'RMS_X', 'RMS_Y', 'RMS_Z')
l2norm(df_input_raw, 'accel_speed', 'ACCEL_SPEED')
unrollAccel(df_input_raw)
#l2norm(df_input_raw, 'accel_power', 'ACCEL_POWER')

df_input = df_input_raw.filter(listAttr, axis=1)
df_input_raw.describe()
# -

# #### Pandas Profiling
#
# Try Pandas Profiling to get an overview about the data, mostly its distributions and correlations
# <br>
#

# +
# df_input[['accel_power_0','accel_anomaly']].head(20)

# removed 'rms'
#features=['accel_speed','accel_power_0','accel_power_1','accel_power_2','accel_power_3','accel_power_4']
#df_input['rms2'] = df_input['rms']
features=['rms']
targets=['rms']
predictions=['rms_std']
predictions2=['rms_rob']
# -

# #### Customer suggested a correlation between vibration and acceleration
#
# so let's try to predict (although correlation tests do not really indicate it)

# +
# Run Monitoring's anomaly detector functions
EngineLogging.configure_console_logging(logging.DEBUG)
print(features, targets, predictions)

stdii = Standard_Scaler(features=features, targets=targets, predictions=predictions)
robii = MinMax_Scaler(features=features, targets=targets, predictions=predictions2)

jobsettings = { 'db': db, 
               '_db_schema': 'public', 'save_trace_to_file' : True}
et = stdii._build_entity_type(columns = [Column('rms',Float()), Column('accel_speed',Float()),
                                        Column('vibrations',Float())], **jobsettings)
et2 = robii._build_entity_type(columns = [Column('rms',Float()), Column('accel_speed',Float()),
                                        Column('vibrations',Float())], **jobsettings)

stdii._entity_type = et
robii._entity_type = et2

# allow training and delete existing models
stdii.auto_train = True
stdii.delete_existing_models = True
robii.auto_train = True
robii.delete_existing_models = True

df_input = stdii.execute(df=df_input)
# -

df_input = robii.execute(df=df_input)

fig, ax = plt.subplots(3, 1, figsize=(15, 9))
df_input.plot(ax=ax[0], y='rms')
df_input.plot(ax=ax[1], y='rms_std', color='green')
df_input.plot(ax=ax[2], y='rms_rob', color='green')
ax[0].set_xticklabels([])
ax[1].set_xticklabels([])
ax[2].set_xticklabels([])

df_input['rms'] = df_input['accel_speed']

# +
predictions3 = ['rms_std']
stdii2 = Standard_Scaler(features=features, targets=targets, predictions=predictions3)
stdii2.correlation_threshold = 0.001

jobsettings = { 'db': db, 
               '_db_schema': 'public', 'save_trace_to_file' : True}
et = stdii2._build_entity_type(columns = [Column('accel_power_0',Float()), Column('accel_power_1',Float()),
                                        Column('vibrations',Float())], **jobsettings)
stdii2._entity_type = et

# disallow training and preserve existing models for predict
stdii2.auto_train = False
stdii2.delete_existing_models = False
df_input = stdii2.execute(df=df_input)
# -

df_input

fig, ax = plt.subplots(3, 1, figsize=(15, 9))
df_input.plot(ax=ax[0], y='rms')
df_input.plot(ax=ax[1], y='rms_std', color='green')
#df_input.plot(ax=ax[2], y='rms_std2', color='green')
ax[0].set_xticklabels([])
ax[1].set_xticklabels([])
ax[2].set_xticklabels([])

# +
# Run Monitoring's anomaly detector functions
# EngineLogging.configure_console_logging(logging.DEBUG)

simpleii = SimpleRegressor(features=['accel_power_0','accel_power_1'], # max_depth=20, num_leaves=40,
                     n_estimators=4000, learning_rate=0.00001, targets=['rms'],
                     predictions=['rms_pred'])
simpleii.correlation_threshold = 0.001

jobsettings = { 'db': db, 
               '_db_schema': 'public', 'save_trace_to_file' : True}
et = simpleii._build_entity_type(columns = [Column('accel_power_0',Float()), Column('accel_power_1',Float()),
                                            Column('vibrations',Float())], **jobsettings)
simpleii._entity_type = et

# allow training and delete existing models
simpleii.auto_train = True
simpleii.delete_existing_models = True
df_input = simpleii.execute(df=df_input)

# +

simpleii = SimpleRegressor(features=['accel_power_0','accel_power_1'], # max_depth=20, num_leaves=40,
                     n_estimators=4000, learning_rate=0.00001, targets=['rms'],
                     predictions=['rms_pred'])
simpleii.correlation_threshold = 0.001

jobsettings = { 'db': db, 
               '_db_schema': 'public', 'save_trace_to_file' : True}
et = simpleii._build_entity_type(columns = [Column('accel_power_0',Float()), Column('accel_power_1',Float()),
                                            Column('vibrations',Float())], **jobsettings)
simpleii._entity_type = et

# disallow training and preserve existing models for predict
simpleii.auto_train = False
simpleii.delete_existing_models = False
df_input = simpleii.execute(df=df_input)

# -



StarcoderdataJupyterScriptsDedupFiltered