# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: dapnn
#     language: python
#     name: dapnn
# ---

# <h1>Table of Contents<span class="tocSkip"></span></h1>
# <div class="toc"><ul class="toc-item"><li><span><a href="#PDC-2020" data-toc-modified-id="PDC-2020-1"><span class="toc-item-num">1&nbsp;&nbsp;</span>PDC 2020</a></span></li><li><span><a href="#PDC-2021" data-toc-modified-id="PDC-2021-2"><span class="toc-item-num">2&nbsp;&nbsp;</span>PDC 2021</a></span></li><li><span><a href="#Datasets-utilized-in-the-BINET-Paper" data-toc-modified-id="Datasets-utilized-in-the-BINET-Paper-3"><span class="toc-item-num">3&nbsp;&nbsp;</span>Datasets utilized in the BINET Paper</a></span></li></ul></div>

# +
#default_exp conversion
# -

# # Log Conversion
#
# > Converts the event logs into csv format to make it easier to load them
#
# We did not store the unprocessed data in the github repository, because of github file size limits and total repository size. Therefore, this notebook cannot be run without the original data. However, this is also not required, because the resulting csvs are stored. We keep it here, to show how we converted the data into csvs. If you are interested in the original data, please check out the original sources.

# %load_ext autoreload
# %autoreload 2
# %matplotlib inline

from dapnn.imports import *


# ## PDC 2020

# Convert Training Logs:

for fn in progress_bar(glob.glob('data/orig/unzipped/PDC2020_training/*')):
    new_fn=fn.split('.')[0]+'.csv.gz'
    new_fn='data/csv/'+"/".join(new_fn.split('/')[3:])
    log = xes_importer.apply(fn)
    df = log_converter.apply(log, variant=log_converter.Variants.TO_DATA_FRAME)
    df.to_csv(new_fn,index=False,compression='gzip')


# Convert Ground Truth Logs:

for fn in progress_bar(glob.glob('data/orig/unzipped/PDC2020_ground_truth/*')):
    new_fn=fn.split('.')[0]+'.csv.gz'
    new_fn='data/csv/'+"/".join(new_fn.split('/')[3:])
    log = xes_importer.apply(fn)
    df = log_converter.apply(log, variant=log_converter.Variants.TO_DATA_FRAME)
    df.to_csv(new_fn,index=False,compression='gzip')


# ## PDC 2021

# Convert Training Logs:

for fn in progress_bar(glob.glob('data/orig/unzipped/PDC2021_training/*')):
    new_fn=fn.split('.')[0]+'.csv.gz'
    new_fn='data/csv/'+"/".join(new_fn.split('/')[3:])
    log = xes_importer.apply(fn)
    df = log_converter.apply(log, variant=log_converter.Variants.TO_DATA_FRAME)
    df.to_csv(new_fn,index=False,compression='gzip')


# Convert Ground Truth Logs:

for fn in progress_bar(glob.glob('data/orig/unzipped/PDC2021_ground_truth/*')):
    new_fn=fn.split('.')[0]+'.csv.gz'
    new_fn='data/csv/'+"/".join(new_fn.split('/')[3:])
    log = xes_importer.apply(fn)
    df = log_converter.apply(log, variant=log_converter.Variants.TO_DATA_FRAME)
    df.to_csv(new_fn,index=False,compression='gzip')


# ## Datasets utilized in the BINET Paper

# +
import gzip
import json

def load_binet_data (path):
    with gzip.open(path, "r") as f:
        data = f.read()
        j = json.loads (data.decode('utf-8'))
        
    event_df = pd.DataFrame({'attributes': [], 'name': [], 'timestamp': [],'timestamp_end': [], 'event_id': [], 'trace_id': []})
    truth_df = pd.DataFrame({'case': [], 'anomalies': []})
    res=[]
    res=pd.DataFrame()
    for case in (j['cases']):
        trace = pd.DataFrame.from_dict(case['events'])
        trace['anomaly']= case['attributes']['label'] if type(case['attributes']['label'])==str else case['attributes']['label']['anomaly']
        trace['trace_id']=case['id']
        res=res.append(trace)
    res=pd.concat([res.drop(['attributes'], axis=1), res['attributes'].apply(pd.Series)], axis=1)
    return res


# -

import warnings
warnings.filterwarnings('ignore')

# Convert Synthethic Datasets and BPI Challenge Datasets, that already include artifical anomalies from BINET Paper

for fn in progress_bar((list(glob.glob('data/orig/unzipped/binet_logs/*-0.3-*')))):
    new_fn=fn[:-7]+'csv.gz'
    new_fn='data/csv/'+"/".join(new_fn.split('/')[3:])
    df=load_binet_data(fn)
    df.to_csv(new_fn,index=False,compression='gzip')
    

StarcoderdataJupyterScriptsDedupFiltered