# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# ## Iris Monitoring Example

# In this example notebook, we utilize iris dataset, trained with `sklearn` and `tensorflow` models, upload outputs to S3 bucket. And then kubeflow artifact can read from S3 bucket and show us the monitoring results including `Markdown`, `Confusion Matrix`, `ROC Curve` and `Tensorboard`.

# ### Prerequiste
# 1. Create a `aws-secret` with at least `S3ReadOnlyAccess` policy in `kubeflow` namespace

# ```yaml
#   apiVersion: v1
#   kind: Secret
#   metadata:
#     name: aws-secret
#     namespace: kubeflow
#   type: Opaque
#   data:
#     AWS_ACCESS_KEY_ID: YOUR_BASE64_ACCESS_KEY
#     AWS_SECRET_ACCESS_KEY: YOUR_BASE64_SECRET_ACCESS
# ```

# Note: To get base64 string, run `echo -n $AWS_ACCESS_KEY_ID | base64`

# 2. Create a configmap to store template spec of viewer tensorboard pod. Replace `<your_region>` with your S3 region.

# ```yaml
#   apiVersion: v1
#   kind: ConfigMap
#   metadata:
#     name: ml-pipeline-ui-viewer-template
#   data:
#     viewer-tensorboard-template.json: |
#       {
#           "spec": {
#               "containers": [
#                   {
#                       "env": [
#                           {
#                               "name": "AWS_ACCESS_KEY_ID",
#                               "valueFrom": {
#                                   "secretKeyRef": {
#                                       "name": "aws-secret",
#                                       "key": "AWS_ACCESS_KEY_ID"
#                                   }
#                               }
#                           },
#                           {
#                               "name": "AWS_SECRET_ACCESS_KEY",
#                               "valueFrom": {
#                                   "secretKeyRef": {
#                                       "name": "aws-secret",
#                                       "key": "AWS_SECRET_ACCESS_KEY"
#                                   }
#                               }
#                           },
#                           {
#                               "name": "AWS_REGION",
#                               "value": "<your_region>"
#                           }
#                       ]
#                   }
#               ]
#           }
#       }
# ```

# 3. Update deployment `ml-pipeline-ui` to use this sepc by running `kubectl edit deployment ml-pipeline-ui -n kubeflow`.

# ```yaml
#
#   apiVersion: extensions/v1beta1
#   kind: Deployment
#   metadata:
#     name: ml-pipeline-ui
#     namespace: kubeflow
#     ...
#   spec:
#     template:
#       spec:
#         containers:
#         - env:
#           - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH
#             value: /etc/config/viewer-tensorboard-template.json
#           ....
#           volumeMounts:
#           - mountPath: /etc/config
#             name: config-volume
#         .....
#         volumes:
#         - configMap:
#             defaultMode: 420
#             name: ml-pipeline-ui-viewer-template
#           name: config-volume
# ```

# 4. Create an S3 bucket to store pipeline data

import random, string
HASH = ''.join([random.choice(string.ascii_lowercase) for n in range(16)] + [random.choice(string.digits) for n in range(16)])
AWS_REGION = 'us-west-2'
S3_BUCKET = '{}-kubeflow-pipeline-data'.format(HASH)
# !aws s3 mb s3://$S3_BUCKET --region $AWS_REGION

# ### Build Pipeline

# 1. Run the following command to load Kubeflow Pipelines SDK

import kfp
from kfp import components
from kfp import dsl
from kfp.aws import use_aws_secret

# 2. Load reusable components

iris_op = components.load_component_from_file('./src/pipeline_artifact_monitoring/component.yaml')


@dsl.pipeline(
    name='IRIS Classification pipeline',
    description='IRIS Classification using LR in SKLEARN'
)
def iris_pipeline(s3_bucket=S3_BUCKET,
                 aws_region=AWS_REGION):
    
    iris_task = iris_op(
        s3_bucket=s3_bucket,
        aws_region=aws_region,
    ).apply(use_aws_secret('aws-secret', 'AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY'))


# 3. Compile and deploy your pipeline

kfp.compiler.Compiler().compile(iris_pipeline, 'iris-classification-pipeline.zip')
client = kfp.Client()
aws_experiment = client.create_experiment(name='aws')
my_run = client.run_pipeline(aws_experiment.id, 'iris-classification-pipeline', 
  'iris-classification-pipeline.zip')

# Note: After you saw the artifact with Tensorboard, and create a tensorboard 1.15.0, please wait 3-5 minutes for tensorboard to load data from S3

# ![tensorboard-artifact](./images/tbartifact.png)

# ### Clean Up

# Delete S3 bucket that was created for this exercise

# !aws s3 rb s3://$S3_BUCKET --force

StarcoderdataJupyterScriptsDedupFiltered