# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: conda_python3
#     language: python
#     name: conda_python3
# ---

# Let's begin with import of required packages

# +
#CONFIGURATION STEP

import numpy as np                                # For matrix operations and numerical processing
import pandas as pd                               # For munging tabular data
import matplotlib.pyplot as plt                   # For charts and visualizations
from IPython.display import Image                 # For displaying images in the notebook
from IPython.display import display               # For displaying outputs in the notebook
from time import gmtime, strftime                 # For labeling SageMaker models, endpoints, etc.
import sys                                        # For writing outputs to notebook
import math                                       # For ceiling function
import json                                       # For parsing hosting outputs
import os                                         # For manipulating filepath names
import sagemaker                                  # Amazon SageMaker's Python SDK provides many helper functions
from sagemaker.predictor import csv_serializer    # Converts strings for HTTP POST requests on inference
import boto3
import re
from sagemaker import get_execution_role
# -

# Get an read HIGGS dataset. Split it to train/test

# +
# !wget https://s3.amazonaws.com/h2o-public-test-data/bigdata/laptop/higgs_head_2M.csv
    
dataset = pd.read_csv('./higgs_head_2M.csv',header=None)
print(dataset.head())

train = dataset.head(1000000)
test = dataset.tail(1000000)
# -

# !rm train.csv
# !rm validation.csv

# Load train/test data into separate CSV files

train.to_csv('higgs_train.csv', index=None, header=None)
test.to_csv('higgs_validation.csv', index=None, header=None)

# Upload files to S3

# +
bucket = 'okremnyo'
prefix = 'sagemaker/DEMO-xgboost-dm'

boto3.Session().resource('s3').Bucket(bucket).Object(os.path.join(prefix, 'higgs_train/train.csv')).upload_file('higgs_train.csv')
boto3.Session().resource('s3').Bucket(bucket).Object(os.path.join(prefix, 'higgs_validation/validation.csv')).upload_file('higgs_validation.csv')

# +
from sagemaker.amazon.amazon_estimator import get_image_uri
container = get_image_uri(boto3.Session().region_name, 'xgboost')

s3_input_train = sagemaker.s3_input(s3_data='s3://{}/{}/higgs_train'.format(bucket, prefix), content_type='csv')
s3_input_validation = sagemaker.s3_input(s3_data='s3://{}/{}/higgs_validation/'.format(bucket, prefix), content_type='csv')
# -

# Run the SageMaker Xgboost algorithm

# +
sess = sagemaker.Session()
role = get_execution_role()

xgb = sagemaker.estimator.Estimator(container,
                                    role, 
                                    train_instance_count=1, 
                                    train_instance_type='ml.c5.18xlarge',
                                    output_path='s3://{}/{}/output'.format(bucket, prefix),
                                    sagemaker_session=sess)
xgb.set_hyperparameters(max_depth=6,
                        eta=0.3,
                        silent=0,
                        objective='binary:logistic',
                        num_round=50)

xgb.fit({'train': s3_input_train})
# -

# Run the prediction

xgb_predictor = xgb.deploy(initial_instance_count=1,
                           instance_type='ml.m4.xlarge')

xgb_predictor.content_type = 'text/csv'
xgb_predictor.serializer = csv_serializer


# Get the predictions

# +
def predict(data, rows=500):
    split_array = np.array_split(data, int(data.shape[0] / float(rows) + 1))
    predictions = ''
    for array in split_array:
        predictions = ','.join([predictions, xgb_predictor.predict(array).decode('utf-8')])

    return np.fromstring(predictions[1:], sep=',')

predictions = predict(test.drop(0, axis=1).as_matrix())
# -

# Get the SageMaker Xgboost classification report

# +
from sklearn.metrics import accuracy_score
print('Accuracy = ', accuracy_score(test[0], np.round(predictions)))

from sklearn.metrics import classification_report
print(classification_report(test[0], np.round(predictions)))

StarcoderdataJupyterScriptsDedupFiltered