<jupyter_start><jupyter_text>Automated ML<jupyter_code>COLAB = True
if COLAB:
  !sudo apt-get install git-lfs && git lfs install
  !rm -rf dl-projects
  !git clone https://github.com/mengwangk/dl-projects
  #!cd dl-projects && ls -l --block-size=M
if COLAB:
  !cp dl-projects/utils* .
  !cp dl-projects/preprocess* .
%reload_ext autoreload
%autoreload 2

%matplotlib inline
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import scipy.stats as ss
import math 
import matplotlib

from scipy import stats
from collections import Counter
from pathlib import Path

plt.style.use('fivethirtyeight')

sns.set(style="ticks")

# Automated feature engineering
import featuretools as ft

# Machine learning
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import Imputer, MinMaxScaler, StandardScaler
from sklearn.impute import SimpleImputer
from sklearn.metrics import precision_score, recall_score, f1_score, roc_auc_score, precision_recall_curve, roc_curve, mean_squared_error, accuracy_score
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.ensemble import RandomForestClassifier

from IPython.display import display

from utils import *
from preprocess import *

# The Answer to the Ultimate Question of Life, the Universe, and Everything.
np.random.seed(42)
%aimport<jupyter_output>Modules to reload:
all-except-skipped

Modules to skip:<jupyter_text>Preparation<jupyter_code>if COLAB:
  from google.colab import drive
  drive.mount('/content/gdrive')
  GDRIVE_DATASET_FOLDER = Path('gdrive/My Drive/datasets/')
if COLAB:
  DATASET_PATH = GDRIVE_DATASET_FOLDER
  ORIGIN_DATASET_PATH = Path('dl-projects/datasets')
else:
  DATASET_PATH = Path("datasets")
  ORIGIN_DATASET_PATH = Path('datasets')

DATASET = DATASET_PATH/"feature_matrix_2.csv"
ORIGIN_DATASET = ORIGIN_DATASET_PATH/'4D.zip'

if COLAB:
  !ls -l gdrive/"My Drive"/datasets/ --block-size=M
  !ls -l dl-projects/datasets --block-size=M
data = pd.read_csv(DATASET, header=0, sep=',', quotechar='"', parse_dates=['time'])
origin_data = format_tabular(ORIGIN_DATASET)
data.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 699972 entries, 0 to 699971
Columns: 123 entries, index to year
dtypes: bool(3), datetime64[ns](1), float64(36), int64(83)
memory usage: 642.8 MB<jupyter_text>Exploratory Data Analysis<jupyter_code>feature_matrix = data
feature_matrix.columns
feature_matrix.head(4).T
origin_data[origin_data['LuckyNo']==911].head(10)
# feature_matrix.groupby('time')['COUNT(Results)'].mean().plot()
# plt.title('Average Monthly Count of Results')
# plt.ylabel('Strike Per Number')<jupyter_output><empty_output><jupyter_text>Feature Selection<jupyter_code>from utils import feature_selection

%load_ext autoreload
%autoreload 2
feature_matrix_selection = feature_selection(feature_matrix.drop(columns = ['time', 'NumberId']))
feature_matrix_selection['time'] = feature_matrix['time']
feature_matrix_selection['NumberId'] = feature_matrix['NumberId']
feature_matrix_selection['Label'] = feature_matrix['Label']
feature_matrix_selection.columns<jupyter_output><empty_output><jupyter_text>Correlations<jupyter_code>feature_matrix_selection.shape
corrs = feature_matrix_selection.corr().sort_values('TotalStrike')
corrs['TotalStrike'].head()
corrs['Label'].dropna().tail(8)
corrs['TotalStrike'].dropna().tail(8)<jupyter_output><empty_output><jupyter_text>Visualization<jupyter_code>#pip install autoviz
#from autoviz.AutoViz_Class import AutoViz_Class<jupyter_output><empty_output><jupyter_text>XgBoost<jupyter_code>import xgboost as xgb
model = xgb.XGBClassifier()
def predict_dt(dt, feature_matrix, return_probs = False):

    feature_matrix['date'] = feature_matrix['time']

    # Subset labels
    test_labels = feature_matrix.loc[feature_matrix['date'] == dt, 'Label']
    train_labels = feature_matrix.loc[feature_matrix['date'] < dt, 'Label']

    print(f"Size of test labels {len(test_labels)}")
    print(f"Size of train labels {len(train_labels)}")
    
    # Features
    X_train = feature_matrix[feature_matrix['date'] < dt].drop(columns = ['NumberId', 'time',
                                                                                     'date', 'Label', 'TotalStrike', 'month', 'year', 'index'], errors='ignore')
    X_test = feature_matrix[feature_matrix['date'] == dt].drop(columns = ['NumberId', 'time',
                                                                                     'date', 'Label', 'TotalStrike', 'month', 'year', 'index'], errors='ignore')
    print(f"Size of X train {len(X_train)}")
    print(f"Size of X test  {len(X_test)}")
   
    feature_names = list(X_train.columns)
    
    # Impute and scale features
    pipeline = Pipeline([('imputer', SimpleImputer(strategy = 'median')), 
                      ('scaler', MinMaxScaler())])

    # Fit and transform training data
    X_train = pipeline.fit_transform(X_train)
    X_test = pipeline.transform(X_test)
    
    # Labels
    y_train = np.array(train_labels).reshape((-1, ))
    y_test = np.array(test_labels).reshape((-1, ))
    
    print('Training on {} observations.'.format(len(X_train)))
    print('Testing on {} observations.\n'.format(len(X_test)))
    
    # Train 
    model.fit(X_train, y_train)
    
    # Make predictions
    predictions = model.predict(X_test)
    probs = model.predict_proba(X_test)[:, 1]
    
    # Total positive
    positive = np.where((predictions==1))
    print('Total predicted to be positive: ', len(positive[0]))
  
    # Calculate metrics
    p = precision_score(y_test, predictions)
    r = recall_score(y_test, predictions)
    f = f1_score(y_test, predictions)
    auc = roc_auc_score(y_test, probs)
    a = accuracy_score(y_test, predictions)
    cm = confusion_matrix(y_test, predictions)
    
    print(f'Precision: {round(p, 5)}')
    print(f'Recall: {round(r, 5)}')
    print(f'F1 Score: {round(f, 5)}')
    print(f'ROC AUC: {round(auc, 5)}')
    print(f'Accuracy: {round(a, 5)}')

    #print('Probability')
    #print(len(probs), probs)

    # print('Probability >= Avg proba')
    # avg_p = np.average(probs)
    # print(f'Average probablity: {avg_p}')
    # hp = np.where((probs >= avg_p * 2) & (predictions==1) )
    # print(len(hp[0]), probs[hp[0]], hp[0])

    print('Confusion matrix')
    print(cm)

    # Total predicted matches
    print('Predicted matches')
    m = np.where((predictions==1))
    print(len(m[0]), m)

    if len(positive[0]) > 0:
      # Matching draws
      print('Matched draws')
      m = np.where((predictions==1) & (y_test == 1))
      print(len(m[0]), m)
      data = feature_matrix.loc[feature_matrix['date'] == dt]
      display(data.iloc[m[0]][
                              ['NumberId', 'Label', 'month', 'MODE(Results.PrizeType)_1stPrizeNo',
                                'MODE(Results.PrizeType)_2ndPrizeNo',
                                'MODE(Results.PrizeType)_3rdPrizeNo',
                                'MODE(Results.PrizeType)_ConsolationNo1',
                                'MODE(Results.PrizeType)_ConsolationNo10',
                                'MODE(Results.PrizeType)_ConsolationNo2',
                                'MODE(Results.PrizeType)_ConsolationNo3',
                                'MODE(Results.PrizeType)_ConsolationNo4',
                                'MODE(Results.PrizeType)_ConsolationNo5',
                                'MODE(Results.PrizeType)_ConsolationNo6',
                                'MODE(Results.PrizeType)_ConsolationNo7',
                                'MODE(Results.PrizeType)_ConsolationNo8',
                                'MODE(Results.PrizeType)_ConsolationNo9',
                                'MODE(Results.PrizeType)_SpecialNo1',
                                'MODE(Results.PrizeType)_SpecialNo10',
                                'MODE(Results.PrizeType)_SpecialNo2',
                                'MODE(Results.PrizeType)_SpecialNo3',
                                'MODE(Results.PrizeType)_SpecialNo4',
                                'MODE(Results.PrizeType)_SpecialNo5',
                                'MODE(Results.PrizeType)_SpecialNo6',
                                'MODE(Results.PrizeType)_SpecialNo7',
                                'MODE(Results.PrizeType)_SpecialNo8',
                                'MODE(Results.PrizeType)_SpecialNo9']].T)   
    else:
      print('No luck this month')                 

    # Feature importances
    fi = pd.DataFrame({'feature': feature_names, 'importance': model.feature_importances_})
    
    if return_probs:
        return fi, probs
    
    return fi
    
# All the months
len(feature_matrix_selection['time'].unique()), feature_matrix_selection['time'].unique()<jupyter_output><empty_output><jupyter_text>Prediction by months<jupyter_code>from utils import plot_feature_importances
%time oct_2018 = predict_dt(pd.datetime(2018,10,1), feature_matrix_selection)
norm_oct_2018_fi = plot_feature_importances(oct_2018)
%time may_2019 = predict_dt(pd.datetime(2019,5,1), feature_matrix_selection)
norm_may_2019_fi = plot_feature_importances(may_2019)
 %time june_2019 = predict_dt(pd.datetime(2019,6,1), feature_matrix_selection)
norm_june_2019_fi = plot_feature_importances(june_2019)
%time july_2019 = predict_dt(pd.datetime(2019,7,1), feature_matrix_selection)
norm_july_2019_fi = plot_feature_importances(july_2019)
%time aug_2019 = predict_dt(pd.datetime(2019,8,1), feature_matrix_selection)
norm_aug_2019_fi = plot_feature_importances(aug_2019)
%time oct_2019 = predict_dt(pd.datetime(2019,10,1), feature_matrix_selection)
norm_oct_2019_fi = plot_feature_importances(oct_2019)
%time sep_2019 = predict_dt(pd.datetime(2019,9,1), feature_matrix_selection)<jupyter_output>Size of test labels 10000
Size of train labels 679972
Size of X train 679972
Size of X test  10000
Training on 679972 observations.
Testing on 10000 observations.

Total predicted to be positive:  0
Precision: 0.0
Recall: 0.0
F1 Score: 0.0
ROC AUC: 0.54424
Accuracy: 0.9684
Confusion matrix
[[9684    0]
 [ 316    0]]
Predicted matches
0 (array([], dtype=int64),)
No luck this month
CPU times: user 2min 12s, sys: 499 ms, total: 2min 13s
Wall time: 2min 13s<jupyter_text>Tuning - GridSearchCV  Check Raw Data<jupyter_code>origin_data.tail(10)
origin_data[(origin_data['DrawDate'].dt.year == 2019) & (origin_data['DrawDate'].dt.month == 6)]['DrawNo'].nunique()
origin_data[(origin_data['DrawDate'].dt.year == 2019) & (origin_data['DrawDate'].dt.month == 10)]['DrawNo'].nunique()
print(15 * 45 + 14 * 45)<jupyter_output>1305<jupyter_text>Testing<jupyter_code>import numpy as np
import pandas as pd

data = [['no_1', 1], ['no_2', 2], ['no_3', 3], ['no_4', 4], ['no_5', 5], ['no_6', 6], ['no_7', 7]] 
  
# Create the pandas DataFrame 
df = pd.DataFrame(data, columns = ['Name', 'Age']) 

a = np.array([0,0,0,1,0,1, 1])
b = np.array([0,0,0,1,0,0, 1])

print(len(a))
m = np.where((a==1) & (b ==1))
print(len(m[0]), m[0], a[m[0]])
print(df.iloc[m[0]])
probs = np.array([0.03399902, 0.03295987, 0.03078781, 0.04921166, 0.03662422, 0.03233755])
print(np.average(probs))
mydict = [{'a': 1, 'b': 2, 'c': 3, 'd': 4},
         {'a': 100, 'b': 200, 'c': 300, 'd': 400},
         {'a': 1000, 'b': 2000, 'c': 3000, 'd': 4000 }]
df = pd.DataFrame(mydict)
df.iloc[[0]][['a','b']]<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup