<jupyter_start><jupyter_text><jupyter_code>!pip install category_encoders
!wget https://archive.ics.uci.edu/ml/machine-learning-databases/00222/bank-additional.zip
!unzip bank-additional.zip
import pandas as pd

df = pd.read_csv("bank-additional/bank-additional-full.csv", delimiter=";")

y_col = "y"

train_ratio = 0.8
val_ratio = 0.1

shuffle = True
override_model = None
override_params = {}
override_scorer = None
from sklearn.linear_model import LogisticRegression
from sklearn.linear_model import LinearRegression

from sklearn.metrics import accuracy_score
from sklearn.metrics import r2_score

model = None
params = {}
scorer = None

if not override_model:
  if df[y_col].dtype == 'object':
    if len(df[y_col].astype('category').cat.categories) < 3 and df[y_col].isna().sum() == 0:
      m = {x: i for i,x in enumerate(df[y_col].astype('category').cat.categories)}
      n_col = "{}:{}".format(y_col, df[y_col].astype('category').cat.categories[-1])
      df[n_col] = df[y_col].map(m)
      df = df.drop(columns=[y_col], axis=1)
      y_col = [n_col]

      model = LogisticRegression
      params = {'solver': 'lbfgs'}
      scorer = accuracy_score
    else:
      dummy = pd.get_dummies(df[y_col])
      Y = []
      for v in dummy:
        df["{}:{}".format(y_col,v)] = dummy[v]
        Y.append("{}:{}".format(y_col,v))
      df.drop(columns=[y_col], axis=1)
      y_col = Y

      model = LogisticRegression
      params = {'solver': 'lbfgs', 'multi_class': 'multinomial'}
      scorer = accuracy_score
  else:
    model = LinearRegression
    params = {}
    scorer = r2_score
else:
  model = override_model
  
if override_params != {}:
  params = override_params
if override_scorer != None:
  scorer = override_scorer
from sklearn.model_selection import train_test_split

cols = [x for x in df if x not in y_col]

X_train = None
X_val = None
X_test = None
y_train = None
y_val = None
y_test = None

if train_ratio < 1.0:
  if cvs > 0:
    X_train, X_test, y_train, y_test = train_test_split(df[cols], df[y_col], train_size=train_ratio, shuffle=shuffle)
  elif val_ratio > 0.0:
    X_train, X_test, y_train, y_test = train_test_split(df[cols], df[y_col], train_size=train_ratio+val_ratio, shuffle=shuffle)
    X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, train_size=train_ratio/(train_ratio+val_ratio), shuffle=shuffle)
  else:
    X_train, X_test, y_train, y_test = train_test_split(df[cols], df[y_col], train_size=train_ratio, shuffle=shuffle)
    X_val = X_test.copy()
    y_val = y_test.copy()
else:
  X_val = X_train.copy()
  y_val = y_train.copy()
  X_test = X_train.copy()
  y_test = y_train.copy()
import numpy as np

from sklearn.metrics import mean_squared_error
from scipy import stats

def calc_p(pipeline, X, y, y_pred):
  for i in range(y.shape[1]):
    params = np.append(pipeline.named_steps['model'].intercept_[i],pipeline.named_steps['model'].coef_[i])

    colX = pipeline.named_steps['drop'].transform(pipeline.named_steps['encode'].transform(X))
    newX = pd.DataFrame({"Constant":np.ones(len(X))}).join(pd.DataFrame(pipeline.named_steps['transform'].transform(colX)))

    var_b = mean_squared_error(y, y_pred)*(np.linalg.pinv(np.dot(newX.T,newX)).diagonal())
    sd_b = np.sqrt(var_b)
    ts_b = params/ sd_b

    return pd.Series([2*(1-stats.t.cdf(np.abs(i),(len(newX)-1))) for i in ts_b], index=["Intercept"]+[x for x in colX])
class DropFeatures(object):
  def __init__(self, cols) :
    self.cols = cols
  
  def set_params(self, cols):
    self.cols = cols
  
  def transform(self, x) :
    dat = x.copy()
    return dat[[i for i in x if i not in self.cols]]

  def fit(self, x, y=None) :
    return self
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
import category_encoders as ce

import warnings
warnings.filterwarnings("ignore")

remove_cols = []
pipeline = Pipeline([('encode', ce.OneHotEncoder(use_cat_names=True)),
                     ('drop', DropFeatures(remove_cols)),
                        ('transform', StandardScaler()),
                        ('model', model(**params))])

while True:
  pipeline.fit(X_train, y_train)
  y_pred = pipeline.predict(X_test)
  print(scorer(y_test, y_pred))
  p_vals = calc_p(pipeline, X_test, y_test, pipeline.predict(X_test))
  rem = [p_vals.index[i] for i in range(len(p_vals)) if p_vals.index[i] != "Intercept" and (p_vals[i] > 0.05 or p_vals.isna()[i])]
  if len(rem) == 0:
    break
  remove_cols += rem
  pipeline.set_params(drop__cols=remove_cols)
  

coef_table = None
for i in range(len(y_col)):
  if i == 0:
    coef_table = p_vals.to_frame(name="p_{}".format(i))
  else:
    coef_table = coef_table.join(p_vals.to_frame(name="p_{}".format(i)))
  coef_table = coef_table.join(pd.Series(np.append(pipeline.named_steps['model'].intercept_[i], pipeline.named_steps['model'].coef_[i]), index=["Intercept"]+[x for x in pipeline.named_steps['drop'].transform(pipeline.named_steps['encode'].transform(X_test))]).to_frame(name="Coefficient_{}".format(i)))
  coef_table["abs(Coefficient_{})".format(i)] = coef_table["Coefficient_{}".format(i)].abs()
coef_table = coef_table.sort_values(by=["p_0", "abs(Coefficient_0)"], ascending=[False, True])
with pd.option_context('display.max_rows', None, 'display.max_columns', None):
    print(coef_table)<jupyter_output><empty_output>