<jupyter_start><jupyter_text>Transformers<jupyter_code># https://zablo.net/blog/post/pandas-dataframe-in-scikit-learn-feature-union
from scipy import sparse
from sklearn.externals.joblib import Parallel, delayed
from sklearn.pipeline import FeatureUnion, _fit_transform_one, _transform_one


class PandasFeatureUnion(FeatureUnion):
    def fit_transform(self, X, y=None, **fit_params):
        self._validate_transformers()
        result = Parallel(n_jobs=self.n_jobs)(
            delayed(_fit_transform_one)(trans,
                                        weight,
                                        X,
                                        y,
                                        **fit_params) for name,
            trans,
            weight in self._iter()
        )

        if not result:
            # All transformers are None
            return np.zeros((X.shape[0], 0))
        Xs, transformers = zip(*result)
        self._update_transformer_list(transformers)
        if any(sparse.issparse(f) for f in Xs):
            Xs = sparse.hstack(Xs).tocsr()
        else:
            Xs = self.merge_dataframes_by_column(Xs)
        return Xs

    def merge_dataframes_by_column(self, Xs):
        return pd.concat(Xs, axis="columns", copy=False)

    def transform(self, X):
        Xs = Parallel(n_jobs=self.n_jobs)(
            delayed(_transform_one)(trans,
                                    weight,
                                    X) for name,
            trans,
            weight in self._iter()
        )
        if not Xs:
            # All transformers are None
            return np.zeros((X.shape[0], 0))
        if any(sparse.issparse(f) for f in Xs):
            Xs = sparse.hstack(Xs).tocsr()
        else:
            Xs = self.merge_dataframes_by_column(Xs)
        return Xs
class NoFitMixin():
    def fit(self, X, y=None):
        return self


class MICEImputer(BaseEstimator, TransformerMixin, NoFitMixin):
    def transform(self, X, *args, **kwargs):
        """
            Fill a 1-D array missing values with MICE
        """
        assert isinstance(X, pd.Series)
        X = X.copy()    # Should avoid error of already full for repeat execution
        has_null = X.isnull().any()    # TODO: Used to avoid error of no null values from MICE
        if has_null:
            mice = MICE(verbose=False, *args, **kwargs)
            imputed = mice.complete(X.values.reshape(-1, 1))
            X.loc[:] = imputed.reshape(X.loc[:].shape)
            return pd.DataFrame(X)
        else:
            return X


class ColumnSelector(BaseEstimator, TransformerMixin, NoFitMixin):
    def __init__(self, columns, one_col=True):
        self.columns = columns
        self.one_col = one_col

    def transform(self, X, y=None):
        if self.one_col:
            return X[self.columns].iloc[:, 0]
        else:
            return X[self.columns]


class ColumnDummifier(BaseEstimator, TransformerMixin, NoFitMixin):
    def transform(self, X, y=None):
        return pd.get_dummies(X, sparse=True, drop_first=True)


class LabelEncoderWNaN(TransformerMixin, BaseEstimator):
    """
    Applies the sklearn.LabelEncoder while keeping missing values
    """

    def fit(self, X, y=None):
        self.le_ = LabelEncoder()
        self.le_.fit(X.loc[X.notnull()])
        return self

    def transform(self, X, y=None):
        X = X.copy(deep=True)    # Do not apply tranform to the actual DF
        X.loc[X.notnull()] = self.le_.transform(X.loc[X.notnull()])
        return X.astype("float")<jupyter_output><empty_output><jupyter_text>Explore<jupyter_code>train_x = train.drop(columns=["survived"])
train_y = train.survived
benchmark_pipeline = Pipeline([(
    "prep",
    PandasFeatureUnion([
        ("age",
         make_pipeline(ColumnSelector(["age"]),
                       MICEImputer())),
        ("sex_dummy",
         make_pipeline(ColumnSelector(["sex"]),
                       ColumnDummifier())),
        (
            "embarked",
            make_pipeline(ColumnSelector(["embarked"]),
                          LabelEncoderWNaN(),
                          MICEImputer(),
                          ColumnDummifier())
        ),
        ("fare",
         make_pipeline(ColumnSelector(["fare"]),
                       MICEImputer())),
        ("rest",
         make_pipeline(ColumnSelector(["parch",
                                       "sibsp",
                                       "pclass"],
                                      one_col=False)))
    ])
)])
train.columns<jupyter_output><empty_output><jupyter_text>Benchmark model<jupyter_code>lr_pipeline = make_pipeline(benchmark_pipeline, LogisticRegression())
lr_pipeline.fit(train, train_y)
accuracy_score(train_y, lr_pipeline.predict(train_x))
validate = pd.read_excel("/home/iyed/Downloads/titanic3.xls")
accuracy_score(validate.survived, lr_pipeline.predict(validate))<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup