Python ML 7 - Combining Different Models for Ensemble Learning

-- Learning with ensembles

from scipy.misc import comb
import math
def ensemble_error(n_classifier, error):
     k_start = math.ceil(n_classifier / 2.0)
     probs = [comb(n_classifier, k) *
              error**k *
              (1-error)**(n_classifier - k)
              for k in range(k_start, n_classifier + 1)]
     return sum(probs)
ensemble_error(n_classifier=11, error=0.25)

import numpy as np
error_range = np.arange(0.0, 1.01, 0.01)
ens_errors = [ensemble_error(n_classifier=11, error=error)
               for error in error_range]
import matplotlib.pyplot as plt
plt.plot(error_range, ens_errors,
          label='Ensemble error',
          linewidth=2)
plt.plot(error_range, error_range,
          linestyle='--', label='Base error',
          linewidth=2)
plt.xlabel('Base error')
plt.ylabel('Base/Ensemble error')
plt.legend(loc='upper left')
plt.grid()
plt.show()

-- Implementing a simple majority vote classifier

import numpy as np
np.argmax(np.bincount([0, 0, 1],
           weights=[0.2, 0.2, 0.6]))

ex = np.array([[0.9, 0.1],
                [0.8, 0.2],
                [0.4, 0.6]])
p = np.average(ex, axis=0, weights=[0.2, 0.2, 0.6])
p
np.argmax(p)

ex = np.array([[0.9, 0.1],
                [0.8, 0.2],
                [0.4, 0.6]])
p = np.average(ex, axis=0, weights=[0.2, 0.2, 0.6])
p
np.argmax(p)

from sklearn.base import BaseEstimator
from sklearn.base import ClassifierMixin
from sklearn.preprocessing import LabelEncoder
from sklearn.externals import six
from sklearn.base import clone
from sklearn.pipeline import _name_estimators
import numpy as np
import operator


class MajorityVoteClassifier(BaseEstimator,
                             ClassifierMixin):
    """ A majority vote ensemble classifier

    Parameters
    ----------
    classifiers : array-like, shape = [n_classifiers]
      Different classifiers for the ensemble

    vote : str, {'classlabel', 'probability'}
      Default: 'classlabel'
      If 'classlabel' the prediction is based on
      the argmax of class labels. Else if
      'probability', the argmax of the sum of
      probabilities is used to predict the class label
      (recommended for calibrated classifiers).

    weights : array-like, shape = [n_classifiers]
      Optional, default: None
      If a list of `int` or `float` values are
      provided, the classifiers are weighted by
      importance; Uses uniform weights if `weights=None`.

    """
    def __init__(self, classifiers,
                 vote='classlabel', weights=None):

        self.classifiers = classifiers
        self.named_classifiers = {key: value for
                                  key, value in
                                  _name_estimators(classifiers)}
        self.vote = vote
        self.weights = weights

    def fit(self, X, y):
        """ Fit classifiers.

        Parameters
        ----------
        X : {array-like, sparse matrix},
            shape = [n_samples, n_features]
            Matrix of training samples.

        y : array-like, shape = [n_samples]
            Vector of target class labels.

        Returns
        -------
        self : object

        """
        # Use LabelEncoder to ensure class labels start
        # with 0, which is important for np.argmax
        # call in self.predict
        self.lablenc_ = LabelEncoder()
        self.lablenc_.fit(y)
        self.classes_ = self.lablenc_.classes_
        self.classifiers_ = []
        for clf in self.classifiers:
            fitted_clf = clone(clf).fit(X,
                              self.lablenc_.transform(y))
            self.classifiers_.append(fitted_clf)
        return self


def predict(self, X):
        """ Predict class labels for X.

        Parameters
        ----------
        X : {array-like, sparse matrix},
            Shape = [n_samples, n_features]
            Matrix of training samples.

        Returns
        ----------
        maj_vote : array-like, shape = [n_samples]
            Predicted class labels.

        """
        if self.vote == 'probability':
            maj_vote = np.argmax(self.predict_proba(X),
                                 axis=1)
        else:  # 'classlabel' vote

            #  Collect results from clf.predict calls
            predictions = np.asarray([clf.predict(X)
                                      for clf in
                                      self.classifiers_]).T

            maj_vote = np.apply_along_axis(
                           lambda x:
                           np.argmax(np.bincount(x,                                          
                                        weights=self.weights)),
                           axis=1,
                           arr=predictions)
        maj_vote = self.lablenc_.inverse_transform(maj_vote)
        return maj_vote

    def predict_proba(self, X):
        """ Predict class probabilities for X.

        Parameters
        ----------
        X : {array-like, sparse matrix},
            shape = [n_samples, n_features]
            Training vectors, where n_samples is
            the number of samples and
            n_features is the number of features.

        Returns
        ----------
        avg_proba : array-like,
            shape = [n_samples, n_classes]
            Weighted average probability for
            each class per sample.

        """
        probas = np.asarray([clf.predict_proba(X)
                             for clf in self.classifiers_])
        avg_proba = np.average(probas,
                               axis=0, weights=self.weights)
        return avg_proba

    def get_params(self, deep=True):
        """ Get classifier parameter names for GridSearch"""
        if not deep:
            return super(MajorityVoteClassifier,
                         self).get_params(deep=False)
        else:
            out = self.named_classifiers.copy()
            for name, step in\
                    six.iteritems(self.named_classifiers):
                for key, value in six.iteritems(
                        step.get_params(deep=True)):
                    out['%s__%s' % (name, key)] = value
            return out

Combining different algorithms for classification with majority vote

from sklearn import datasets
from sklearn.cross_validation import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import LabelEncoder
iris = datasets.load_iris()
X, y = iris.data[50:, [1, 2]], iris.target[50:]
le = LabelEncoder()
y = le.fit_transform(y)

X_train, X_test, y_train, y_test =\
        train_test_split(X, y,
                         test_size=0.5,
                         random_state=1)

from sklearn.cross_validation import cross_val_score
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.pipeline import Pipeline
import numpy as np
clf1 = LogisticRegression(penalty='l2',
                           C=0.001,
                           random_state=0)
clf2 = DecisionTreeClassifier(max_depth=1,
                               criterion='entropy',
                               random_state=0)
clf3 = KNeighborsClassifier(n_neighbors=1,
                             p=2,
                             metric='minkowski')
pipe1 = Pipeline([['sc', StandardScaler()],
                   ['clf', clf1]])
pipe3 = Pipeline([['sc', StandardScaler()],
                   ['clf', clf3]])
clf_labels = ['Logistic Regression', 'Decision Tree', 'KNN']
print('10-fold cross validation:\n')
for clf, label in zip([pipe1, clf2, pipe3], clf_labels):
     scores = cross_val_score(estimator=clf,
                              X=X_train,
                              y=y_train,
                              cv=10,
                              scoring='roc_auc')
     print("ROC AUC: %0.2f (+/- %0.2f) [%s]"
                % (scores.mean(), scores.std(), label))


mv_clf = MajorityVoteClassifier(
                 classifiers=[pipe1, clf2, pipe3])
clf_labels += ['Majority Voting']
all_clf = [pipe1, clf2, pipe3, mv_clf]
for clf, label in zip(all_clf, clf_labels):
     scores = cross_val_score(estimator=clf,
                              X=X_train,
                              y=y_train,
                              cv=10,
                              scoring='roc_auc')
     print("Accuracy: %0.2f (+/- %0.2f) [%s]"
                % (scores.mean(), scores.std(), label))


-- Evaluating and tuning the ensemble classifier

from sklearn.metrics import roc_curve
from sklearn.metrics import auc
colors = ['black', 'orange', 'blue', 'green']
linestyles = [':', '--', '-.', '-']
for clf, label, clr, ls \
         in zip(all_clf, clf_labels, colors, linestyles):
     # assuming the label of the positive class is 1
     y_pred = clf.fit(X_train,
                      y_train).predict_proba(X_test)[:, 1]
     fpr, tpr, thresholds = roc_curve(y_true=y_test,
                                      y_score=y_pred)
     roc_auc = auc(x=fpr, y=tpr)
     plt.plot(fpr, tpr,
              color=clr,
              linestyle=ls,
              label='%s (auc = %0.2f)' % (label, roc_auc))
plt.legend(loc='lower right')
plt.plot([0, 1], [0, 1],
          linestyle='--',
          color='gray',
          linewidth=2)
plt.xlim([-0.1, 1.1])
plt.ylim([-0.1, 1.1])
plt.grid()
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.show()

sc = StandardScaler()
X_train_std = sc.fit_transform(X_train)
from itertools import product
x_min = X_train_std[:, 0].min() - 1
x_max = X_train_std[:, 0].max() + 1
y_min = X_train_std[:, 1].min() - 1
y_max = X_train_std[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, 0.1),
                      np.arange(y_min, y_max, 0.1))
f, axarr = plt.subplots(nrows=2, ncols=2,
                         sharex='col',
                         sharey='row',
                         figsize=(7, 5))
for idx, clf, tt in zip(product([0, 1], [0, 1]),
                         all_clf, clf_labels):
     clf.fit(X_train_std, y_train)
     Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])
     Z = Z.reshape(xx.shape)
     axarr[idx[0], idx[1]].contourf(xx, yy, Z, alpha=0.3)  
     axarr[idx[0], idx[1]].scatter(X_train_std[y_train==0, 0],
                                   X_train_std[y_train==0, 1],
                                   c='blue',
                                   marker='^',
                                   s=50)  
     axarr[idx[0], idx[1]].scatter(X_train_std[y_train==1, 0],
                                   X_train_std[y_train==1, 1],
                                   c='red',
                                   marker='o',
                                   s=50)
     axarr[idx[0], idx[1]].set_title(tt)
plt.text(-3.5, -4.5,
          s='Sepal width [standardized]',
          ha='center', va='center', fontsize=12)
plt.text(-10.5, 4.5,
          s='Petal length [standardized]',
          ha='center', va='center',
          fontsize=12, rotation=90)
plt.show()

how to access the individual parameters inside a Grid Search Object:
mv_clf.get_params()

from sklearn.grid_search import GridSearchCV
params = {'decisiontreeclassifier__max_depth': [1, 2],
           'pipeline-1__clf__C': [0.001, 0.1, 100.0]}
grid = GridSearchCV(estimator=mv_clf,
                     param_grid=params,
                     cv=10,
                     scoring='roc_auc')
grid.fit(X_train, y_train)

for params, mean_score, scores in grid.grid_scores_:
     print("%0.3f+/-%0.2f %r"
            % (mean_score, scores.std() / 2, params))

print('Best parameters: %s' % grid.best_params_)

print('Accuracy: %.2f' % grid.best_score_)

-- Bagging - building an ensemble of classifier from bootstrap samples

import pandas as pd
df_wine = pd.read_csv('https://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.data', header=None)
df_wine.columns = ['Class label', 'Alcohol',
                    'Malic acid', 'Ash',
                    'Alcalinity of ash',
                    'Magnesium', 'Total phenols',
                    'Flavanoids', 'Nonflavanoid phenols',
                    'Proanthocyanins',
                    'Color intensity', 'Hue',
                    'OD280/OD315 of diluted wines',
                    'Proline']
df_wine = df_wine[df_wine['Class label'] != 1]
y = df_wine['Class label'].values
X = df_wine[['Alcohol', 'Hue']].values

from sklearn.preprocessing import LabelEncoder
from sklearn.cross_validation import train_test_split
le = LabelEncoder()
y = le.fit_transform(y)
X_train, X_test, y_train, y_test =\
            train_test_split(X, y,
                             test_size=0.40,
                             random_state=1)

from sklearn.ensemble import BaggingClassifier
tree = DecisionTreeClassifier(criterion='entropy',
                               max_depth=None)
bag = BaggingClassifier(base_estimator=tree,
                         n_estimators=500,
                         max_samples=1.0,
                         max_features=1.0,
                         bootstrap=True,
                         bootstrap_features=False,
                         n_jobs=1,
                         random_state=1)

compare the performance of the bagging classifier to the performance of a single unpruned decision tree
from sklearn.metrics import accuracy_score
tree = tree.fit(X_train, y_train)
y_train_pred = tree.predict(X_train)
y_test_pred = tree.predict(X_test)
tree_train = accuracy_score(y_train, y_train_pred)
tree_test = accuracy_score(y_test, y_test_pred)
print('Decision tree train/test accuracies %.3f/%.3f'
        % (tree_train, tree_test))

bag = bag.fit(X_train, y_train)
y_train_pred = bag.predict(X_train)
y_test_pred = bag.predict(X_test)
bag_train = accuracy_score(y_train, y_train_pred)
bag_test = accuracy_score(y_test, y_test_pred)
print('Bagging train/test accuracies %.3f/%.3f'
        % (bag_train, bag_test))

x_min = X_train[:, 0].min() - 1
x_max = X_train[:, 0].max() + 1
y_min = X_train[:, 1].min() - 1
y_max = X_train[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, 0.1),
                      np.arange(y_min, y_max, 0.1))
f, axarr = plt.subplots(nrows=1, ncols=2,
                         sharex='col',
                         sharey='row',
                         figsize=(8, 3))
for idx, clf, tt in zip([0, 1],
                         [tree, bag],
                         ['Decision Tree', 'Bagging']):
     clf.fit(X_train, y_train)
   
     Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])
     Z = Z.reshape(xx.shape)
     axarr[idx].contourf(xx, yy, Z, alpha=0.3)
     axarr[idx].scatter(X_train[y_train==0, 0],
                        X_train[y_train==0, 1],
                        c='blue', marker='^')  
     axarr[idx].scatter(X_train[y_train==1, 0],
                        X_train[y_train==1, 1],
                        c='red', marker='o')  
     axarr[idx].set_title(tt)
axarr[0].set_ylabel(Alcohol', fontsize=12)
plt.text(10.2, -1.2,
          s=Hue',
          ha='center', va='center', fontsize=12)
plt.show()

-- Leveraging weak learners via adaptive boosting

from sklearn.ensemble import AdaBoostClassifier
tree = DecisionTreeClassifier(criterion='entropy',
                               max_depth=1)
ada = AdaBoostClassifier(base_estimator=tree,
                          n_estimators=500,
                          learning_rate=0.1,
                          random_state=0)
tree = tree.fit(X_train, y_train)
y_train_pred = tree.predict(X_train)
y_test_pred = tree.predict(X_test)
tree_train = accuracy_score(y_train, y_train_pred)
tree_test = accuracy_score(y_test, y_test_pred)
print('Decision tree train/test accuracies %.3f/%.3f'
       % (tree_train, tree_test))

ada = ada.fit(X_train, y_train)
y_train_pred = ada.predict(X_train)
y_test_pred = ada.predict(X_test)
ada_train = accuracy_score(y_train, y_train_pred)
ada_test = accuracy_score(y_test, y_test_pred)
print('AdaBoost train/test accuracies %.3f/%.3f'
       % (ada_train, ada_test))


x_min = X_train[:, 0].min() - 1
x_max = X_train[:, 0].max() + 1
y_min = X_train[:, 1].min() - 1
y_max = X_train[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, 0.1),
                      np.arange(y_min, y_max, 0.1))
f, axarr = plt.subplots(1, 2,
                         sharex='col',
                         sharey='row',
                         figsize=(8, 3))
for idx, clf, tt in zip([0, 1],
                         [tree, ada],
                         ['Decision Tree', 'AdaBoost']):
     clf.fit(X_train, y_train)
     Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])
     Z = Z.reshape(xx.shape)
     axarr[idx].contourf(xx, yy, Z, alpha=0.3)
     axarr[idx].scatter(X_train[y_train==0, 0],
                        X_train[y_train==0, 1],
                        c='blue',
                        marker='^')
     axarr[idx].scatter(X_train[y_train==1, 0],
                        X_train[y_train==1, 1],
                        c='red',
                        marker='o')
 axarr[idx].set_title(tt)
 axarr[0].set_ylabel('Alcohol', fontsize=12)
plt.text(10.2, -1.2,
          s=Hue',
          ha='center',
          va='center',
          fontsize=12)  
plt.show()

Python ML 6 - Learning Best Practices for Model Evaluation and Hyperparamter Tuning

Streamlining workflows with pipelines

Loading the Breast Cancer Wisconsin dataset

import pandas as pd
df = pd.read_csv('https://archive.ics.uci.edu/ml/machine-learning-databases/breast-cancer-wisconsin/wdbc.data', header=None)
le.transform(['M', 'B'])


from sklearn.preprocessing import LabelEncoder
X = df.loc[:, 2:].values
y = df.loc[:, 1].values
le = LabelEncoder()
y = le.fit_transform(y)
from sklearn.cross_validation import train_test_split
X_train, X_test, y_train, y_test = \
     train_test_split(X, y, test_size=0.20, random_state=1)

Combining transformers and estimators in a pipeline

from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import Pipeline
pipe_lr = Pipeline([('scl', StandardScaler()),
            ('pca', PCA(n_components=2)),
            ('clf', LogisticRegression(random_state=1))])
pipe_lr.fit(X_train, y_train)
print('Test Accuracy: %.3f' % pipe_lr.score(X_test, y_test))


-- Using k-fold cross-validation to assess model performance

The holdout method

K-fold cross-validation

import numpy as np
from sklearn.cross_validation import StratifiedKFold
kfold = StratifiedKFold(y=y_train,
                         n_folds=10,
                         random_state=1)
scores = []
for k, (train, test) in enumerate(kfold):
    pipe_lr.fit(X_train[train], y_train[train])
    score = pipe_lr.score(X_train[test], y_train[test])
    scores.append(score)
    print('Fold: %s, Class dist.: %s, Acc: %.3f' % (k+1,
                 np.bincount(y_train[train]), score))  
Fold: 1, Class dist.: [256 153], Acc: 0.891
Fold: 2, Class dist.: [256 153], Acc: 0.978
Fold: 3, Class dist.: [256 153], Acc: 0.978
Fold: 4, Class dist.: [256 153], Acc: 0.913
Fold: 5, Class dist.: [256 153], Acc: 0.935
Fold: 6, Class dist.: [257 153], Acc: 0.978
Fold: 7, Class dist.: [257 153], Acc: 0.933
Fold: 8, Class dist.: [257 153], Acc: 0.956
Fold: 9, Class dist.: [257 153], Acc: 0.978
Fold: 10, Class dist.: [257 153], Acc: 0.956
print('CV accuracy: %.3f +/- %.3f' % (
                 np.mean(scores), np.std(scores)))


from sklearn.cross_validation import cross_val_score
scores = cross_val_score(estimator=pipe_lr,
                          X=X_train,
                          y=y_train,
                          cv=10,
                          n_jobs=1)
print('CV accuracy scores: %s' % scores)
print('CV accuracy: %.3f +/- %.3f' % (np.mean(scores), np.std(scores)))

-- Debugging algorithms with learning and validation curves

Diagnosing bias and variance problems with learning curves

import matplotlib.pyplot as plt
from sklearn.learning_curve import learning_curve
pipe_lr = Pipeline([
           ('scl', StandardScaler()),
           ('clf', LogisticRegression(
                        penalty='l2', random_state=0))])
train_sizes, train_scores, test_scores =\
        learning_curve(estimator=pipe_lr,
                       X=X_train,
                       y=y_train,
                       train_sizes=np.linspace(0.1, 1.0, 10),
                       cv=10,
                       n_jobs=1)
train_mean = np.mean(train_scores, axis=1)
train_std = np.std(train_scores, axis=1)
test_mean = np.mean(test_scores, axis=1)
test_std = np.std(test_scores, axis=1)
plt.plot(train_sizes, train_mean,
          color='blue', marker='o',
          markersize=5,
          label='training accuracy')
plt.fill_between(train_sizes,
                  train_mean + train_std,
                  train_mean - train_std,
                  alpha=0.15, color='blue')
plt.plot(train_sizes, test_mean,
          color='green', linestyle='--',
          marker='s', markersize=5,
          label='validation accuracy')
plt.fill_between(train_sizes,
                  test_mean + test_std,
                  test_mean - test_std,
                  alpha=0.15, color='green')
plt.grid()
plt.xlabel('Number of training samples')
plt.ylabel('Accuracy')
plt.legend(loc='lower right')
plt.ylim([0.8, 1.0])
plt.show()

-- Addressing overfitting and underfitting with validation curves

from sklearn.learning_curve import validation_curve
param_range = [0.001, 0.01, 0.1, 1.0, 10.0, 100.0]
train_scores, test_scores = validation_curve(
                estimator=pipe_lr,
                 X=X_train,
                 y=y_train,
                 param_name='clf__C',
                 param_range=param_range,
                 cv=10)
train_mean = np.mean(train_scores, axis=1)
train_std = np.std(train_scores, axis=1)
test_mean = np.mean(test_scores, axis=1)
test_std = np.std(test_scores, axis=1)
plt.plot(param_range, train_mean,
          color='blue', marker='o',
          markersize=5,
          label='training accuracy')
plt.fill_between(param_range, train_mean + train_std,
                  train_mean - train_std, alpha=0.15,
                  color='blue')
plt.plot(param_range, test_mean,
          color='green', linestyle='--',
          marker='s', markersize=5,
          label='validation accuracy')
plt.fill_between(param_range,
                  test_mean + test_std,
                  test_mean - test_std,
                  alpha=0.15, color='green')
plt.grid()
plt.xscale('log')
plt.legend(loc='lower right')
plt.xlabel('Parameter C')
plt.ylabel('Accuracy')
plt.ylim([0.8, 1.0])
plt.show()

-- Fine-tuning machine learning models via grid search

Tuning hyperparameters via grid search

from sklearn.grid_search import GridSearchCV
from sklearn.svm import SVC
pipe_svc = Pipeline([('scl', StandardScaler()),
                      ('clf', SVC(random_state=1))])
param_range = [0.0001, 0.001, 0.01, 0.1, 1.0, 10.0, 100.0, 1000.0]
param_grid = [{'clf__C': param_range,
                'clf__kernel': ['linear']},
               {'clf__C': param_range,
                'clf__gamma': param_range,
                'clf__kernel': ['rbf']}]
gs = GridSearchCV(estimator=pipe_svc,
                   param_grid=param_grid,
                   scoring='accuracy',
                   cv=10,
                   n_jobs=-1)
gs = gs.fit(X_train, y_train)
print(gs.best_score_)
print(gs.best_params_)

clf = gs.best_estimator_
clf.fit(X_train, y_train)
print('Test accuracy: %.3f' % clf.score(X_test, y_test))

Algorithm selection with nested cross-validation

gs = GridSearchCV(estimator=pipe_svc,
                   param_grid=param_grid,
                   scoring='accuracy',
                   cv=5,
                   n_jobs=-1)
scores = cross_val_score(gs, X, y, scoring='accuracy', cv=5)
print('CV accuracy: %.3f +/- %.3f' % (
               np.mean(scores), np.std(scores)))


from sklearn.tree import DecisionTreeClassifier
gs = GridSearchCV(
       estimator=DecisionTreeClassifier(random_state=0),
       param_grid=[
            {'max_depth': [1, 2, 3, 4, 5, 6, 7, None]}],
       scoring='accuracy',
       cv=5)
scores = cross_val_score(gs,
                          X_train,
                          y_train,
                          scoring='accuracy',
                          cv=5)
print('CV accuracy: %.3f +/- %.3f' % (
                     np.mean(scores), np.std(scores)))

-- Looking at different performance evaluation metrics

Reading a confusion matrix

from sklearn.metrics import confusion_matrix
pipe_svc.fit(X_train, y_train)
y_pred = pipe_svc.predict(X_test)
confmat = confusion_matrix(y_true=y_test, y_pred=y_pred)
print(confmat)

fig, ax = plt.subplots(figsize=(2.5, 2.5))
ax.matshow(confmat, cmap=plt.cm.Blues, alpha=0.3)
for i in range(confmat.shape[0]):
     for j in range(confmat.shape[1]):
         ax.text(x=j, y=i,
                 s=confmat[i, j],
                 va='center', ha='center')
plt.xlabel('predicted label')
plt.ylabel('true label')
plt.show()

Optimizing the precision and recall of a classification model

from sklearn.metrics import precision_score
from sklearn.metrics  import recall_score, f1_score
print('Precision: %.3f' % precision_score(
              y_true=y_test, y_pred=y_pred))
print('Recall: %.3f' % recall_score(
              y_true=y_test, y_pred=y_pred))
print('F1: %.3f' % f1_score(
              y_true=y_test, y_pred=y_pred))

from sklearn.metrics import make_scorer, f1_score
scorer = make_scorer(f1_score, pos_label=0)
gs = GridSearchCV(estimator=pipe_svc,
                   param_grid=param_grid,
                   scoring=scorer,
                   cv=10)

Plotting a receiver operating characteristic

from sklearn.metrics import roc_curve, auc
from scipy import interp
X_train2 = X_train[:, [4, 14]]
cv = StratifiedKFold(y_train,
                      n_folds=3,
                      random_state=1)
fig = plt.figure(figsize=(7, 5))
mean_tpr = 0.0
mean_fpr = np.linspace(0, 1, 100)
all_tpr = []

for i, (train, test) in enumerate(cv):
     probas = pipe_lr.fit(X_train2[train],                        
y_train[train]).predict_proba(X_train2[test])  
     fpr, tpr, thresholds = roc_curve(y_train[test],
                                     probas[:, 1],
                                     pos_label=1)
     mean_tpr += interp(mean_fpr, fpr, tpr)
     mean_tpr[0] = 0.0
     roc_auc = auc(fpr, tpr)
     plt.plot(fpr,
              tpr,
              lw=1,
              label='ROC fold %d (area = %0.2f)'
                     % (i+1, roc_auc))
plt.plot([0, 1],
          [0, 1],
          linestyle='--',
          color=(0.6, 0.6, 0.6),
          label='random guessing')
mean_tpr /= len(cv)
mean_tpr[-1] = 1.0
mean_auc = auc(mean_fpr, mean_tpr)
plt.plot(mean_fpr, mean_tpr, 'k--',
          label='mean ROC (area = %0.2f)' % mean_auc, lw=2)
plt.plot([0, 0, 1],
          [0, 1, 1],
          lw=2,
          linestyle=':',
          color='black',
          label='perfect performance')
plt.xlim([-0.05, 1.05])
plt.ylim([-0.05, 1.05])
plt.xlabel('false positive rate')
plt.ylabel('true positive rate')
plt.title('Receiver Operator Characteristic')
plt.legend(loc="lower right")
plt.show()


pipe_svc = pipe_svc.fit(X_train2, y_train)
y_pred2 = pipe_svc.predict(X_test[:, [4, 14]])
from sklearn.metrics import roc_auc_score
from sklearn.metrics import accuracy_score
print('ROC AUC: %.3f' % roc_auc_score(
        y_true=y_test, y_score=y_pred2))

print('Accuracy: %.3f' % accuracy_score(
        y_true=y_test, y_pred=y_pred2))

The scoring metrics for multiclass classification

pre_scorer = make_scorer(score_func=precision_score,
                          pos_label=1,
                          greater_is_better=True,
                          average='micro')

Python ML 5 - Compressing Data via Dimensionality Reduction

-- Unsupervised dimensionality reduction via principal component analysis

Total and explained variance

import pandas as pd
df_wine = pd.read_csv('https://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.data', header=None)

from sklearn.cross_validation import train_test_split
from sklearn.preprocessing import StandardScaler
X, y = df_wine.iloc[:, 1:].values, df_wine.iloc[:, 0].values
X_train, X_test, y_train, y_test = \
              train_test_split(X, y,
              test_size=0.3, random_state=0)
sc = StandardScaler()
X_train_std = sc.fit_transform(X_train)
X_test_std = sc.fit_transform(X_test)

import numpy as np
cov_mat = np.cov(X_train_std.T)
eigen_vals, eigen_vecs = np.linalg.eig(cov_mat)
print('\nEigenvalues \n%s' % eigen_vals)


tot = sum(eigen_vals)
var_exp = [(i / tot) for i in
            sorted(eigen_vals, reverse=True)]
cum_var_exp = np.cumsum(var_exp)

import matplotlib.pyplot as plt
plt.bar(range(1,14), var_exp, alpha=0.5, align='center',
         label='individual explained variance')
plt.step(range(1,14), cum_var_exp, where='mid',
         label='cumulative explained variance')
plt.ylabel('Explained variance ratio')
plt.xlabel('Principal components')
plt.legend(loc='best')
plt.show()

Feature transformation

eigen_pairs =[(np.abs(eigen_vals[i]),eigen_vecs[:,i]) for i inrange(len(eigen_vals))]
eigen_pairs.sort(reverse=True)

w= np.hstack((eigen_pairs[0][1][:, np.newaxis], eigen_pairs[1][1][:, np.newaxis]))
print('Matrix W:\n',w)

X_train_std[0].dot(w)

X_train_pca = X_train_std.dot(w)

colors = ['r', 'b', 'g']
markers = ['s', 'x', 'o']
for l, c, m in zip(np.unique(y_train), colors, markers):
     plt.scatter(X_train_pca[y_train==l, 0],
                 X_train_pca[y_train==l, 1],
                 c=c, label=l, marker=m)
plt.xlabel('PC 1')
plt.ylabel('PC 2')
plt.legend(loc='lower left')
plt.show()

Principal component analysis in scikit-learn

from matplotlib.colors import ListedColormap

def plot_decision_regions(X, y, classifier, resolution=0.02):

    # setup marker generator and color map
    markers = ('s', 'x', 'o', '^', 'v')
    colors = ('red', 'blue', 'lightgreen', 'gray', 'cyan')
    cmap = ListedColormap(colors[:len(np.unique(y))])

    # plot the decision surface
    x1_min, x1_max = X[:, 0].min() - 1, X[:, 0].max() + 1
    x2_min, x2_max = X[:, 1].min() - 1, X[:, 1].max() + 1
    xx1, xx2 = np.meshgrid(np.arange(x1_min, x1_max, resolution),
                         np.arange(x2_min, x2_max, resolution))
    Z = classifier.predict(np.array([xx1.ravel(), xx2.ravel()]).T)
    Z = Z.reshape(xx1.shape)
    plt.contourf(xx1, xx2, Z, alpha=0.4, cmap=cmap)
    plt.xlim(xx1.min(), xx1.max())
    plt.ylim(xx2.min(), xx2.max())

    # plot class samples
    for idx, cl in enumerate(np.unique(y)):
        plt.scatter(x=X[y == cl, 0], y=X[y == cl, 1],
                    alpha=0.8, c=cmap(idx),
                    marker=markers[idx], label=cl)

from sklearn.linear_model import LogisticRegression
from sklearn.decomposition import PCA
pca = PCA(n_components=2)
lr = LogisticRegression()
X_train_pca = pca.fit_transform(X_train_std)
X_test_pca = pca.transform(X_test_std)
lr.fit(X_train_pca, y_train)
plot_decision_regions(X_train_pca, y_train, classifier=lr)
plt.xlabel('PC1')
plt.ylabel('PC2')
plt.legend(loc='lower left')
plt.show()

pca = PCA(n_components=None)
X_train_pca = pca.fit_transform(X_train_std)
pca.explained_variance_ratio_


-- Supervised data compression vis linear discriminant analysis

-- Computing the scatter matrices

np.set_printoptions(precision=4)
mean_vecs = []
for label in range(1,4):
     mean_vecs.append(np.mean(
                X_train_std[y_train==label], axis=0))
     print('MV %s: %s\n' %(label, mean_vecs[label-1]))

d = 13 # number of features
S_W = np.zeros((d, d))
for label,mv in zip(range(1,4), mean_vecs):
     class_scatter = np.zeros((d, d))
     for row in X[y == label]:
         row, mv = row.reshape(d, 1), mv.reshape(d, 1)
         class_scatter += (row-mv).dot((row-mv).T)
     S_W += class_scatter                            
print('Within-class scatter matrix: %sx%s' % (S_W.shape[0], S_W.shape[1]))
print('Class label distribution: %s'  % np.bincount(y_train)[1:])

mean_overall = np.mean(X_train_std, axis=0)
d = 13 # number of features
S_B = np.zeros((d, d))
for i,mean_vec in enumerate(mean_vecs):
     n = X[y==i+1, :].shape[0]
     mean_vec = mean_vec.reshape(d, 1)
     mean_overall = mean_overall.reshape(d, 1)
    S_B += n * (mean_vec - mean_overall).dot(
                (mean_vec - mean_overall).T)
print('Between-class scatter matrix: %sx%s'
    % (S_B.shape[0], S_B.shape[1]))


Selecting linear discriminants for the new feature subspace

eigen_vals, eigen_vecs =np.linalg.eig(np.linalg.inv(S_W).dot(S_B))

eigen_pairs = [(np.abs(eigen_vals[i]), eigen_vecs[:,i])
              for i in range(len(eigen_vals))]
eigen_pairs = sorted(eigen_pairs,
               key=lambda k: k[0], reverse=True)
print('Eigenvalues in decreasing order:\n')
for eigen_val in eigen_pairs:
     print(eigen_val[0])

tot = sum(eigen_vals.real)
discr = [(i / tot) for i in sorted(eigen_vals.real, reverse=True)]
cum_discr = np.cumsum(discr)
plt.bar(range(1, 14), discr, alpha=0.5, align='center',
         label='individual "discriminability"')
plt.step(range(1, 14), cum_discr, where='mid',
          label='cumulative "discriminability"')
plt.ylabel('"discriminability" ratio')
plt.xlabel('Linear Discriminants')
plt.ylim([-0.1, 1.1])
plt.legend(loc='best')
plt.show()

w = np.hstack((eigen_pairs[0][1][:, np.newaxis].real,
                eigen_pairs[1][1][:, np.newaxis].real))
print('Matrix W:\n', w)

Projecting samples onto the new feature space

X_train_lda = X_train_std.dot(w)
colors = ['r', 'b', 'g']
markers = ['s', 'x', 'o']
for l, c, m in zip(np.unique(y_train), colors, markers):
     plt.scatter(X_train_lda[y_train==l, 0],
                 X_train_lda[y_train==l, 1],
                 c=c, label=l, marker=m)
plt.xlabel('LD 1')
plt.ylabel('LD 2')
plt.legend(loc='upper right')
plt.show()

LDA via scikit-learn

from sklearn.lda import LDA
lda = LDA(n_components=2)
X_train_lda = lda.fit_transform(X_train_std, y_train)


lr = LogisticRegression()
lr = lr.fit(X_train_lda, y_train)
plot_decision_regions(X_train_lda, y_train, classifier=lr)
plt.xlabel('LD 1')
plt.ylabel('LD 2')
plt.legend(loc='lower left')
plt.show()

X_test_lda = lda.transform(X_test_std)
plot_decision_regions(X_test_lda, y_test, classifier=lr)
plt.xlabel('LD 1')
plt.ylabel('LD 2')
plt.legend(loc='lower left')
plt.show()

-- Using kernel principal component analysis for nonlinear mappings

Kernel functions and the kernel trick

Implementing a kernel principal component analysis in Python

from scipy.spatial.distance import pdist, squareform
from scipy import exp
from scipy.linalg import eigh
import numpy as np

def rbf_kernel_pca(X, gamma, n_components):
    """
    RBF kernel PCA implementation.

    Parameters
    ------------
    X: {NumPy ndarray}, shape = [n_samples, n_features]

    gamma: float
      Tuning parameter of the RBF kernel

    n_components: int
      Number of principal components to return

    Returns
    ------------
     X_pc: {NumPy ndarray}, shape = [n_samples, k_features]
       Projected dataset  

    """
    # Calculate pairwise squared Euclidean distances
    # in the MxN dimensional dataset.
    sq_dists = pdist(X, 'sqeuclidean')

    # Convert pairwise distances into a square matrix.
    mat_sq_dists = squareform(sq_dists)

    # Compute the symmetric kernel matrix.
    K = exp(-gamma * mat_sq_dists)

    # Center the kernel matrix.
    N = K.shape[0]
    one_n = np.ones((N,N)) / N
    K = K - one_n.dot(K) - K.dot(one_n) + one_n.dot(K).dot(one_n)

    # Obtaining eigenpairs from the centered kernel matrix
    # numpy.eigh returns them in sorted order
    eigvals, eigvecs = eigh(K)

    # Collect the top k eigenvectors (projected samples)
    X_pc = np.column_stack((eigvecs[:, -i]
                            for i in range(1, n_components + 1)))

    return X_pc

Example 1 ñ separating half-moon shapes

from sklearn.datasets import make_moons
X, y = make_moons(n_samples=100, random_state=123)
plt.scatter(X[y==0, 0], X[y==0, 1],
             color='red', marker='^', alpha=0.5)
plt.scatter(X[y==1, 0], X[y==1, 1],
             color='blue', marker='o', alpha=0.5)
plt.show()


from sklearn.decomposition import PCA
scikit_pca = PCA(n_components=2)
X_spca = scikit_pca.fit_transform(X)
fig, ax = plt.subplots(nrows=1,ncols=2, figsize=(7,3))
ax[0].scatter(X_spca[y==0, 0], X_spca[y==0, 1],
               color='red', marker='^', alpha=0.5)
ax[0].scatter(X_spca[y==1, 0], X_spca[y==1, 1],
               color='blue', marker='o', alpha=0.5)
ax[1].scatter(X_spca[y==0, 0], np.zeros((50,1))+0.02,
               color='red', marker='^', alpha=0.5)
ax[1].scatter(X_spca[y==1, 0], np.zeros((50,1))-0.02,
               color='blue', marker='o', alpha=0.5)
ax[0].set_xlabel('PC1')
ax[0].set_ylabel('PC2')
ax[1].set_ylim([-1, 1])
ax[1].set_yticks([])
ax[1].set_xlabel('PC1')
plt.show()

from matplotlib.ticker import FormatStrFormatter
X_kpca = rbf_kernel_pca(X, gamma=15, n_components=2)
fig, ax = plt.subplots(nrows=1,ncols=2, figsize=(7,3))
ax[0].scatter(X_kpca[y==0, 0], X_kpca[y==0, 1],
               color='red', marker='^', alpha=0.5)
ax[0].scatter(X_kpca[y==1, 0], X_kpca[y==1, 1],
               color='blue', marker='o', alpha=0.5)
ax[1].scatter(X_kpca[y==0, 0], np.zeros((50,1))+0.02,
               color='red', marker='^', alpha=0.5)
ax[1].scatter(X_kpca[y==1, 0], np.zeros((50,1))-0.02,
               color='blue', marker='o', alpha=0.5)
ax[0].set_xlabel('PC1')
ax[0].set_ylabel('PC2')
ax[1].set_ylim([-1, 1])
ax[1].set_yticks([])
ax[1].set_xlabel('PC1')
ax[0].xaxis.set_major_formatter(FormatStrFormatter('%0.1f'))
ax[1].xaxis.set_major_formatter(FormatStrFormatter('%0.1f'))
plt.show()


Example 2 ñ separating concentric circles

from sklearn.datasets import make_circles
X, y = make_circles(n_samples=1000,
            random_state=123, noise=0.1, factor=0.2)
plt.scatter(X[y==0, 0], X[y==0, 1],
            color='red', marker='^', alpha=0.5)
plt.scatter(X[y==1, 0], X[y==1, 1],
            color='blue', marker='o', alpha=0.5)
plt.show()

scikit_pca = PCA(n_components=2)
X_spca = scikit_pca.fit_transform(X)
fig, ax = plt.subplots(nrows=1,ncols=2, figsize=(7,3))
ax[0].scatter(X_spca[y==0, 0], X_spca[y==0, 1],
               color='red', marker='^', alpha=0.5)
ax[0].scatter(X_spca[y==1, 0], X_spca[y==1, 1],
               color='blue', marker='o', alpha=0.5)
ax[1].scatter(X_spca[y==0, 0], np.zeros((500,1))+0.02,
              color='red', marker='^', alpha=0.5)
ax[1].scatter(X_spca[y==1, 0], np.zeros((500,1))-0.02,
               color='blue', marker='o', alpha=0.5)
ax[0].set_xlabel('PC1')
ax[0].set_ylabel('PC2')
ax[1].set_ylim([-1, 1])
ax[1].set_yticks([])
ax[1].set_xlabel('PC1')
plt.show()

X_kpca = rbf_kernel_pca(X, gamma=15, n_components=2)
fig, ax = plt.subplots(nrows=1,ncols=2, figsize=(7,3))
ax[0].scatter(X_kpca[y==0, 0], X_kpca[y==0, 1],
               color='red', marker='^', alpha=0.5)
ax[0].scatter(X_kpca[y==1, 0], X_kpca[y==1, 1],
               color='blue', marker='o', alpha=0.5)
ax[1].scatter(X_kpca[y==0, 0], np.zeros((500,1))+0.02,
               color='red', marker='^', alpha=0.5)
ax[1].scatter(X_kpca[y==1, 0], np.zeros((500,1))-0.02,
               color='blue', marker='o', alpha=0.5)
ax[0].set_xlabel('PC1')
ax[0].set_ylabel('PC2')
ax[1].set_ylim([-1, 1])
ax[1].set_yticks([])
ax[1].set_xlabel('PC1')
plt.show()

from scipy.spatial.distance import pdist, squareform
from scipy import exp
from scipy.linalg import eigh
import numpy as np

def rbf_kernel_pca(X, gamma, n_components):
    """
    RBF kernel PCA implementation.

    Parameters
    ------------
    X: {NumPy ndarray}, shape = [n_samples, n_features]

    gamma: float
      Tuning parameter of the RBF kernel

    n_components: int
      Number of principal components to return

    Returns
    ------------
     X_pc: {NumPy ndarray}, shape = [n_samples, k_features]
       Projected dataset  

     lambdas: list
       Eigenvalues

    """
    # Calculate pairwise squared Euclidean distances
    # in the MxN dimensional dataset.
    sq_dists = pdist(X, 'sqeuclidean')

    # Convert pairwise distances into a square matrix.
    mat_sq_dists = squareform(sq_dists)

    # Compute the symmetric kernel matrix.
    K = exp(-gamma * mat_sq_dists)

    # Center the kernel matrix.
    N = K.shape[0]
    one_n = np.ones((N,N)) / N
    K = K - one_n.dot(K) - K.dot(one_n) + one_n.dot(K).dot(one_n)

    # Obtaining eigenpairs from the centered kernel matrix
    # numpy.eigh returns them in sorted order
    eigvals, eigvecs = eigh(K)

    # Collect the top k eigenvectors (projected samples)
    alphas = np.column_stack((eigvecs[:,-i]
                    for i in range(1,n_components+1)))

    # Collect the corresponding eigenvalues
    lambdas = [eigvals[-i] for i in range(1,n_components+1)]

    return alphas, lambdas

X, y = make_moons(n_samples=100, random_state=123)
alphas, lambdas =rbf_kernel_pca(X, gamma=15, n_components=1)

x_new = X[25]
x_new

x_proj = alphas[25] # original projection
x_proj

def project_x(x_new, X, gamma, alphas, lambdas):
     pair_dist = np.array([np.sum(
                  (x_new-row)**2) for row in X])
     k = np.exp(-gamma * pair_dist)
return k.dot(alphas / lambdas)

x_reproj = project_x(x_new, X,
       gamma=15, alphas=alphas, lambdas=lambdas)
x_reproj


plt.scatter(alphas[y==0, 0], np.zeros((50)),
             color='red', marker='^',alpha=0.5)
plt.scatter(alphas[y==1, 0], np.zeros((50)),
             color='blue', marker='o', alpha=0.5)
plt.scatter(x_proj, 0, color='black',
             label='original projection of point X[25]',
             marker='^', s=100)
plt.scatter(x_reproj, 0, color='green',
             label='remapped point X[25]',
             marker='x', s=500)
plt.legend(scatterpoints=1)
plt.show()

Kernel principal component analysis in scikit-learn

from sklearn.decomposition import KernelPCA
X, y = make_moons(n_samples=100, random_state=123)
scikit_kpca = KernelPCA(n_components=2,
               kernel='rbf', gamma=15)
X_skernpca = scikit_kpca.fit_transform(X)

plt.scatter(X_skernpca[y==0, 0], X_skernpca[y==0, 1],
             color='red', marker='^', alpha=0.5)
plt.scatter(X_skernpca[y==1, 0], X_skernpca[y==1, 1],
             color='blue', marker='o', alpha=0.5)
plt.xlabel('PC1')
plt.ylabel('PC2')
plt.show()

Python ML 4 - Building Good Training Sets ñ Data Preprocessing

-- Dealing with missing data

import pandas as pd
from io import StringIO
csv_data = '''A,B,C,D
1.0,2.0,3.0,4.0
5.0,6.0,,8.0
0.0,11.0,12.0,'''
# If you are using Python 2.7, you need
# to convert the string to unicode:
# csv_data = unicode(csv_data)
df = pd.read_csv(StringIO(csv_data))
df
df.isnull().sum()

Eliminating samples or features with missing values

df.dropna()
df.dropna(axis=1)
df.dropna(how='all')
df.dropna(thresh=4)
df.dropna(subset=['C'])

Imputing missing values

from sklearn.preprocessing import Imputer
imr = Imputer(missing_values='NaN', strategy='mean', axis=0)
imr = imr.fit(df)
imputed_data = imr.transform(df.values)
imputed_data

Understanding the scikit-learn estimator API

Training Data/Test Data
est.fit(X_train)
est.transform(X_train)
est.transform(X_test)

Test
est.fit(X_train, y_train)
est.predict(X_test)


-- Handling categorical data

import pandas as pd
df = pd.DataFrame([
            ['green', 'M', 10.1, 'class1'],
            ['red', 'L', 13.5, 'class2'],
            ['blue', 'XL', 15.3, 'class1']])
df.columns = ['color', 'size', 'price', 'classlabel']
df

Mapping ordinal features
size_mapping = {
                 'XL': 3,
                 'L': 2,
                 'M': 1}
df['size'] = df['size'].map(size_mapping)
df
inv_size_mapping = {v: k for k, v in size_mapping.items()}

Encoding class labels
import numpy as np
class_mapping = {label:idx for idx,label in
                  enumerate(np.unique(df['classlabel']))}
class_mapping
df['classlabel'] = df['classlabel'].map(class_mapping)
df
inv_class_mapping = {v: k for k, v in class_mapping.items()}
df['classlabel'] = df['classlabel'].map(inv_class_mapping)
df

from sklearn.preprocessing import LabelEncoder
class_le = LabelEncoder()
y = class_le.fit_transform(df['classlabel'].values)
y
class_le.inverse_transform(y)

Performing one-hot encoding on nominal features
X = df[['color', 'size', 'price']].values
color_le = LabelEncoder()
X[:, 0] = color_le.fit_transform(X[:, 0])
X

from sklearn.preprocessing import OneHotEncoder
ohe = OneHotEncoder(categorical_features=[0])
ohe.fit_transform(X).toarray()
pd.get_dummies(df[['price', 'color', 'size']])

-- Partitioning a dataset in training and test sets

df_wine = pd.read_csv('https://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.data', header=None)
df_wine.columns = ['Class label', 'Alcohol',
                    'Malic acid', 'Ash',
                    'Alcalinity of ash', 'Magnesium',
                    'Total phenols', 'Flavanoids',
                    'Nonflavanoid phenols',
                    'Proanthocyanins',
                    'Color intensity', 'Hue',
                    'OD280/OD315 of diluted wines',
                    'Proline']
print('Class labels', np.unique(df_wine['Class label']))
df_wine.head()

from sklearn.cross_validation import train_test_split
X, y = df_wine.iloc[:, 1:].values, df_wine.iloc[:, 0].values
X_train, X_test, y_train, y_test = \
        train_test_split(X, y, test_size=0.3, random_state=0)

-- Bringing features onto the same scale

from sklearn.preprocessing import MinMaxScaler
mms = MinMaxScaler()
X_train_norm = mms.fit_transform(X_train)
X_test_norm = mms.transform(X_test)

Python ML 3 - A Tour of Machine Learning Classifiers Using Scikit-learn

-- Choosing a classification algorithm

1.Selection of features.

2.Choosing a performance metric.

3.Choosing a classifier and optimization algorithm.

4.Evaluating the performance of the model.

5.Tuning the algorithm.

-- First steps with scikit-learn

Training a perceptron via scikit-learn

from sklearn import datasets

import numpy as np

iris = datasets.load_iris()

X = iris.data[:, [2, 3]]

y = iris.target

from sklearn.cross_validation import train_test_split

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)

from sklearn.preprocessing import StandardScaler

sc = StandardScaler()

sc.fit(X_train)

X_train_std = sc.transform(X_train)

X_test_std = sc.transform(X_test)

from sklearn.linear_model import Perceptron

ppn = Perceptron(n_iter=40, eta0=0.1, random_state=0)

ppn.fit(X_train_std, y_train)

y_pred = ppn.predict(X_test_std)

print('Misclassified samples: %d' % (y_test != y_pred).sum())

from sklearn.metrics import accuracy_score

print('Accuracy: %.2f' % accuracy_score(y_test, y_pred))

--------------------------------------------------------------------------------------------------------------------

from matplotlib.colors import ListedColormap

import matplotlib.pyplot as plt

def plot_decision_regions(X, y, classifier, 

                    test_idx=None, resolution=0.02):

    # setup marker generator and color map

    markers = ('s', 'x', 'o', '^', 'v')

    colors = ('red', 'blue', 'lightgreen', 'gray', 'cyan')

    cmap = ListedColormap(colors[:len(np.unique(y))])

    # plot the decision surface

    x1_min, x1_max = X[:, 0].min() - 1, X[:, 0].max() + 1

    x2_min, x2_max = X[:, 1].min() - 1, X[:, 1].max() + 1

    xx1, xx2 = np.meshgrid(np.arange(x1_min, x1_max, resolution),

                         np.arange(x2_min, x2_max, resolution))

    Z = classifier.predict(np.array([xx1.ravel(), xx2.ravel()]).T)

    Z = Z.reshape(xx1.shape)

    plt.contourf(xx1, xx2, Z, alpha=0.4, cmap=cmap)

    plt.xlim(xx1.min(), xx1.max())

    plt.ylim(xx2.min(), xx2.max())

    # plot all samples

    X_test, y_test = X[test_idx, :], y[test_idx]                               

    for idx, cl in enumerate(np.unique(y)):

        plt.scatter(x=X[y == cl, 0], y=X[y == cl, 1],

                    alpha=0.8, c=cmap(idx),

                    marker=markers[idx], label=cl)

        

    # highlight test samples

    if test_idx:

        X_test, y_test = X[test_idx, :], y[test_idx]   

        plt.scatter(X_test[:, 0], X_test[:, 1], c='', 

                alpha=1.0, linewidth=1, marker='o', 

                s=55, label='test set')

--------------------------------------------------------------------------------------------------------------------

X_combined_std = np.vstack((X_train_std, X_test_std))

y_combined = np.hstack((y_train, y_test))

plot_decision_regions(X=X_combined_std, 

                      y=y_combined, 

                      classifier=ppn,

                      test_idx=range(105,150))

plt.xlabel('petal length [standardized]') 

plt.ylabel('petal width [standardized]') 

plt.legend(loc='upper left')

plt.show()

-- Modeling class probabilities via logistic regression

-- Logistic regression intuition and conditional probabilities

import matplotlib.pyplot as plt

import numpy as np

def sigmoid(z):

...     return 1.0 / (1.0 + np.exp(-z))

z = np.arange(-7, 7, 0.1)

phi_z = sigmoid(z)

plt.plot(z, phi_z)

plt.axvline(0.0, color='k')

plt.axhspan(0.0, 1.0, facecolor='1.0', alpha=1.0, ls='dotted')

plt.axhline(y=0.5, ls='dotted', color='k')

plt.yticks([0.0, 0.5, 1.0])

plt.ylim(-0.1, 1.1)

plt.xlabel('z')

plt.ylabel('$\phi (z)$')

plt.show() 

-- Learning the weights of the logistic cost function

-- Training a logistic regression model with scikit-learn

from sklearn.linear_model import LogisticRegression

lr = LogisticRegression(C=1000.0, random_state=0)

lr.fit(X_train_std, y_train)

plot_decision_regions(X_combined_std, 

                       y_combined, classifier=lr,

                       test_idx=range(105,150))

plt.xlabel('petal length [standardized]')

plt.ylabel('petal width [standardized]')

plt.legend(loc='upper left')

plt.show()

lr.predict_proba(X_test_std[0,:])

weights, params = [], []

for c in np.arange(-5, 5):

     lr = LogisticRegression(C=10**c, random_state=0)

     lr.fit(X_train_std, y_train)

     weights.append(lr.coef_[1])

     params.append(10**c)

weights = np.array(weights)

plt.plot(params, weights[:, 0], 

          label='petal length')

plt.plot(params, weights[:, 1], linestyle='--', 

          label='petal width')

plt.ylabel('weight coefficient')

plt.xlabel('C')

plt.legend(loc='upper left')

plt.xscale('log')

plt.show()

lr.predict_proba(X_test_std[0,:])

-- Tackling overfitting via regularization

weights, params = [], []

for c in np.arange(-5, 5):

     lr = LogisticRegression(C=10**c, random_state=0)

     lr.fit(X_train_std, y_train)

     weights.append(lr.coef_[1])

     params.append(10**c)

weights = np.array(weights)

plt.plot(params, weights[:, 0], 

          label='petal length')

plt.plot(params, weights[:, 1], linestyle='--', 

          label='petal width')

plt.ylabel('weight coefficient')

plt.xlabel('C')

plt.legend(loc='upper left')

plt.xscale('log')

plt.show()

-- Maximum margin classification with support vector machines

Maximum margin intuition

-- Dealing with the nonlinearly separable case using slack variables

from sklearn.svm import SVC

svm = SVC(kernel='linear', C=1.0, random_state=0)

svm.fit(X_train_std, y_train)

plot_decision_regions(X_combined_std, 

                       y_combined, classifier=svm,

                       test_idx=range(105,150))

plt.xlabel('petal length [standardized]')

plt.ylabel('petal width [standardized]')

plt.legend(loc='upper left')

plt.show()

-- Alternative implementations in scikit-learn

from sklearn.linear_model import SGDClassifier

ppn = SGDClassifier(loss='perceptron')

lr = SGDClassifier(loss='log')

svm = SGDClassifier(loss='hinge')

-- Solving nonlinear problems using a kernel SVM

np.random.seed(0)

X_xor = np.random.randn(200, 2)

y_xor = np.logical_xor(X_xor[:, 0] > 0, X_xor[:, 1] > 0)

y_xor = np.where(y_xor, 1, -1)

plt.scatter(X_xor[y_xor==1, 0], X_xor[y_xor==1, 1],

             c='b', marker='x', label='1')

plt.scatter(X_xor[y_xor==-1, 0], X_xor[y_xor==-1, 1],

             c='r', marker='s', label='-1')

plt.ylim(-3.0)

plt.legend()

plt.show()

-- Using the kernel trick to find separating hyperplanes in higher dimensional space

svm = SVC(kernel='rbf', random_state=0, gamma=0.10, C=10.0)

svm.fit(X_xor, y_xor)

plot_decision_regions(X_xor, y_xor, classifier=svm)

plt.legend(loc='upper left')

plt.show()

svm = SVC(kernel='rbf', random_state=0, gamma=100.0, C=1.0)

svm.fit(X_train_std, y_train)

plot_decision_regions(X_combined_std,

                       y_combined, classifier=svm,

                       test_idx=range(105,150))

plt.xlabel('petal length [standardized]')

plt.ylabel('petal width [standardized]')

plt.legend(loc='upper left')

plt.show()

--- Decision tree learning

--- Maximizing information gain ñ getting the most bang for the buck

--------------------------------------------------------------------------------------------------------------------

import matplotlib.pyplot as plt

import numpy as np

def gini(p):

     return (p)*(1 - (p)) + (1 - p)*(1 - (1-p))

def entropy(p):

     return - p*np.log2(p) - (1 - p)*np.log2((1 - p))

def error(p):

     return 1 - np.max([p, 1 - p])

x = np.arange(0.0, 1.0, 0.01)

ent = [entropy(p) if p != 0 else None for p in x]

sc_ent = [e*0.5 if e else None for e in ent]

err = [error(i) for i in x]

fig = plt.figure()

ax = plt.subplot(111)

for i, lab, ls, c, in zip([ent, sc_ent, gini(x), err], 

                   ['Entropy', 'Entropy (scaled)', 

                   'Gini Impurity', 

                   'Misclassification Error'],

                   ['-', '-', '--', '-.'],

                   ['black', 'lightgray',

                      'red', 'green', 'cyan']):

     line = ax.plot(x, i, label=lab, 

                    linestyle=ls, lw=2, color=c)

ax.legend(loc='upper center', bbox_to_anchor=(0.5, 1.15),

           ncol=3, fancybox=True, shadow=False)

ax.axhline(y=0.5, linewidth=1, color='k', linestyle='--')

ax.axhline(y=1.0, linewidth=1, color='k', linestyle='--')

plt.ylim([0, 1.1])

plt.xlabel('p(i=1)')

plt.ylabel('Impurity Index')

plt.show()

--------------------------------------------------------------------------------------------------------------------

--- Building a decision tree

from sklearn.tree import DecisionTreeClassifier

tree = DecisionTreeClassifier(criterion='entropy', 

                               max_depth=3, random_state=0)

tree.fit(X_train, y_train)

X_combined = np.vstack((X_train, X_test))

y_combined = np.hstack((y_train, y_test))

plot_decision_regions(X_combined, y_combined, 

                    classifier=tree, test_idx=range(105,150))

plt.xlabel('petal length [cm]')

plt.ylabel('petal width [cm]') 

plt.legend(loc='upper left')

plt.show()

from sklearn.tree import export_graphviz

export_graphviz(tree, 

                 out_file='tree.dot',

                 feature_names=['petal length', 'petal width'])

dot -Tpng tree.dot -o tree.png

--- Combining weak to strong learners via random forests

from sklearn.ensemble import RandomForestClassifier

forest = RandomForestClassifier(criterion='entropy',

                                 n_estimators=10, 

                                 random_state=1,

                                 n_jobs=2)

forest.fit(X_train, y_train)

plot_decision_regions(X_combined, y_combined, 

                classifier=forest, test_idx=range(105,150))

plt.xlabel('petal length')

plt.ylabel('petal width')

plt.legend(loc='upper left')

plt.show()

--- K-nearest neighbors ñ a lazy learning algorithm

from sklearn.neighbors import KNeighborsClassifier

knn = KNeighborsClassifier(n_neighbors=5, p=2,

                            metric='minkowski')

knn.fit(X_train_std, y_train)

plot_decision_regions(X_combined_std, y_combined, 

                       classifier=knn, test_idx=range(105,150))

plt.xlabel('petal length [standardized]')

plt.ylabel('petal width [standardized]')

plt.show()

Python ML 2 - Training Machine Learning Algorithms for Classification

-- Artificial neurons ñ a brief glimpse into the early history of machine learning

NumPy: http://wiki.scipy.org/Tentative_NumPy_Tutorial

Pandas: http://pandas.pydata.org/pandas-docs/stable/tutorials.html

Matplotlib: http://matplotlib.org/users/beginner.html

-- Implementing a perceptron learning algorithm in Python

-------------------------------------------------------------------------------------------

import numpy as np    

class Perceptron(object):    

"""Perceptron classifier.    

Parameters    

------------    

eta : float        

Learning rate (between 0.0 and 1.0)    

n_iter : int        

Passes over the training dataset.    

Attributes    

-----------    

w_ : 1d-array        

Weights after fitting.    

errors_ : list        

Number of misclassifications in every epoch.    

"""    

def __init__(self, eta=0.01, n_iter=10):        

self.eta = eta        

self.n_iter = n_iter    

def fit(self, X, y):        

"""Fit training data.        

Parameters        

----------        

X : {array-like}, shape = [n_samples, n_features]            

Training vectors, where n_samples             

is the number of samples and            

n_features is the number of features.        

y : array-like, shape = [n_samples]            

Target values.        

Returns        

-------        

self : object        

"""        

self.w_ = np.zeros(1 + X.shape[1])        

self.errors_ = []        

for _ in range(self.n_iter):            

errors = 0            

for xi, target in zip(X, y):                

update = self.eta * (target - self.predict(xi))                

self.w_[1:] += update * xi                

self.w_[0] += update                

errors += int(update != 0.0)            

self.errors_.append(errors)        

return self    

def net_input(self, X):        

"""Calculate net input"""        

return np.dot(X, self.w_[1:]) + self.w_[0]    

def predict(self, X):        

"""Return class label after unit step"""        

return np.where(self.net_input(X) >= 0.0, 1, -1)

-------------------------------------------------------------------------------------------

Training a perceptron model on the Iris dataset

import pandas as pd

df = pd.read_csv('https://archive.ics.uci.edu/ml/'...'machine-learning-databases/iris/iris.data', header=None)

df.tail()

import matplotlib.pyplot as plt

import numpy as np

y = df.iloc[0:100, 4].values

y = np.where(y == 'Iris-setosa', -1, 1)

X = df.iloc[0:100, [0, 2]].values

plt.scatter(X[:50, 0], X[:50, 1],

...             color='red', marker='o', label='setosa')

plt.scatter(X[50:100, 0], X[50:100, 1],

...             color='blue', marker='x', label='versicolor')

plt.xlabel('petal length')

plt.ylabel('sepal length')

plt.legend(loc='upper left')

plt.show()

ppn = Perceptron(eta=0.1, n_iter=10)

ppn.fit(X, y)

plt.plot(range(1, len(ppn.errors_) + 1), ppn.errors_, 

...         marker='o')

plt.xlabel('Epochs')

plt.ylabel('Number of misclassifications')

plt.show()

-------------------------------------------------------------------------------------------

from matplotlib.colors import ListedColormap

def plot_decision_regions(X, y, classifier, resolution=0.02):

    # setup marker generator and color map

    markers = ('s', 'x', 'o', '^', 'v')

    colors = ('red', 'blue', 'lightgreen', 'gray', 'cyan')

    cmap = ListedColormap(colors[:len(np.unique(y))])

    # plot the decision surface

    x1_min, x1_max = X[:, 0].min() - 1, X[:, 0].max() + 1

    x2_min, x2_max = X[:, 1].min() - 1, X[:, 1].max() + 1

    xx1, xx2 = np.meshgrid(np.arange(x1_min, x1_max, resolution),

                         np.arange(x2_min, x2_max, resolution))

    Z = classifier.predict(np.array([xx1.ravel(), xx2.ravel()]).T)

    Z = Z.reshape(xx1.shape)

    plt.contourf(xx1, xx2, Z, alpha=0.4, cmap=cmap)

    plt.xlim(xx1.min(), xx1.max())

    plt.ylim(xx2.min(), xx2.max())

    # plot class samples

    for idx, cl in enumerate(np.unique(y)):

        plt.scatter(x=X[y == cl, 0], y=X[y == cl, 1],

                    alpha=0.8, c=cmap(idx),

                    marker=markers[idx], label=cl)

-------------------------------------------------------------------------------------------

plot_decision_regions(X, y, classifier=ppn)

plt.xlabel('sepal length [cm]')

plt.ylabel('petal length [cm]')

plt.legend(loc='upper left')

plt.show()

-- Adaptive linear neurons and the convergence of learning

-- Minimizing cost functions with gradient descent

-- Implementing an Adaptive Linear Neuron in Python

-------------------------------------------------------------------------------------------

class AdalineGD(object):

    """ADAptive LInear NEuron classifier.

    Parameters

    ------------

    eta : float

        Learning rate (between 0.0 and 1.0)

    n_iter : int

        Passes over the training dataset.

    

    Attributes

    -----------

    w_ : 1d-array

        Weights after fitting.

    errors_ : list

        Number of misclassifications in every epoch.

    """

    def __init__(self, eta=0.01, n_iter=50):

        self.eta = eta

        self.n_iter = n_iter

    def fit(self, X, y):

        """ Fit training data.

        Parameters

        ----------

        X : {array-like}, shape = [n_samples, n_features]

            Training vectors, 

            where n_samples is the number of samples and

            n_features is the number of features.

        y : array-like, shape = [n_samples]

            Target values.

        Returns

        -------

        self : object

        """

        self.w_ = np.zeros(1 + X.shape[1])

        self.cost_ = []

        for i in range(self.n_iter):

            output = self.net_input(X)

            errors = (y - output)

            self.w_[1:] += self.eta * X.T.dot(errors)

            self.w_[0] += self.eta * errors.sum()

            cost = (errors**2).sum() / 2.0

            self.cost_.append(cost)

        return self

    def net_input(self, X):

        """Calculate net input"""

        return np.dot(X, self.w_[1:]) + self.w_[0]

    def activation(self, X):

        """Compute linear activation"""

        return self.net_input(X)

    def predict(self, X):

        """Return class label after unit step"""

        return np.where(self.activation(X) >= 0.0, 1, -1)

-------------------------------------------------------------------------------------------

fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(8, 4))

ada1 = AdalineGD(n_iter=10, eta=0.01).fit(X, y)

ax[0].plot(range(1, len(ada1.cost_) + 1),

...            np.log10(ada1.cost_), marker='o')

ax[0].set_xlabel('Epochs')

ax[0].set_ylabel('log(Sum-squared-error)')

ax[0].set_title('Adaline - Learning rate 0.01')

ada2 = AdalineGD(n_iter=10, eta=0.0001).fit(X, y)

ax[1].plot(range(1, len(ada2.cost_) + 1),

...            ada2.cost_, marker='o')

ax[1].set_xlabel('Epochs')

ax[1].set_ylabel('Sum-squared-error')

ax[1].set_title('Adaline - Learning rate 0.0001')

plt.show()

X_std = np.copy(X)

X_std[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std()

X_std[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std()

ada = AdalineGD(n_iter=15, eta=0.01)

ada.fit(X_std, y)

plot_decision_regions(X_std, y, classifier=ada)

plt.title('Adaline - Gradient Descent')

plt.xlabel('sepal length [standardized]')

plt.ylabel('petal length [standardized]')

plt.legend(loc='upper left')

plt.show()

plt.plot(range(1, len(ada.cost_) + 1), ada.cost_, marker='o')

plt.xlabel('Epochs')

plt.ylabel('Sum-squared-error')

plt.show()

-- Large scale machine learning and stochastic gradient descent

-------------------------------------------------------------------------------------------

from numpy.random import seed

class AdalineSGD(object):

    """ADAptive LInear NEuron classifier.

    Parameters

    ------------

    eta : float

        Learning rate (between 0.0 and 1.0)

    n_iter : int

        Passes over the training dataset.

    Attributes

    -----------

    w_ : 1d-array

        Weights after fitting.

    errors_ : list

        Number of misclassifications in every epoch.

    shuffle : bool (default: True)

        Shuffles training data every epoch 

        if True to prevent cycles.

    random_state : int (default: None)

        Set random state for shuffling 

        and initializing the weights.

        

    """

    def __init__(self, eta=0.01, n_iter=10, 

               shuffle=True, random_state=None):

        self.eta = eta

        self.n_iter = n_iter

        self.w_initialized = False

        self.shuffle = shuffle

        if random_state:

            seed(random_state)

        

    def fit(self, X, y):

        """ Fit training data.

        Parameters

        ----------

        X : {array-like}, shape = [n_samples, n_features]

            Training vectors, where n_samples 

            is the number of samples and

            n_features is the number of features.

        y : array-like, shape = [n_samples]

            Target values.

        Returns

        -------

        self : object

         """

        self._initialize_weights(X.shape[1])

        self.cost_ = []

        for i in range(self.n_iter):

            if self.shuffle:

                X, y = self._shuffle(X, y)

            cost = []

            for xi, target in zip(X, y):

                cost.append(self._update_weights(xi, target))

            avg_cost = sum(cost)/len(y)

            self.cost_.append(avg_cost)

        return self

    def partial_fit(self, X, y):

        """Fit training data without reinitializing the weights"""

        if not self.w_initialized:

            self._initialize_weights(X.shape[1])

        if y.ravel().shape[0] > 1:

            for xi, target in zip(X, y):

                self._update_weights(xi, target)

        else:

            self._update_weights(X, y)

        return self

    def _shuffle(self, X, y):

        """Shuffle training data"""

        r = np.random.permutation(len(y))

        return X[r], y[r]

    

    def _initialize_weights(self, m):

        """Initialize weights to zeros"""

        self.w_ = np.zeros(1 + m)

        self.w_initialized = True

        

    def _update_weights(self, xi, target):

        """Apply Adaline learning rule to update the weights"""

        output = self.net_input(xi)

        error = (target - output)

        self.w_[1:] += self.eta * xi.dot(error)

        self.w_[0] += self.eta * error

        cost = 0.5 * error**2

        return cost

    

    def net_input(self, X):

        """Calculate net input"""

        return np.dot(X, self.w_[1:]) + self.w_[0]

    def activation(self, X):

        """Compute linear activation"""

        return self.net_input(X)

    def predict(self, X):

        """Return class label after unit step"""

        return np.where(self.activation(X) >= 0.0, 1, -1)

-------------------------------------------------------------------------------------------

ada = AdalineSGD(n_iter=15, eta=0.01, random_state=1)

ada.fit(X_std, y)

plot_decision_regions(X_std, y, classifier=ada)

plt.title('Adaline - Stochastic Gradient Descent')

plt.xlabel('sepal length [standardized]')

plt.ylabel('petal length [standardized]')

plt.legend(loc='upper left')

plt.show()

plt.plot(range(1, len(ada.cost_) + 1), ada.cost_, marker='o')

plt.xlabel('Epochs')

plt.ylabel('Average Cost')

plt.show()