Get Dataset & Create Workspace¶
In [1]:
import os
import tarfile
from six.moves import urllib
import pandas as pd
import numpy as np
In [2]:
DOWNLOAD_ROOT = "https://raw.githubusercontent.com/ageron/handson-ml/master/"
HOUSING_PATH = "datasets/housing"
HOUSING_URL = DOWNLOAD_ROOT + HOUSING_PATH + "/housing.tgz"
In [3]:
def fetch_housing_data(
housing_url=HOUSING_URL,
housing_path=HOUSING_PATH):
# create datasets/housing directory if needed
if not os.path.isdir(housing_path):
os.makedirs(housing_path)
tgz_path = os.path.join(housing_path, "housing.tgz")
# retrieve tarfile
urllib.request.urlretrieve(housing_url, tgz_path)
# extract tarfile & close path
housing_tgz = tarfile.open(tgz_path)
housing_tgz.extractall(path=housing_path)
housing_tgz.close()
In [4]:
def load_housing_data(
housing_path=HOUSING_PATH):
csv_path = os.path.join(housing_path, "housing.csv")
return pd.read_csv(csv_path)
In [5]:
# do it
#fetch_housing_data() -- already downloaded - static dataset
housing = load_housing_data()
Data structure - quick peek¶
In [6]:
housing.head()
Out[6]:
So... what's in the dataset?¶
In [7]:
# housing is a Pandas DataFrame.
# untouched datafile: 20640 records, 10 cols (9 float, 1 text)
housing.info()
In [8]:
# let's see if ocean_proximity can be lumped into categories:
housing['ocean_proximity'].value_counts()
Out[8]:
In [9]:
# percentiles analysis of each feature
housing.describe()
Out[9]:
In [10]:
# feature histograms
%matplotlib inline
import matplotlib.pyplot as plt
housing.hist(bins=50, figsize=(20,15))
Out[10]:
Create a test set¶
In [11]:
# split dataset into training (80%) and test (20%) subsets
import numpy as np
def split_train_test(
data, test_ratio):
shuffled_indices = np.random.permutation(len(data))
test_set_size = int(len(data) * test_ratio)
test_indices = shuffled_indices[:test_set_size]
train_indices = shuffled_indices[test_set_size:]
return data.iloc[train_indices], data.iloc[test_indices]
In [12]:
train_set, test_set = split_train_test(housing, 0.2)
In [13]:
print(len(train_set), "train +", len(test_set), "test")
In [14]:
# create method for ensuring consistent test sets across multiple runs
# (new test sets won't contain instances in previous training sets.)
# example method:
# compute hash of each instance
# keep only the last byte
# include instance in test set if value < 51 (20% of 256)
import hashlib
def test_set_check(
identifier, test_ratio, hash):
return hash(np.int64(identifier)).digest()[-1] < 256 * test_ratio
def split_train_test_by_id(
data, test_ratio, id_column, hash=hashlib.md5):
ids = data[id_column]
in_test_set = ids.apply(
lambda id_: test_set_check(
id_, test_ratio, hash))
return data.loc[~in_test_set], data.loc[in_test_set]
In [15]:
# housing dataset doesn't have ID attribute,
# so let's add an index to it.
housing_with_id = housing.reset_index()
train_set, test_set = split_train_test_by_id(
housing_with_id, 0.2, "index")
In [16]:
train_set.head()
Out[16]:
In [17]:
test_set.head()
Out[17]:
In [18]:
# a better index:
# let's use longitude & latitude to build stable identifier
housing_with_id["id"] = housing["longitude"] * 1000 + housing["latitude"]
train_set, test_set = split_train_test_by_id(
housing_with_id, 0.2, "id")
In [19]:
train_set.head()
Out[19]:
In [20]:
test_set.head()
Out[20]:
In [21]:
# another option: scikit-learn splitters
from sklearn.model_selection import train_test_split
train_set, test_set = train_test_split(
housing, test_size=0.2, random_state=42)
In [22]:
test_set.head()
Out[22]:
In [23]:
train_set.head()
Out[23]:
In [24]:
# does sampling plan have a sampling bias?
# each strata in test dataset should mimic reality
housing['median_income'].hist(bins=5)
Out[24]:
In [25]:
housing.describe()
Out[25]:
In [26]:
housing["income_cat"]=np.ceil(housing["median_income"]/1.5)
housing["income_cat"].where(housing["income_cat"]<5, 5.0, inplace=True)
In [27]:
housing.describe()
Out[27]:
In [28]:
from sklearn.model_selection import StratifiedShuffleSplit
split = StratifiedShuffleSplit(
n_splits=1, test_size=0.2, random_state=42)
for train_index, test_index in split.split(housing, housing["income_cat"]):
strat_train_set = housing.loc[train_index]
strat_test_set = housing.loc[test_index]
In [29]:
# review income category proportions
housing["income_cat"].value_counts() / len(housing)
Out[29]:
In [30]:
# remove income_cat attribute (return dataset to original state)
for set in (strat_train_set, strat_test_set):
set.drop(["income_cat"], axis=1, inplace=True)
Visualization¶
In [31]:
housing = strat_train_set.copy()
# first: basic geographic distribution
housing.plot(kind="scatter", x="longitude", y="latitude", alpha=0.1)
Out[31]:
In [32]:
# next: housing prices.
# color = price
# radius = population
# use predefined "jet" color map
housing.plot(
kind="scatter",
x="longitude",
y="latitude",
alpha=0.4,
#s=housing["population"].apply(lambda n: n/100),
s=housing["population"]/100,
label="population",
c="median_house_value",
cmap=plt.get_cmap("jet"),
colorbar=True,
)
plt.legend()
Out[32]:
Correlations¶
In [33]:
# next: look for correlatons to median house value.
corr_matrix = housing.corr()
corr_matrix['median_house_value'].sort_values(ascending=False)
Out[33]:
In [34]:
# another way of looking for correlations: scatter_matrix
# focus on top 3 factors from above
from pandas.tools.plotting import scatter_matrix
attributes = ["median_house_value", "median_income", "total_rooms",
"housing_median_age"]
scatter_matrix(housing[attributes], figsize=(12, 8))
Out[34]:
In [35]:
# median house value to median income seems to be the most promising.
# let's zoom in.
housing.plot(
kind="scatter", x="median_income", y="median_house_value",
alpha=0.1)
Out[35]:
In [36]:
# combine some attributes to create more useful ones
# then rebuild the correlation matrix.
housing["rooms_per_household"] = housing["total_rooms"]/housing["households"]
housing["bedrooms_per_room"] = housing["total_bedrooms"]/housing["total_rooms"]
housing["population_per_household"]=housing["population"]/housing["households"]
corr_matrix = housing.corr()
corr_matrix['median_house_value'].sort_values(ascending=False)
# *** NOTE: rooms_per_household corr (in book) show more improvement, ~0.199
# compared to our 0.146. Not sure of root cause yet. ***
Out[36]:
Data Cleanup¶
In [37]:
# revert to clean copy of stratified training dataset
# separate predictors from labels
housing = strat_train_set.drop("median_house_value", axis=1)
housing_labels = strat_train_set["median_house_value"].copy()
In [38]:
# 'total bedrooms' has some missing values - fix
# can use DataFrame dropna(), drop(), fillna()
# use Scikit-Learn class to handle missing values
from sklearn.preprocessing import Imputer
imputer = Imputer(strategy="median")
In [39]:
# drop ocean_proximity attribute, since it's non-numeric.
# then fit to training data.
housing_num = housing.drop("ocean_proximity", axis=1)
imputer.fit(housing_num)
Out[39]:
In [40]:
# now what do we have?
imputer.statistics_
Out[40]:
In [41]:
housing_num.median().values
Out[41]:
In [42]:
# update training set by replacing missing values with learned medians
X = imputer.transform(housing_num)
In [43]:
pd.DataFrame(X, columns=housing_num.columns).info()
In [44]:
# convert ocean_proximity feature to numbers using LabelEncoder.
from sklearn.preprocessing import LabelEncoder
encoder = LabelEncoder()
housing_cat = housing['ocean_proximity']
housing_cat_encoded = encoder.fit_transform(housing_cat)
housing_cat_encoded
Out[44]:
In [45]:
# how is 'ocean_proximity' mapped?
print(encoder.classes_)
In [46]:
# a better solution for categorical data: one-hot encoding
from sklearn.preprocessing import OneHotEncoder
encoder = OneHotEncoder()
# output = SciPy sparse matrix, better for memory usage
# if you need a dense NumPy array, call toarray()
housing_cat_1hot = encoder.fit_transform(housing_cat_encoded.reshape(-1,1))
housing_cat_1hot
Out[46]:
Label Binarization:¶
- A shortcut (text categories => integer categories => one-hot vectors)
In [47]:
from sklearn.preprocessing import LabelBinarizer
encoder = LabelBinarizer()
housing_cat_1hot = encoder.fit_transform(housing_cat)
housing_cat_1hot
Out[47]:
Custom Transformers:¶
- Create your own using SciKit-Learn classes
- implement fit(), transform() and fit_transform() methods
- (fit_transform comes for free by using TransformerMixin as a base class.)
In [48]:
from sklearn.base import BaseEstimator, TransformerMixin
rooms_ix, bedrooms_ix, population_ix, household_ix = 3, 4, 5, 6
class CombinedAttributesAdder(BaseEstimator, TransformerMixin):
def __init__(self, add_bedrooms_per_room = True): # no *args or **kargs
self.add_bedrooms_per_room = add_bedrooms_per_room
def fit(self, X, y=None):
return self # nothing else to do
def transform(self, X, y=None):
rooms_per_household = X[:, rooms_ix] / X[:, household_ix]
population_per_household = X[:, population_ix] / X[:, household_ix]
if self.add_bedrooms_per_room:
bedrooms_per_room = X[:, bedrooms_ix] / X[:, rooms_ix]
return np.c_[X,
rooms_per_household,
population_per_household,
bedrooms_per_room]
else:
return np.c_[X,
rooms_per_household,
population_per_household]
attr_adder = CombinedAttributesAdder(add_bedrooms_per_room=False)
housing_extra_attribs = attr_adder.transform(housing.values)
Feature Scaling¶
- Min-max scaling (normalization) = shift & rescale to [0,1]
- SciKit MinMaxScaler will do this for you.
- Standardization subtracts mean & divides by variance - result has unit variance
- SciKit StandardScaler does this for you.
Pipelining¶
- SciKit Pipeline class helps to standardize the sequence of transforms you need for your project.
- Pipelines = list of estimator steps. All but the last must be transformers (they must have fit_transform() method.)
In [49]:
# "DataFrameSelector" is a custom transformer class.
# grabs the specified feature, drops the rest, converts the DF into a NumPy array.
from sklearn.base import BaseEstimator, TransformerMixin
class DataFrameSelector(BaseEstimator, TransformerMixin):
def __init__ (self, attribute_names):
self.attribute_names = attribute_names
def fit (self, X, y=None):
return self
def transform (self, X):
return X[self.attribute_names].values
In [50]:
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.preprocessing import StandardScaler
num_attribs = list(housing_num)
cat_attribs = ['ocean_proximity']
num_pipeline = Pipeline([
('selector', DataFrameSelector(num_attribs)),
('imputer', Imputer(strategy="median")),
('attribs_adder', CombinedAttributesAdder()),
('std_scaler', StandardScaler()),
])
cat_pipeline = Pipeline([
('selector', DataFrameSelector(cat_attribs)),
('label_binarizer', LabelBinarizer()),
])
full_pipeline = FeatureUnion(transformer_list =[
('num_pipeline', num_pipeline),
('cat_pipeline', cat_pipeline)
])
In [51]:
# let's try it out:
housing_prepared = full_pipeline.fit_transform(housing)
housing_prepared
Out[51]:
In [52]:
housing_prepared.shape
Out[52]:
- Note: pip3 install sklearn-pandas => gets a DataFrameMapper class
Model Selection & Training¶
In [53]:
# let's start with a linear regression
from sklearn.linear_model import LinearRegression
lin_reg = LinearRegression()
lin_reg.fit(housing_prepared, housing_labels)
Out[53]:
In [54]:
# first try. NOT very accurate.
some_data = housing.iloc[:5]
some_labels = housing_labels.iloc[:5]
some_data_prepared = full_pipeline.transform(some_data)
print ("predictions:\t", lin_reg.predict(some_data_prepared))
print ("labels:\t", list(some_labels))
In [55]:
# why? look at RMSE on whole training set.
from sklearn.metrics import mean_squared_error
housing_predictions = lin_reg.predict(housing_prepared)
lin_mse = mean_squared_error(housing_labels, housing_predictions)
lin_rmse = np.sqrt(lin_mse)
print ("typical prediction error:\t", lin_rmse)
In [56]:
# Hmmm. Not good. Underfit situation.
# Let's try a more powerful model, like a Decision Tree.
from sklearn.tree import DecisionTreeRegressor
tree_reg = DecisionTreeRegressor()
tree_reg.fit(housing_prepared, housing_labels)
Out[56]:
In [57]:
# Zero error? No way...
housing_predictions = tree_reg.predict(housing_prepared)
tree_mse = mean_squared_error(housing_labels, housing_predictions)
tree_rmse = np.sqrt(tree_mse)
print ("typical prediction error:\t", tree_rmse)
In [58]:
# Use K-fold cross-validation
# Train & eval Decision Tree model against 10 splits of training dataset
# Returns 10 evaluation scores.
from sklearn.model_selection import cross_val_score
scores = cross_val_score(
tree_reg,
housing_prepared,
housing_labels,
scoring="neg_mean_squared_error",
cv=10)
rmse_scores = np.sqrt(-scores)
def display_scores(scores):
print("Scores:", scores)
print("Mean:", scores.mean())
print("Standard deviation:", scores.std())
display_scores(rmse_scores)
In [59]:
# So, Decision Tree RMSE: mean ~71097, stdev 2165 (still sucks.)
# compare to earlier Linear Regression:
lin_scores = cross_val_score(
lin_reg,
housing_prepared,
housing_labels,
scoring="neg_mean_squared_error",
cv=10)
lin_rmse_scores = np.sqrt(-lin_scores)
display_scores(lin_rmse_scores)
In [60]:
# Yep, DT overfit is just about as bad. (RMSE mean 69052, stdev 2731)
# Let's try a RandomForest.
from sklearn.ensemble import RandomForestRegressor
forest_reg = RandomForestRegressor()
forest_reg.fit(housing_prepared, housing_labels)
forest_scores = cross_val_score(
forest_reg,
housing_prepared,
housing_labels,
scoring="neg_mean_squared_error",
cv=10)
forest_rmse_scores = np.sqrt(-forest_scores)
display_scores(forest_rmse_scores)
In [61]:
# OK, RandomForest is a little better.
# RMSE mean ~52495, stdev ~1569
Fine-Tuning Model with Grid Search of Hyperparameters¶
In [62]:
from sklearn.model_selection import GridSearchCV
param_grid = [
{'n_estimators': [3, 10, 30],
'max_features': [2, 4, 6, 8]},
{'bootstrap': [False], # bootstrap = True = default setting
'n_estimators': [3, 10],
'max_features': [2, 3, 4]},
]
forest_reg = RandomForestRegressor()
grid_search = GridSearchCV(
forest_reg,
param_grid,
cv=5,
scoring = 'neg_mean_squared_error')
grid_search.fit(housing_prepared, housing_labels)
Out[62]:
In [63]:
# Best combination of parameters?
grid_search.best_params_
Out[63]:
In [64]:
# Best estimator?
grid_search.best_estimator_
Out[64]:
In [65]:
# Evaluation scores:
cvres = grid_search.cv_results_
for mean_score, params in zip(cvres["mean_test_score"],
cvres["params"]):
print(np.sqrt(-mean_score), params)
In [66]:
# best solution:
# max_features = 6, n_estimators = 30 (RMSE ~49,960)
In [67]:
feature_importances = grid_search.best_estimator_.feature_importances_
feature_importances
Out[67]:
In [68]:
# display feature "importance" scores next to their names:
extra_attribs = ["rooms_per_hhold", "pop_per_hhold", "bedrooms_per_room"]
cat_one_hot_attribs = list(encoder.classes_)
attributes = num_attribs + extra_attribs + cat_one_hot_attribs
sorted(zip(feature_importances, attributes), reverse=True)
Out[68]:
Time to Eval System on Test dataset¶
In [69]:
final_model = grid_search.best_estimator_
X_test = strat_test_set.drop("median_house_value", axis=1)
y_test = strat_test_set["median_house_value"].copy()
X_test_prepared = full_pipeline.transform(X_test)
final_predictions = final_model.predict(X_test_prepared)
final_mse = mean_squared_error(y_test, final_predictions)
final_rmse = np.sqrt(final_mse)
final_rmse
Out[69]:
In [ ]:
