<!DOCTYPE html>
In [45]:
import tensorflow as tf
print(tf.__version__)
Validation, regularisation and callbacks¶
Load the data¶
In [46]:
# Load the diabetes dataset
from sklearn.datasets import load_diabetes
diabetes_dataset = load_diabetes()
print(diabetes_dataset['DESCR'])
In [47]:
# Save the input and target variables
#print(diabetes_dataset.keys())
data = diabetes_dataset["data"]
target = diabetes_dataset["target"]
In [48]:
# Normalise the target data (this will make clearer training curves)
targets = (target - target.mean(axis=0)) / target.std()
targets
Out[48]:
In [49]:
# Split the data into train and test sets
from sklearn.model_selection import train_test_split
train_data, test_data, train_targets, test_targets = train_test_split(data, targets, test_size=0.1)
Train a feedforward neural network model¶
In [50]:
print(train_data.shape)
print(test_data.shape)
print(train_targets.shape)
print(test_targets.shape)
In [51]:
# Build the model
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
def get_model():
model = Sequential([
Dense(128, activation="relu", input_shape=(train_data.shape[1],)),
Dense(128, activation="relu"),
Dense(128, activation="relu"),
Dense(128, activation="relu"),
Dense(128, activation="relu"),
Dense(128, activation="relu"),
Dense(1)
])
return model
model = get_model()
In [52]:
# Print the model summary
model.summary()
In [53]:
# Compile the model
model.compile(optimizer="adam", loss="mse", metrics=["mae"])
In [54]:
# Train the model, with some of the data reserved for validation
history = model.fit(train_data, train_targets, epochs=100, validation_split=0.15, batch_size=64, verbose=False)
In [55]:
# Evaluate the model on the test set
model.evaluate(test_data, test_targets,verbose=2)
Out[55]:
Plot the learning curves¶
In [56]:
import matplotlib.pyplot as plt
%matplotlib inline
In [57]:
# Plot the training and validation loss
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('Loss vs. epochs')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Training', 'Validation'], loc='upper right')
plt.show()
Adding regularisation with weight decay and dropout¶
In [58]:
from tensorflow.keras.layers import Dropout
from tensorflow.keras import regularizers
In [74]:
def get_regularised_model(wd, rate):
model = Sequential([
Dense(128, kernel_regularizer=tf.keras.regularizers.l2(wd), activation="relu", input_shape=(train_data.shape[1],)),
Dropout(rate),
Dense(128, kernel_regularizer=tf.keras.regularizers.l2(wd), activation="relu"),
Dropout(rate),
Dense(128, kernel_regularizer=tf.keras.regularizers.l2(wd), activation="relu"),
Dropout(rate),
Dense(128, kernel_regularizer=tf.keras.regularizers.l2(wd), activation="relu"),
Dropout(rate),
Dense(128, kernel_regularizer=tf.keras.regularizers.l2(wd), activation="relu"),
Dropout(rate),
Dense(128, kernel_regularizer=tf.keras.regularizers.l2(wd), activation="relu"),
Dropout(rate),
Dense(1)
])
return model
In [75]:
# Re-build the model with weight decay and dropout layers
model = get_regularised_model(1e-5,0.3)
In [76]:
# Compile the model
model.compile(optmizer="adam", loss="mse", metrics=["mae"])
In [77]:
# Train the model, with some of the data reserved for validation
history = model.fit(train_data, train_targets, epochs=100, validation_split=0.15, batch_size=64, verbose=False)
In [78]:
# Evaluate the model on the test set
model.evaluate(test_data, test_targets, verbose=2)
Out[78]:
Plot the learning curves¶
In [79]:
# Plot the training and validation loss
import matplotlib.pyplot as plt
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('Loss vs. epochs')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Training', 'Validation'], loc='upper right')
plt.show()
Example training callback¶
In [92]:
# Write a custom callback
from tensorflow.keras.callbacks import Callback
class TrainingCallback(Callback):
def on_train_begin(self, logs=None):
print("Starting training....")
def on_epoch_begin(self, epoch, logs=None):
print(f"Starting epoch {epoch}")
def on_train_batch_begin(self, batch,logs=None):
print(f"Training: Starting batch {batch}")
def on_train_batch_end(self, batch, logs=None):
print(f"Training: Finished batch {batch}")
def on_epoch_end(self, epoch, logs=None):
print(f"Finished epoch {epoch}")
def on_train_end(self, logs=None):
print("Finished trainging!")
In [93]:
# Re-build the model
model = get_regularised_model(1e-5, 0.3)
In [94]:
# Compile the model
model.compile(optimizer="adam", loss="mse")
Train the model with the callback¶
In [98]:
# Train the model, with some of the data reserved for validation
model.fit(train_data, train_targets, epochs=3, batch_size=128, verbose=False, callbacks=[TrainingCallback()])
Out[98]:
In [105]:
# Evaluate the model
class TestingCallback(Callback):
def on_test_begin(self, logs=None):
print("Starting testing....")
def on_test_batch_begin(self, batch,logs=None):
print(f"Testing: Starting batch {batch}")
def on_test_batch_end(self, batch, logs=None):
print(f"Testing: Finished batch {batch}")
def on_test_end(self, logs=None):
print("Finished testing!")
model.evaluate(test_data, test_targets, verbose=False, callbacks=[TestingCallback()])
Out[105]:
In [106]:
# Make predictions with the model
class PredictionCallback(Callback):
def on_predict_begin(self, logs=None):
print("Starting Prediction....")
def on_predict_batch_begin(self, batch,logs=None):
print(f"Prediction: Starting batch {batch}")
def on_predict_batch_end(self, batch, logs=None):
print(f"Prediction: Finished batch {batch}")
def on_predict_end(self, logs=None):
print("Finished Prediction!")
model.predict(test_data, verbose=False, callbacks=[PredictionCallback()])
Out[106]:
Re-train the models with early stopping¶
In [129]:
# Re-train the unregularised model
unregularised_model = get_model()
unregularised_model.compile(optimizer="adam", loss="mse")
unreg_history = unregularised_model.fit(train_data, train_targets, epochs=100,
validation_split=0.15, batch_size=64,
callbacks=[tf.keras.callbacks.EarlyStopping(monitor="val_loss",min_delta=0.01,patience=10,mode="min")])
In [130]:
# Evaluate the model on the test set
unregularised_model.evaluate(test_data, test_targets, verbose=2)
Out[130]:
In [133]:
# Re-train the regularised model
regularised_model = get_regularised_model(1e-8, 0.2)
regularised_model.compile(optimizer="adam", loss="mse")
reg_history = regularised_model.fit(train_data, train_targets, epochs=100,
validation_split=0.15, batch_size=64,
callbacks=[tf.keras.callbacks.EarlyStopping(monitor="val_loss",min_delta=0.01,patience=10,mode="min")])
In [134]:
# Evaluate the model on the test set
regularised_model.evaluate(test_data, test_targets, verbose=2)
Out[134]:
Plot the learning curves¶
In [135]:
# Plot the training and validation loss
import matplotlib.pyplot as plt
fig = plt.figure(figsize=(12, 5))
fig.add_subplot(121)
plt.plot(unreg_history.history['loss'])
plt.plot(unreg_history.history['val_loss'])
plt.title('Unregularised model: loss vs. epochs')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Training', 'Validation'], loc='upper right')
fig.add_subplot(122)
plt.plot(reg_history.history['loss'])
plt.plot(reg_history.history['val_loss'])
plt.title('Regularised model: loss vs. epochs')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Training', 'Validation'], loc='upper right')
plt.show()
In [ ]: