machine learning Neural Networks for Handwritten Digit Recogn

""" lab_utils_common contains common routines and variable definitions used by all the labs in this week. by contrast, specific, large plotting routines will be in separate files and are generally imported into the week where they are used. those files will import this file """ import copy import math import numpy as np import matplotlib.pyplot as plt from matplotlib.patches import FancyArrowPatch from ipywidgets import Output from matplotlib.widgets import Button, CheckButtons np.set_printoptions(precision=2) dlc = dict(dlblue = '#0096ff', dlorange = '#FF9300', dldarkred='#C00000', dlmagenta='#FF40FF', dlpurple='#7030A0', dldarkblue = '#0D5BDC', dlmedblue='#4285F4') dlblue = '#0096ff'; dlorange = '#FF9300'; dldarkred='#C00000'; dlmagenta='#FF40FF'; dlpurple='#7030A0'; dldarkblue = '#0D5BDC'; dlmedblue='#4285F4' dlcolors = [dlblue, dlorange, dldarkred, dlmagenta, dlpurple] plt.style.use('./deeplearning.mplstyle') def sigmoid(z): """ Compute the sigmoid of z Parameters ---------- z : array_like A scalar or numpy array of any size. Returns ------- g : array_like sigmoid(z) """ z = np.clip( z, -500, 500 ) # protect against overflow g = 1.0/(1.0+np.exp(-z)) return g ########################################################## # Regression Routines ########################################################## def predict_logistic(X, w, b): """ performs prediction """ return sigmoid(X @ w + b) def predict_linear(X, w, b): """ performs prediction """ return X @ w + b def compute_cost_logistic(X, y, w, b, lambda_=0, safe=False): """ Computes cost using logistic loss, non-matrix version Args: X (ndarray): Shape (m,n) matrix of examples with n features y (ndarray): Shape (m,) target values w (ndarray): Shape (n,) parameters for prediction b (scalar): parameter for prediction lambda_ : (scalar, float) Controls amount of regularization, 0 = no regularization safe : (boolean) True-selects under/overflow safe algorithm Returns: cost (scalar): cost """ m,n = X.shape cost = 0.0 for i in range(m): z_i = np.dot(X[i],w) + b #(n,)(n,) or (n,) () if safe: #avoids overflows cost += -(y[i] * z_i ) + log_1pexp(z_i) else: f_wb_i = sigmoid(z_i) #(n,) cost += -y[i] * np.log(f_wb_i) - (1 - y[i]) * np.log(1 - f_wb_i) # scalar cost = cost/m reg_cost = 0 if lambda_ != 0: for j in range(n): reg_cost += (w[j]**2) # scalar reg_cost = (lambda_/(2*m))*reg_cost return cost + reg_cost def log_1pexp(x, maximum=20): ''' approximate log(1+exp^x) https://stats.stackexchange.com/questions/475589/numerical-computation-of-cross-entropy-in-practice Args: x : (ndarray Shape (n,1) or (n,) input out : (ndarray Shape matches x output ~= np.log(1+exp(x)) ''' out = np.zeros_like(x,dtype=float) i = x <= maximum ni = np.logical_not(i) out[i] = np.log(1 + np.exp(x[i])) out[ni] = x[ni] return out def compute_cost_matrix(X, y, w, b, logistic=False, lambda_=0, safe=True): """ Computes the cost using using matrices Args: X : (ndarray, Shape (m,n)) matrix of examples y : (ndarray Shape (m,) or (m,1)) target value of each example w : (ndarray Shape (n,) or (n,1)) Values of parameter(s) of the model b : (scalar ) Values of parameter of the model verbose : (Boolean) If true, print out intermediate value f_wb Returns: total_cost: (scalar) cost """ m = X.shape[0] y = y.reshape(-1,1) # ensure 2D w = w.reshape(-1,1) # ensure 2D if logistic: if safe: #safe from overflow z = X @ w + b #(m,n)(n,1)=(m,1) cost = -(y * z) + log_1pexp(z) cost = np.sum(cost)/m # (scalar) else: f = sigmoid(X @ w + b) # (m,n)(n,1) = (m,1) cost = (1/m)*(np.dot(-y.T, np.log(f)) - np.dot((1-y).T, np.log(1-f))) # (1,m)(m,1) = (1,1) cost = cost[0,0] # scalar else: f = X @ w + b # (m,n)(n,1) = (m,1) cost = (1/(2*m)) * np.sum((f - y)**2) # scalar reg_cost = (lambda_/(2*m)) * np.sum(w**2) # scalar total_cost = cost + reg_cost # scalar return total_cost # scalar def compute_gradient_matrix(X, y, w, b, logistic=False, lambda_=0): """ Computes the gradient using matrices Args: X : (ndarray, Shape (m,n)) matrix of examples y : (ndarray Shape (m,) or (m,1)) target value of each example w : (ndarray Shape (n,) or (n,1)) Values of parameters of the model b : (scalar ) Values of parameter of the model logistic: (boolean) linear if false, logistic if true lambda_: (float) applies regularization if non-zero Returns dj_dw: (array_like Shape (n,1)) The gradient of the cost w.r.t. the parameters w dj_db: (scalar) The gradient of the cost w.r.t. the parameter b """ m = X.shape[0] y = y.reshape(-1,1) # ensure 2D w = w.reshape(-1,1) # ensure 2D f_wb = sigmoid( X @ w + b ) if logistic else X @ w + b # (m,n)(n,1) = (m,1) err = f_wb - y # (m,1) dj_dw = (1/m) * (X.T @ err) # (n,m)(m,1) = (n,1) dj_db = (1/m) * np.sum(err) # scalar dj_dw += (lambda_/m) * w # regularize # (n,1) return dj_db, dj_dw # scalar, (n,1) def gradient_descent(X, y, w_in, b_in, alpha, num_iters, logistic=False, lambda_=0, verbose=True, Trace=True): """ Performs batch gradient descent to learn theta. Updates theta by taking num_iters gradient steps with learning rate alpha Args: X (ndarray): Shape (m,n) matrix of examples y (ndarray): Shape (m,) or (m,1) target value of each example w_in (ndarray): Shape (n,) or (n,1) Initial values of parameters of the model b_in (scalar): Initial value of parameter of the model logistic: (boolean) linear if false, logistic if true lambda_: (float) applies regularization if non-zero alpha (float): Learning rate num_iters (int): number of iterations to run gradient descent Returns: w (ndarray): Shape (n,) or (n,1) Updated values of parameters; matches incoming shape b (scalar): Updated value of parameter """ # An array to store cost J and w's at each iteration primarily for graphing later J_history = [] w = copy.deepcopy(w_in) #avoid modifying global w within function b = b_in w = w.reshape(-1,1) #prep for matrix operations y = y.reshape(-1,1) last_cost = np.Inf for i in range(num_iters): # Calculate the gradient and update the parameters dj_db,dj_dw = compute_gradient_matrix(X, y, w, b, logistic, lambda_) # Update Parameters using w,