梯度下降法在linear regression中的应用

% this script is a demo to implement linear regression, linear regression % with a linear combination of basis functions, and ridge regression--least % squares with L2 regulariztion clear all load test load training % plot training data hold on plot(Xtrain,Ytrain,'linestyle','.','linewidth',2,'color','r'); final_w = cell(3,1); %% using normal linear regression to fit the data with gradient descent method % initializing weighted parameters y=Ytrain; n = length(y); % the length of labels w0 = ones(size(Xtrain,2)+1,1); w = w0; count = 1; W(:,count)=w; X = [Xtrain ones(size(Xtrain))]; %%% computation of the objective function value at w; variable name is objective objective = mean((y - X*w).^2); iter = 0; %%% stopping criteria convergence = 0; maxiter = 1000; tol = 1e-6; %%% parameters for step size selection , here using backtracking line %%% search for choosing stepsize alpha = 0.1; % parameter for Armijo rule beta = 0.95; t0 = 5; t=zeros(5,1); t(1)=t0; while(convergence==0 && iter<maxiter) %%% the computation of the gradient, 'grad' and the descent direction 'dir' grad = -2/n*X'*(y-X*w); dir = -grad; %%% step size selection stepsize = 1; wnew = w + stepsize*dir; %%% computation of the objective function value at 'wnew'; variable name is newobjective newobjective = mean((y-X*wnew).^2); %%% backtracking line search while newobjective > objective + alpha* stepsize* (grad'*dir) % Wolfe conditions stepsize = stepsize * beta; wnew = w + stepsize*dir; %%% computation of the objective function value at 'wnew'; variable name is newobjective newobjective = mean((y-X*wnew).^2); end %%% Update the variable convergence if the stopping criterion is %%% met so that the loop is terminated after this iteration. if norm(grad)<tol convergence = 1; end w = wnew; objective = newobjective; iter = iter+1; if iter==t(count) t(count+1)=2*t(count); count = count + 1; W(:,count) = w; end end count=count+1; W(:,count)=w; final_w{1} = w; x=0:0.01:1; x=x'; x1 = [x ones(size(x))]; colors = jet(count); n=size(x,1); fs=zeros(n,1); for i=1:count fs(:,i)=x1*W(:,i); plot(x,fs(:,i),'linestyle','-','linewidth',2,'color',colors(i,:)); end xlabel('x');ylabel('y'); legend('training data','initial curve',['iter=' num2str(t(1))],['iter=' num2str(t(2))],... ['iter=' num2str(t(3))],['iter=' num2str(t(4))],['iter=' num2str(t(5))],'final curve'); title('normal linear regression'); hold off %% using linear regression with a linear combination of basis functions to fit the data k=2; % the dimension of basis function Phi = Basis(Xtrain, k); % initializing weighted parameters w w0 = ones(size(Phi,2),1); w = w0; %%% computation of the objective function value at w; variable name is objective objective = mean((y-Phi*w).^2); % ridge regression problem iter = 0; %%% stopping criteria convergence = 0; maxiter = 1000; tol = 1e-6; %%% parameters for step size selection , here using backtracking line %%% search for choosing stepsize alpha = 0.1; % parameter for Armijo rule beta = 0.95; while(convergence==0 && iter<maxiter) %%% the computation of the gradient, 'grad' and the descent direction 'dir' grad = -2/n*Phi'*(y-Phi*w); dir = -grad; %%% step size selection stepsize = 1; wnew = w + stepsize*dir; %%% computation of the objective function value at 'wnew'; variable name is newobjective newobjective = mean((y-Phi*wnew).^2); %%% backtracking line search while newobjective > objective + alpha* stepsize* (grad'*dir) % Wolfe conditions stepsize = stepsize * beta; wnew = w + stepsize*dir; %%% computation of the objective function value at 'wnew'; variable name is newobjective newobjective = mean((y-Phi*wnew).^2); end %%% Update the variable convergence if the stopping criterion is %%% met so that the loop is terminated after this iteration. if norm(grad)<tol convergence = 1; end w = wnew; objective = newobjective; iter = iter+1; end x=0:0.01:1; x=x'; n=size(x,1); fs=zeros(n,k); figure hold on % plot training data plot(Xtrain, Ytrain, 'linestyle','.','linewidth',2,'color','r'); phi=Basis(x,k); fs=phi*w; plot(x,fs,'linestyle','-','linewidth',2,'color','g'); xlabel('x');ylabel('y'); legend('training data','final curve'); title('linear regression with a linear combination of basis functions'); hold off final_w{2} = w; %% using ridge regression to fit the data with gradient descent method lambda = 0.02; k=2; % the dimension of basis function Phi = Basis(Xtrain, k); % initializing weighted parameters w w0 = ones(size(Phi,2),1); w = w0; %%% computation of the objective function value at w; variable name is objective objective = mean((y-Phi*w).^2)+lambda*norm(w)^2; % ridge regression problem iter = 0; %%% stopping criteria convergence = 0; maxiter = 1000; tol = 1e-6; %%% parameters for step size selection , here using backtracking line %%% search for choosing stepsize alpha = 0.1; % parameter for Armijo rule beta = 0.95; W = zeros(length(w),1); count = 1; W(:,count)=w; while(convergence==0 && iter<maxiter) %%% the computation of the gradient, 'grad' and the descent direction 'dir' grad = -2/n*Phi'*(y-Phi*w)+2*lambda*w; dir = -grad; %%% step size selection stepsize = 1; wnew = w + stepsize*dir; %%% computation of the objective function value at 'wnew'; variable name is newobjective newobjective = mean((y-Phi*wnew).^2)+lambda*norm(wnew)^2; %%% backtracking line search while newobjective > objective + alpha* stepsize* (grad'*dir) % Wolfe conditions stepsize = stepsize * beta; wnew = w + stepsize*dir; %%% computation of the objective function value at 'wnew'; variable name is newobjective newobjective = mean((y-Phi*wnew).^2)+lambda*norm(wnew)^2; end %%% Update the variable convergence if the stopping criterion is %%% met so that the loop is terminated after this iteration. if norm(grad)<tol convergence = 1; end w = wnew; objective = newobjective; iter = iter+1; if mod(iter,50)==0 count=count+1; W(:,count)=w; end end count=count+1; W(:,count)=w; final_w{3} = w; colors=jet(30); % set the color space x=0:0.01:1; x=x'; n=size(x,1); fs=zeros(n,k); figure hold on % plot training data plot(Xtrain, Ytrain, 'linestyle','.','linewidth',2,'color','r'); for i=1:count phi=Basis(x,k); fs(:,i)=phi*W(:,i); plot(x,fs(:,i),'linestyle','-','linewidth',2,'color',colors(i,:)); end xlabel('x');ylabel('y'); legend('training data','initial curve','iter=50',... 'final curve'); title('ridge regression'); %% compare these three different regression method figure; hold on % plot train data plot(Xtrain, Ytrain, 'linestyle','.','linewidth',2,'color','r'); % plot the curve of normal linear regression plot(x,x1*final_w{1},'linestyle','-','linewidth',2,'color','g'); % plot the curve of linear regression with a combinaiton of basis functions plot(x,phi*final_w{2},'linestyle','-','linewidth',2,'color','y'); % plot the curve of ridge regression plot(x,phi*final_w{3},'linestyle','-','linewidth',2,'color','b'); xlabel('x');ylabel('y'); legend('training data', 'normal linear regression','linear regression with a combinaiton of basis functions','ridge regressi