% this script is a demo to implement linear regression, linear regression
% with a linear combination of basis functions, and ridge regression--least
% squares with L2 regulariztion
clear all
load test
load training
% plot training data
hold on
plot(Xtrain,Ytrain,'linestyle','.','linewidth',2,'color','r');
final_w = cell(3,1);
%% using normal linear regression to fit the data with gradient descent method
% initializing weighted parameters
y=Ytrain;
n = length(y); % the length of labels
w0 = ones(size(Xtrain,2)+1,1);
w = w0;
count = 1;
W(:,count)=w;
X = [Xtrain ones(size(Xtrain))];
%%% computation of the objective function value at w; variable name is objective
objective = mean((y - X*w).^2);
iter = 0;
%%% stopping criteria
convergence = 0;
maxiter = 1000;
tol = 1e-6;
%%% parameters for step size selection , here using backtracking line
%%% search for choosing stepsize
alpha = 0.1; % parameter for Armijo rule
beta = 0.95;
t0 = 5;
t=zeros(5,1); t(1)=t0;
while(convergence==0 && iter<maxiter)
%%% the computation of the gradient, 'grad' and the descent direction 'dir'
grad = -2/n*X'*(y-X*w);
dir = -grad;
%%% step size selection
stepsize = 1;
wnew = w + stepsize*dir;
%%% computation of the objective function value at 'wnew'; variable name is newobjective
newobjective = mean((y-X*wnew).^2);
%%% backtracking line search
while newobjective > objective + alpha* stepsize* (grad'*dir) % Wolfe conditions
stepsize = stepsize * beta;
wnew = w + stepsize*dir;
%%% computation of the objective function value at 'wnew'; variable name is newobjective
newobjective = mean((y-X*wnew).^2);
end
%%% Update the variable convergence if the stopping criterion is
%%% met so that the loop is terminated after this iteration.
if norm(grad)<tol
convergence = 1;
end
w = wnew;
objective = newobjective;
iter = iter+1;
if iter==t(count)
t(count+1)=2*t(count);
count = count + 1;
W(:,count) = w;
end
end
count=count+1;
W(:,count)=w;
final_w{1} = w;
x=0:0.01:1;
x=x';
x1 = [x ones(size(x))];
colors = jet(count);
n=size(x,1);
fs=zeros(n,1);
for i=1:count
fs(:,i)=x1*W(:,i);
plot(x,fs(:,i),'linestyle','-','linewidth',2,'color',colors(i,:));
end
xlabel('x');ylabel('y');
legend('training data','initial curve',['iter=' num2str(t(1))],['iter=' num2str(t(2))],...
['iter=' num2str(t(3))],['iter=' num2str(t(4))],['iter=' num2str(t(5))],'final curve');
title('normal linear regression');
hold off
%% using linear regression with a linear combination of basis functions to fit the data
k=2; % the dimension of basis function
Phi = Basis(Xtrain, k);
% initializing weighted parameters w
w0 = ones(size(Phi,2),1);
w = w0;
%%% computation of the objective function value at w; variable name is objective
objective = mean((y-Phi*w).^2); % ridge regression problem
iter = 0;
%%% stopping criteria
convergence = 0;
maxiter = 1000;
tol = 1e-6;
%%% parameters for step size selection , here using backtracking line
%%% search for choosing stepsize
alpha = 0.1; % parameter for Armijo rule
beta = 0.95;
while(convergence==0 && iter<maxiter)
%%% the computation of the gradient, 'grad' and the descent direction 'dir'
grad = -2/n*Phi'*(y-Phi*w);
dir = -grad;
%%% step size selection
stepsize = 1;
wnew = w + stepsize*dir;
%%% computation of the objective function value at 'wnew'; variable name is newobjective
newobjective = mean((y-Phi*wnew).^2);
%%% backtracking line search
while newobjective > objective + alpha* stepsize* (grad'*dir) % Wolfe conditions
stepsize = stepsize * beta;
wnew = w + stepsize*dir;
%%% computation of the objective function value at 'wnew'; variable name is newobjective
newobjective = mean((y-Phi*wnew).^2);
end
%%% Update the variable convergence if the stopping criterion is
%%% met so that the loop is terminated after this iteration.
if norm(grad)<tol
convergence = 1;
end
w = wnew;
objective = newobjective;
iter = iter+1;
end
x=0:0.01:1;
x=x';
n=size(x,1);
fs=zeros(n,k);
figure
hold on
% plot training data
plot(Xtrain, Ytrain, 'linestyle','.','linewidth',2,'color','r');
phi=Basis(x,k);
fs=phi*w;
plot(x,fs,'linestyle','-','linewidth',2,'color','g');
xlabel('x');ylabel('y');
legend('training data','final curve');
title('linear regression with a linear combination of basis functions');
hold off
final_w{2} = w;
%% using ridge regression to fit the data with gradient descent method
lambda = 0.02;
k=2; % the dimension of basis function
Phi = Basis(Xtrain, k);
% initializing weighted parameters w
w0 = ones(size(Phi,2),1);
w = w0;
%%% computation of the objective function value at w; variable name is objective
objective = mean((y-Phi*w).^2)+lambda*norm(w)^2; % ridge regression problem
iter = 0;
%%% stopping criteria
convergence = 0;
maxiter = 1000;
tol = 1e-6;
%%% parameters for step size selection , here using backtracking line
%%% search for choosing stepsize
alpha = 0.1; % parameter for Armijo rule
beta = 0.95;
W = zeros(length(w),1);
count = 1;
W(:,count)=w;
while(convergence==0 && iter<maxiter)
%%% the computation of the gradient, 'grad' and the descent direction 'dir'
grad = -2/n*Phi'*(y-Phi*w)+2*lambda*w;
dir = -grad;
%%% step size selection
stepsize = 1;
wnew = w + stepsize*dir;
%%% computation of the objective function value at 'wnew'; variable name is newobjective
newobjective = mean((y-Phi*wnew).^2)+lambda*norm(wnew)^2;
%%% backtracking line search
while newobjective > objective + alpha* stepsize* (grad'*dir) % Wolfe conditions
stepsize = stepsize * beta;
wnew = w + stepsize*dir;
%%% computation of the objective function value at 'wnew'; variable name is newobjective
newobjective = mean((y-Phi*wnew).^2)+lambda*norm(wnew)^2;
end
%%% Update the variable convergence if the stopping criterion is
%%% met so that the loop is terminated after this iteration.
if norm(grad)<tol
convergence = 1;
end
w = wnew;
objective = newobjective;
iter = iter+1;
if mod(iter,50)==0
count=count+1;
W(:,count)=w;
end
end
count=count+1;
W(:,count)=w;
final_w{3} = w;
colors=jet(30); % set the color space
x=0:0.01:1;
x=x';
n=size(x,1);
fs=zeros(n,k);
figure
hold on
% plot training data
plot(Xtrain, Ytrain, 'linestyle','.','linewidth',2,'color','r');
for i=1:count
phi=Basis(x,k);
fs(:,i)=phi*W(:,i);
plot(x,fs(:,i),'linestyle','-','linewidth',2,'color',colors(i,:));
end
xlabel('x');ylabel('y');
legend('training data','initial curve','iter=50',...
'final curve');
title('ridge regression');
%% compare these three different regression method
figure;
hold on
% plot train data
plot(Xtrain, Ytrain, 'linestyle','.','linewidth',2,'color','r');
% plot the curve of normal linear regression
plot(x,x1*final_w{1},'linestyle','-','linewidth',2,'color','g');
% plot the curve of linear regression with a combinaiton of basis functions
plot(x,phi*final_w{2},'linestyle','-','linewidth',2,'color','y');
% plot the curve of ridge regression
plot(x,phi*final_w{3},'linestyle','-','linewidth',2,'color','b');
xlabel('x');ylabel('y');
legend('training data', 'normal linear regression','linear regression with a combinaiton of basis functions','ridge regressi
