Matlab neural network for regression - matlab

Question

I have implemented 3 function for neural network regression:
1) a forward propagation function that given the training inputs and the net structure calculates the predicted output
function [y_predicted] = forwardProp(Theta,Baias,Inputs,NumberOfLayers,RegressionSwitch)
for i = 1:size(Inputs{1},2)
Activation = (Inputs{1}(:,i))';
for j = 2:NumberOfLayers - RegressionSwitch
Activation = 1./(1+exp(-(Activation*Theta{j-1} + Baias{j-1})));
end
if RegressionSwitch == 1
y_predicted(:,i) = Activation*Theta{end} + Baias{end};
else
y_predicted(:,i) = Activation;
end
end
end
2) a cost function that given the predicted and the desired output, calculates the cost of the network
function [Cost] = costFunction(y_predicted, y, Theta, Baias, Lambda)
Cost = 0;
for j = 1:size(y,2)
for i = 1:size(y,1)
Cost = Cost +(((y(i,j) - y_predicted(i,j))^2)/size(y,2));
end
end
Reg = 0;
for i = 1:size(Theta, 2)
for j = 1:size(Theta{i}, 1)
for k = 1:size(Theta{i}, 2)
Reg = Reg + (Theta{i}(j,k))^2;
end
end
end
for i = 1:size(Baias, 2)
for j = 1:length(Baias{i})
Reg = Reg + (Baias{i}(j))^2;
end
end
Cost = Cost + (Lambda/(2*size(y,2)))*Reg;
end
3) a back propagation function that calculates the partial derivative of the cost function for each weight in the network
function [dTheta, dBaias] = Deltas(Theta,Baias,Inputs,NumberOfLayers,RegressionSwitch, Epsilon, Lambda, y)
for i = 1:size(Theta,2)
for j = 1:size(Theta{i},1)
for k = 1:size(Theta{i},2)
dTp = Theta;
dTm = Theta;
dTp{i}(j,k) = dTp{i}(j,k) + Epsilon;
dTm{i}(j,k) = dTm{i}(j,k) - Epsilon;
y_predicted_p = forwardProp(dTp,Baias,Inputs,NumberOfLayers,RegressionSwitch);
y_predicted_m = forwardProp(dTm,Baias,Inputs,NumberOfLayers,RegressionSwitch);
Cost_p = costFunction(y_predicted_p, y, dTp, Baias, Lambda);
Cost_m = costFunction(y_predicted_m, y, dTm, Baias, Lambda);
dTheta{i}(j,k) = (Cost_p - Cost_m)/(2*Epsilon);
end
end
end
for i = 1:size(Baias,2)
for j = 1:length(Baias{i})
dBp = Baias;
dBm = Baias;
dBp{i}(j) = dTp{i}(j) + Epsilon;
dBm{i}(j) = dTm{i}(j) - Epsilon;
y_predicted_p = forwardProp(Theta,dBp,Inputs,NumberOfLayers,RegressionSwitch);
y_predicted_m =forwardProp(Theta,dBm,Inputs,NumberOfLayers,RegressionSwitch);
Cost_p = costFunction(y_predicted_p, y, Theta, dBp, Lambda);
Cost_m = costFunction(y_predicted_m, y, Theta, dBm, Lambda);
dBaias{i}(j) = (Cost_p - Cost_m)/(2*Epsilon);
end end end
I train the neural network with data from an exact mathematical function of the inputs.
The gradient descent seems to work as the cost decrease each iteration, but when i test the trained network the regression is terrible.
The functions are not meant to be efficient, but they should work so I am really frustrated to see they don't... The main function and the data are ok so the problem should be here. Can you please help me to spot it?
here is the "main":
clear;
clc;
Nodes_X = 5;
Training_Data = 1000;
x = rand(Nodes_X, Training_Data)*3;
y = zeros(2,Training_Data);
for j = 1:Nodes_X
for i = 1:Training_Data
y(1,i) = (x(1,i)^2)+x(2,i)-x(3,i)+2*x(4,i)/x(5,i);
y(2,i) = (x(5,i)^2)+x(2,i)-x(3,i)+2*x(4,i)/x(1,i);
end
end
vx = rand(Nodes_X, Training_Data)*3;
vy = zeros(2,Training_Data);
for j = 1:Nodes_X
for i = 1:Training_Data
vy(1,i) = (vx(1,i)^2)+vx(2,i)-vx(3,i)+2*vx(4,i)/vx(5,i);
vy(2,i) = (vx(5,i)^2)+vx(2,i)-vx(3,i)+2*vx(4,i)/vx(1,i);
end
end
%%%%%%%%%%%%%%%%%%%%%%ASSIGN NODES TO EACH LAYER%%%%%%%%%%%%%%%%%%%%%%%%%%%
NumberOfLayers = 4;
Nodes(1) = 5;
Nodes(2) = 10;
Nodes(3) = 10;
Nodes(4) = 2;
if length(Nodes) ~= NumberOfLayers || (Nodes(1)) ~= size(x, 1)
WARNING = msgbox('Nodes assigned incorrectly!');
end
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%INITIALIZATION%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
for i = 1:NumberOfLayers-1
Theta{i} = rand(Nodes(i),Nodes(i+1));
Baias{i} = rand(1,Nodes(i+1));
end
Inputs{1} = x;
Outputs{1} = y;
RegressionSwitch = 1;
Lambda = 10;
Epsilon = 0.00001;
Alpha = 0.01;
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%TRAINING%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
Epoch = 0;
figure;
hold on;
while Epoch <=20
%%%%%%%%%%%%%%%%%%%%FORWARD PROPAGATION%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
y_predicted = forwardProp(Theta,Baias,Inputs,NumberOfLayers,RegressionSwitch);
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%COST%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
Cost = costFunction(y_predicted, y, Theta, Baias, Lambda);
scatter(Epoch,Cost);
pause(0.01);
%%%%%%%%%%%%%%%%%%%%%%%%%%%%BACK PROPAGATION%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
[dTheta, dBaias] = Deltas(Theta,Baias,Inputs,NumberOfLayers,RegressionSwitch, Epsilon, Lambda, y);
%%%%%%%%%%%%%%%GRADIENT DESCENT%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
for i = 1:size(Theta,2)
Theta{i} = Theta{i}-Alpha*dTheta{i};
end
for i = 1:size(Baias,2)
Baias{i} = Baias{i}-Alpha*dBaias{i};
end
Epoch = Epoch + 1;
end
hold off;
V_Inputs{1} = vx;
V_y_predicted = forwardProp(Theta,Baias,V_Inputs,NumberOfLayers,RegressionSwitch);
figure;
hold on;
for i = 1:size(vy,2)
scatter(vy(1,i),V_y_predicted(1,i));
pause(0.01);
end
hold off;
figure;
hold on;
for i = 1:size(vy,2)
scatter(vy(2,i),V_y_predicted(2,i));
pause(0.01);
end
hold off;