%BCKPRP12 Demonstration of backpropagation in neural nets.
%	This Expo demo adapted from ...
% 	EXAMPLE BCKPROP1: NONLINEAR NETWORK ERROR SURFACE
% 	Chapter 5 in the Neural Network Toolbox Users Manual.

% 	M.H. Beale & H.B. Demuth, 1-31-92
%
%	Expo versions:
%	Ned Gulley, 6-21-93.  Denise L. Chen, 9-3-93.
%	Copyright (c) 1984-94 by The MathWorks, Inc.

% Demo initialization ====================
if ~exist('SlideShowGUIFlag'), figNumber=0; end;

if ssinit(figNumber), 

% initialize demo with Error Surface plot

    % PROBLEM DEFINITION
    %===================
    
    % Define two 1-element input vectors.
    P = [-3.0 +2.0];
    
    % Define the associated two 1-element targets.
    T = [+0.4 +0.8];

    % PLOT ERROR SURFACE
    %================
    Wrange = -4:.3:4;
    Brange = -4:.3:4;
    ES = errsurf(P,T,Wrange,Brange,'logsig');
    Etable=[0 Brange; Wrange' ES];
    h=surfl(Wrange,Brange,ES);
    set(h,'EdgeColor','b');
    colormap(fliplr(pink));
    view([60 65]);
    xlabel('Weight W');
    ylabel('Bias B');
    zlabel('Sum Squared Error')
    title('Error Surface Graph')
% 
    str= ...                                                        
    [' Press the "Start" button to see a demonstration of how'  
     ' backpropagation works in a neural network.            '];
    ssdisp(figNumber,str);                                          

    if figNumber, return; end
end

% Beginning of the demo ==================

% SLIDE 1 ===================
str = str2mat( ...
     ' The backpropagation learning rules are used to adjust the', ...
     ' weights and biases of networks so as to minimize the sum', ...
     ' squared error of the network. This is done by continually', ...
     ' changing the values of the network weights and biases in the', ...
     ' direction of steepest descent with respect to error. This is ', ...
     ' called the gradient descent procedure.');
ssdisp(figNumber,str);                                          
if sspause(figNumber), return; end;

% SLIDE 2 ===================

str = str2mat( ...
     ' Here is a simple problem with two 1-element input vectors and ', ...
     ' two 1-element output vectors:', ...
     ' ', ...
     ' >> P = [-3.0 +2.0];', ...
     ' >> T = [+0.4 +0.8];', ...
     '  ', ...
     ' We use a single layer network for this problem. ', ...
     ' Note that there are only two network variables to be optimized,', ...
     ' the bias and the weight.');
ssdisp(figNumber,str);                                          
if sspause(figNumber), return; end;

% SLIDE 3 ===================

str = str2mat( ...
     ' Since we have only two variables, we can plot the sum of', ...
     ' squared errors for a range of biases and weights. Here is a', ...
     ' graph of the error surface.', ...
     ' ', ...
     ' >> Wrange = -4:.3:4;',...
     ' >> Brange = -4:.3:4;',...
     ' >> ES = errsurf(P,T,Wrange,Brange,''logsig'');',...
     ' >> h = surfl(ES);', ...
     ' >> colormap(fliplr(pink));', ...
     ' >> view([60 65]);');

ssdisp(figNumber, str);
if sspause(figNumber), return; end;

% SLIDE 4 ====================

    % PROBLEM DEFINITION
    %===================
    
    % Define two 1-element input vectors.
    P = [-3.0 +2.0];
    
    % Define the associated two 1-element targets.
    T = [+0.4 +0.8];

    % PLOT ERROR SURFACE
    %================
    Wrange = -4:.3:4;
    Brange = -4:.3:4;
    ES = errsurf(P,T,Wrange,Brange,'logsig');
    Etable=[0 Brange; Wrange' ES];

% INITIALIZE NETWORK ARCHITECTURE
%================================

% Set input vector size R, layer size S, batch size Q.

[R,Q] = size(P); [S,Q] = size(T);

% Initialize weight and bias.
% Initializing Weights & Bias to User''s Guide Initial Conditions

W0 = [-2.1617];
B0 = [-1.7862];

% TRAINING PARAMETERS
disp_freq = 1;
max_epoch = 100;
err_goal = 0.001;
lr = 1;

str = str2mat( ...
    ' To train the network, we first initialize the weight and bias.', ...
    ' We set training parameters to see how often progress is ', ...
    ' displayed, to limit the number of training epochs, to define', ...
    ' an acceptable error, and to set the learning rate.', ...
    ' ', ...
    ' >> W0 = [-2.1617];  B0 = [-1.7862];', ...
    ' >> disp_freq = 1;', ...
    ' >> max_epoch = 100;', ...
    ' >> err_goal = 0.001;', ...
    ' >> lr = 1;');
ssdisp(figNumber, str);
if sspause(figNumber), return; end;

% SLIDE 5 =============================
if sspause(figNumber), return; end;
str = str2mat( ...
    ' Now the network is ready to be trained.', ...
    ' ', ...
    ' >> TP = [disp_freq max_epoch err_goal lr];', ...
    ' >> [W,B,epoch,errors] = trainbp(W0,B0,''logsig'',P,T,TP);');

ssdisp(figNumber,str);
if sspause(figNumber), return; end;

% SLIDE 6 ==========================
str = str2mat( ...
    ' As learning occurs, the weight and bias are adjusted to move', ...
    ' the network down the error gradient.', ...
    ' ', ...
    ' Note that we are using Handle Graphics animation to help', ...
    ' visualize the results.');
ssdisp(figNumber,str);

% PLOT ERROR CONTOUR, INITIAL CONDITIONS
%=======================================
hold on;
E0=table2(Etable,W0,B0);
plot3(W0,B0,E0,'oy', ...
      'EraseMode','none', ...
      'MarkerSize',20);

% TRAIN THE NETWORK
%==================

% TRAINING PARAMETERS
disp_freq = 1;
max_epoch = 100;
err_goal = 0.001;
lr = 1;

% NOTE: The rest of training code may be replaced by:
%
% TP = [disp_freq max_epoch err_goal lr lr_inc lr_dec err_ratio];
% [W,B,epoch,errors] = trainbp(W0,B0,'logsig',P,T,TP);
% Also, in order to use the "Plot Errors Associated with Each Output Vector",
% you must add the following line after the above change:
% A2=purelin(W2*tansig(W1*P,B1),B2);

% NETWORK PARAMETERS
W = W0;
B = B0;

% PRESENTATION PHASE
A = logsig(W*P,B);
E = T-A;
SSE = sumsqr(E);

% TRAINING RECORD
errors = [SSE];

for epoch=1:max_epoch

  % CHECK PHASE
  if SSE < err_goal, epoch=epoch-1; break, end

  LW = W; LB = B;

  % LEARNING PHASE
  D = deltalog(A,E);
  [dW,dB] = learnbp(P,D,lr);
  W = W + dW;
  B = B + dB;

  % PRESENTATION PHASE
  A = logsig(W*P,B);
  E = T-A;
  SSE = sumsqr(E);

  % TRAINING RECORD
  errors = [errors SSE];

  % DISPLAY PROGRESS
  if rem(epoch,disp_freq) == 0
    %EW=table2(Etable,LW,LB);
    EE=table2(Etable,W,B);
    %plot3([LW W],[LB B],[EW EE],'r','erase','none');
    plot3(W,B,EE,'r.', ...
          'EraseMode','none', ...
          'MarkerSize',20);
    drawnow
    end 
  end

% PLOT FINAL VALUES
%==================
plot3(W,B,EE,'oy', ...
      'EraseMode','none', ...
      'MarkerSize',20);
hold off;

if sspause(figNumber), return; end;

% SLIDE 7 ===========================

% PLOT ERROR CURVE
%=================
% CLF RESET in PLOTERR is causing problems, so I'm inlining
% the relevant code from the ploterr function. ewg/1-31-94
% ploterr(errors);
e=errors;
epochs = length(e)-1;
t = sprintf('Sum-Squared Network Error for %g Epochs',epochs);
plot(0:epochs,e);
xlabel('Epoch')
ylabel('Sum-Squared Error')
title(t)
set(gca,'box','on')
set(gca,'xlim',[0 epochs+eps]);
set(gca,'yscale','log');
drawnow
% End of inlined code from PLOTERR.

str = str2mat( ...
    ' The training record variable "errors" holds the network error', ...
    ' throughout the training. Here is a plot of these errors.', ...
   '  ', ...
    ' >> ploterr(errors)', ...
   '  ', ...
    ' Note the steady decrease in error achieved by the', ...
    ' backpropagation algorithm');

ssdisp(figNumber,str);
if sspause(figNumber), return; end;

% SLIDE 8 =========================
str = str2mat( ...
    ' The error plot stops at 71 epochs when the sum squared error', ...
    ' dropped to 0.00096, which is less than the error goal of 0.001.',...
    ' ', ...
    ' Here are the exact final values:', ...
    '', ...
    ' W = 0.3346', ...
    ' B = 0.5473');
ssdisp(figNumber,str);

% End of the demo ========================
