%	This Expo demo adapted from ...
% EXAMPLE BCKPROP6: LEARNING WITH MOMENTUM
% Chapter 5 in the Neural Network Toolbox Users Manual.
 
% 	M.H. Beale & H.B. Demuth, 1-31-92
%
%	Adapted for Expo:
%	Ned Gulley, 6-21-93. Denise Chen, 9-3-93.
%	Copyright (c) 1984-94 by The MathWorks, Inc.

% Demo initialization ====================
if ~exist('SlideShowGUIFlag'), figNumber=0; end;

if ssinit(figNumber), 
    str= ...                                                    
    [' Press the "Start" button to see a demonstration of'  
     ' backpropagation with momentum in neural networks. '];
    ssdisp(figNumber,str);                                      
    if figNumber, return; end
end

% Beginning of the demo ==================

str= str2mat(...                                                  
   ' A 1-input layer of 1 hardlimit neuron is trained with', ...
   ' backpropagation (with momentum) to properly respond to', ...
   ' 8 input vectors.', ... 
   '', ...
   ' Adding momentum to the learning rule results in a network' , ...
   ' that is less sensitive to local features of the error   ' , ...
   ' surface and more sensitive to larger trends. Such a     ' , ...
   ' network can escape from shallow minima.');
ssdisp(figNumber,str);                                    

% PROBLEM DEFINITION
%===================

% Define eight 1-element input vectors.
P = [-6.0 -6.1 -4.1 -4.0 +5.0 +5.1 +6.0 +6.1];

% Define the associated eight 1-element targets.
T = [+0.0 +0.0 +.97 +.99 +.01 +.03 +1.0 +1.0];

% PLOT ERROR SURFACE
%===================

Wrange = -1:.1:1;
Brange = 3;
ES = errsurf(P,T,Wrange,Brange,'logsig');
plot(Wrange,ES);
title('Error Surface Graph')
xlabel('Weight W')
ylabel('Error')

if sspause(figNumber), return; end;

%==========================
str = str2mat( ...
    ' First look at a network learning with no momentum. Here are',...
    ' the initial network weight and bias and training parameters.', ...
    ' ', ...
    ' >> B0 = 3;  W0 = -0.90;', ...
    ' >> disp_freq = 1;', ...
    ' >> max_epoch = 125;', ...
    ' >> err_goal = 0.01;', ...
    ' >> lr = 0.05;', ...
    ' >> momentum = 0;', ...
    ' >> err_ratio = 1.04;');

ssdisp(figNumber,str);                                             
%if sspause(figNumber), return; end;

%==========================

% INITIALIZE NETWORK ARCHITECTURE
%================================
% Set input vector size R, layer size S, batch size Q.
[R,Q] = size(P); [S,Q] = size(T);

% Initialize weight and bias.
%disp('The bias B is fixed at 3.0 and will not learn.')

B0 = 3;
W0 = [-0.90];

% PLOT ERROR CONTOUR, INITIAL CONDITIONS
%=======================================
plot(Wrange,ES);
hold on
title('Error Surface Graph')
xlabel('Weight W')
ylabel('Error')

A = logsig(W0*P,B0);
E = T-A;
SSE = sumsqr(E);
h = plot(W0,SSE,'og');
set(h,'markersize',20,'erase','none');

% TRAIN THE NETWORK
%==================

% TRAINING PARAMETERS
disp_freq = 1;
max_epoch = 125;
%max_epoch = 50;
err_goal = 0.01;
lr = 0.05;

momentum = 0;

err_ratio = 1.04;

% NOTE: The rest of training code may be replaced by:
%
% TP = [disp_freq max_epoch err_goal lr momentum err_ratio];
% [W,B,epoch,errors] = trainbpm(W0,B0,'logsig',P,T,TP);

% NETWORK PARAMETERS
W = W0;
B = B0;

% PRESENTATION PHASE
A = logsig(W*P,B);
E = T-A;
SSE = sumsqr(E);

% TRAINING PARAMETERS
errors = [SSE];

% INITIALIZE MOMENTUM
mc = 0; dW = W*0;

for epoch=1:max_epoch

  % CHECK PHASE
  if SSE < err_goal, epoch=epoch-1; break, end

  % LEARNING PHASE
  D = deltalog(A,E);
  dW = learnbpm(P,D,lr,mc,dW);
  TW = W + dW;

  % PRESENTATION PHASE
  TA = logsig(TW*P,B);
  TE = T-TA;
  TSSE = sumsqr(TE);

  % MOMENTUM PHASE
  if TSSE > SSE*err_ratio
    mc = 0;
  else
    if TSSE < SSE
       mc = momentum;
       end
    W=TW; A=TA; E=TE; SSE=TSSE;
    end

  % TRAINING RECORD
  errors = [errors SSE];

  % DISPLAY PROGRESS
  if rem(epoch,disp_freq) == 0
    delete(h)
    h = plot(W,SSE,'.r', ...
             'markersize',40, ...
             'erasemode','xor');
    drawnow;
    end
  end

% PLOT FINAL VALUES
%==================

plot(W,SSE,'og','markersize',20,'erasemode','none');
hold off;
if sspause(figNumber), return; end;

str= str2mat(...                                                         
   ' Notice we got "stuck" in a local minimum while a tasty global', ...
   ' minimum lay just over the hill.');
ssdisp(figNumber,str);                                           

if sspause(figNumber), return; end;

%===========================
str = str2mat( ...
    ' Let''s now add momentum.', ...
    '', ...
    ' We change only the momentum parameter in the set of', ... 
    ' initial network weights and biases and training parameters', ...
    ' used before.', ...
    ' ', ...
    ' >> momentum = 0.98;');
ssdisp(figNumber,str);                                           

plot(Wrange,ES);
hold on
title('Error Surface Graph')
xlabel('Weight W')
ylabel('Error')

if sspause(figNumber), return; end;

%===========================
str = str2mat( ...
    ' The network can now be trained using the "trainbpm" function.', ... 
    ' ', ...
    ' >> TP = [disp_freq max_epoch err_goal lr momentum err_ratio];', ...
    ' >> [W,B,epoch,errors] = trainbpm(W0,B0,''logsig'',P,T,TP);', ...
    ' ' , ...
    ' The training parameters in the vector TP are the number of' , ...
    ' epochs between reporting progress, the maximum number of', ...
    ' training epochs, the desired error goal, the learning rate,', ...
    ' the momentum constant, and the maximum error ratio. ');

ssdisp(figNumber,str);                                           
%if sspause(figNumber), return; end;

% NETWORK PARAMETERS
W = W0;
B = B0;

% PRESENTATION PHASE
%===================
Wrange = -1:.1:1;
Brange = 3;
ES = errsurf(P,T,Wrange,Brange,'logsig');
plot(Wrange,ES);
title('Error Surface Graph')
xlabel('Weight W')
ylabel('Error')

% INITIALIZE NETWORK ARCHITECTURE
%================================
% Set input vector size R, layer size S, batch size Q.
[R,Q] = size(P); [S,Q] = size(T);

% Initialize weight and bias.
%disp('The bias B is fixed at 3.0 and will not learn.')

B0 = 3;
W0 = [-0.90];

% PLOT ERROR CONTOUR, INITIAL CONDITIONS
%=======================================
plot(Wrange,ES);
hold on
title('Error Surface Graph')
xlabel('Weight W')
ylabel('Error')

A = logsig(W0*P,B0);
E = T-A;
SSE = sumsqr(E);
h = plot(W0,SSE,'og');
set(h,'markersize',20,'erase','none');

% TRAIN THE NETWORK
%==================

% TRAINING PARAMETERS
disp_freq = 1;
max_epoch = 125;
err_goal = 0.01;
lr = 0.05;

momentum = 0.98;

err_ratio = 1.04;

% NOTE: The rest of training code may be replaced by:
%
% TP = [disp_freq max_epoch err_goal lr momentum err_ratio];
% [W,B,epoch,errors] = trainbpm(W0,B0,'logsig',P,T,TP);

% NETWORK PARAMETERS
W = W0;
B = B0;

% PRESENTATION PHASE
A = logsig(W*P,B);
E = T-A;
SSE = sumsqr(E);

% TRAINING PARAMETERS
errors = [SSE];

% INITIALIZE MOMENTUM
mc = 0; dW = W*0;

for epoch=1:max_epoch

  % CHECK PHASE
  if SSE < err_goal, epoch=epoch-1; break, end

  % LEARNING PHASE
  D = deltalog(A,E);
  dW = learnbpm(P,D,lr,mc,dW);
  TW = W + dW;

  % PRESENTATION PHASE
  TA = logsig(TW*P,B);
  TE = T-TA;
  TSSE = sumsqr(TE);

  % MOMENTUM PHASE
  if TSSE > SSE*err_ratio
    mc = 0;
  else
    if TSSE < SSE
       mc = momentum;
       end
    W=TW; A=TA; E=TE; SSE=TSSE;
    end

  % TRAINING RECORD
  errors = [errors SSE];

  % DISPLAY PROGRESS
  if rem(epoch,disp_freq) == 0
    delete(h)
    h = plot(W,SSE,'.r', ...
             'markersize',40, ...
             'erasemode','xor');
    drawnow;
    end
  end

% PLOT FINAL VALUES
%==================

plot(W,SSE,'og','markersize',20,'erasemode','none');
hold off;
if sspause(figNumber), return; end;

str= ...                                                      
    [' Success! With a little extra push, the network error'  
     ' skipped right past the local minimum.               '];
ssdisp(figNumber,str);                                        

if sspause(figNumber), return; end;

% PLOT ERROR CURVE
%=================
% CLF RESET in PLOTERR is causing problems, so I'm inlining
% the relevant code from the ploterr function. ewg/1-31-94
% ploterr(errors);
e=errors;
epochs = length(e)-1;
t = sprintf('Sum-Squared Network Error for %g Epochs',epochs);
plot(0:epochs,e);
xlabel('Epoch')
ylabel('Sum-Squared Error')
title(t)
set(gca,'box','on')
set(gca,'xlim',[0 epochs+eps]);
set(gca,'yscale','log');
drawnow
% End of inlined code from PLOTERR.

str= ...                                                            
    [' Finally, here is a plot of the network training error as a'  
     ' function of epoch for the second learning session.        '];
str = str2mat(str, '', ...
    ' >> ploterr(errors);');
ssdisp(figNumber,str);                                              

% End of the demo ========================
