function net = cnn_gender_init(varargin)
% CNN_gender_INIT  Initialize a standard CNN for CelebA

opts.scale = 1 ;
opts.initBias = 0 ;
opts.weightDecay = 1 ;
opts.weightInitMethod = 'gaussian' ;
opts.model = 'orig' ; %This is the model you're using.
opts.batchNormalization = false ;
opts.networkType = 'simplenn' ;
opts.cudnnWorkspaceLimit = 1024*1024*1204 ; 
opts.classNames = {} ;
opts.classDescriptions = {} ;
opts.averageImage = zeros(3,1) ;
opts.colorDeviation = zeros(3) ;
opts = vl_argparse(opts, varargin) ;

% Define layers
switch opts.model
  case 'orig' 
    net.meta.normalization.imageSize = [64, 64, 3] ;
    net = orig(net, opts) ; %You need to define this model below
    bs = 100; %This is the batch size
  %You will add another case here for your 'mini' network
  otherwise
    error('Unknown model ''%s''', opts.model) ;
end

switch lower(opts.weightInitMethod)
  case {'xavier', 'xavierimproved'}
    net.layers{end}.weights{1} = net.layers{end}.weights{1} / 10 ;
end
%The final layer is added here. This is a softmax loss function, which will
%facilitate training of the network.
net.layers{end+1} = struct('type', 'softmaxloss', 'name', 'loss') ;

% Meta parameters
net.meta.inputSize = [net.meta.normalization.imageSize, 32] ;
net.meta.normalization.cropSize = net.meta.normalization.imageSize(1) / 64 ;
net.meta.normalization.averageImage = opts.averageImage ;
net.meta.classes.name = opts.classNames ;
net.meta.classes.description = opts.classDescriptions;
net.meta.augmentation.jitterLocation = true ;
net.meta.augmentation.jitterFlip = true ;
net.meta.augmentation.jitterBrightness = double(0.1 * opts.colorDeviation) ;
net.meta.augmentation.jitterAspect = [2/3, 3/2] ;

lr = [0.01, 0.01, 0.001, 0.0001] ;


net.meta.trainOpts.learningRate = lr ;
net.meta.trainOpts.numEpochs = numel(lr) ;
net.meta.trainOpts.batchSize = bs ; %100
net.meta.trainOpts.weightDecay = 0.0005 ;

net = vl_simplenn_tidy(net) ;

%YOU WILL BE USING THIS FUNCTION IN YOUR IMPLEMENTATION FOR PROBLEM 3
%add_block adds a convolution layer, a batch normalization layer (if the 
%batch norm parameter is on, which it is), and a ReLU layer.
%Inputs:
%net - the old network is taken as input so layers can be added onto it
%opts - the parameters specified above and in cnn_gender.m
%id - the layer id (string), typically you give the same id to a
      %convolution layer and all the layers that follow it (conv1, norm1, pool1,
      %relu1, etc).
%h - the height of the filter you want to use
%w - the width of the filter you want to use
%in - the depth of the input (image input has 3 channels, so it's depth is
      %3)  For subsequent layers, the depth of the input is the depth of
      % the output of the previous layer.
%out - the depth of the output (number of feature maps you want as output,
       %or in a fully connected layer, the number of neurons you want in the
       %layer.
%stride - number of pixels to skip when applying a filter
%pad - number of pixels to pad the image with before applying the filter
% --------------------------------------------------------------------
function net = add_block(net, opts, id, h, w, in, out, stride, pad)
% --------------------------------------------------------------------
info = vl_simplenn_display(net) ; %just displays the layer info (you will see this when you start training)
fc = (h == info.dataSize(1,end) && w == info.dataSize(2,end)) ; %This is checking to see if you want a fully connected layer or convolution layer
if fc
  name = 'fc' ;
else
  name = 'conv' ;
end
convOpts = {'CudnnWorkspaceLimit', opts.cudnnWorkspaceLimit} ; %ignore this, this is for GPU stuff, which you don't need
%this is a convolution layer
%The layer type is specified by 'type' followed by the name 'conv'
%Each layer has a name, which is specified by 'name' followed by the name,
%which is 'conv'+id or 'fc'+id here
%The initial filter and bias weights are specified by 'weights' and the 
%{{init_weight(opts, h, w, in, out, 'single'), ...
%ones(out, 1, 'single')*opts.initBias}}
%You can just use this without digging into it.
%The stride and pad are specified next, using the input to add_block.
%A 'dilate' parameter is also specified. You do not need to worry about
%this. It adds zeros between the filter elements when the value is > 1.
%There is a learning rate for both the filter weights and the biases.
%Typically the biases are learned more quickly than the weights, and so a
%higher learning rate is used for them. Here the 'learningRate' [1, 2]
%indicates that we multiply the learning rate by 1 for the filter weights
%and by 2 for the bias weights.
%The weight decay is another parameter you do not need to worry about. Just
%specify it like it is done here.
%Similarly with 'opts', {convOpts}.
net.layers{end+1} = struct('type', 'conv', 'name', sprintf('%s%s', name, id), ...
                           'weights', {{init_weight(opts, h, w, in, out, 'single'), ...
                             ones(out, 1, 'single')*opts.initBias}}, ...
                           'stride', stride, ...
                           'pad', pad, ...
                           'dilate', 1, ...
                           'learningRate', [1 2], ...
                           'weightDecay', [opts.weightDecay 0], ...
                           'opts', {convOpts}) ;
                       
%this is batch normalization. You shouldn't need to mess with this
if opts.batchNormalization
  net.layers{end+1} = struct('type', 'bnorm', 'name', sprintf('bn%s',id), ...
                             'weights', {{ones(out, 1, 'single'), zeros(out, 1, 'single'), ...
                               zeros(out, 2, 'single')}}, ...
                             'epsilon', 1e-4, ...
                             'learningRate', [2 1 0.1], ...
                             'weightDecay', [0 0]) ;
end
%This is a ReLU layer. It does not have any parameters since it simply
%zeros out the negative values.
net.layers{end+1} = struct('type', 'relu', 'name', sprintf('relu%s',id)) ;
%NOTICE that each layer is added at the end (net.layers{end+1})

%This function initializes the weights. You shoudn't need to touch this.
% -------------------------------------------------------------------------
function weights = init_weight(opts, h, w, in, out, type)
% -------------------------------------------------------------------------
% See K. He, X. Zhang, S. Ren, and J. Sun. Delving deep into
% rectifiers: Surpassing human-level performance on imagenet
% classification. CoRR, (arXiv:1502.01852v1), 2015.

switch lower(opts.weightInitMethod)
  case 'gaussian'
    sc = 0.01/opts.scale ;
    weights = randn(h, w, in, out, type)*sc;
  case 'xavier'
    sc = sqrt(3/(h*w*in)) ;
    weights = (rand(h, w, in, out, type)*2 - 1)*sc ;
  case 'xavierimproved'
    sc = sqrt(2/(h*w*out)) ;
    weights = randn(h, w, in, out, type)*sc ;
  otherwise
    error('Unknown weight initialization method''%s''', opts.weightInitMethod) ;
end

%This function adds a normalization layer. No parameters are necessary
%here.
% --------------------------------------------------------------------
function net = add_norm(net, opts, id)
% --------------------------------------------------------------------
if ~opts.batchNormalization
  net.layers{end+1} = struct('type', 'normalize', ...
                             'name', sprintf('norm%s', id), ...
                             'param', [5 1 0.0001/5 0.75]) ;
end

%This function adds a 50% dropout between layers. It randomly turns off
%activations for 50% of the neurons. This is done so that certain neurons
%cannot rely too heavily on other ones. No parameters need to be specified
%here.
% --------------------------------------------------------------------
function net = add_dropout(net, opts, id)
% --------------------------------------------------------------------
if ~opts.batchNormalization
  net.layers{end+1} = struct('type', 'dropout', ...
                             'name', sprintf('dropout%s', id), ...
                             'rate', 0.5) ;
end


% --------------------------------------------------------------------
function net = orig(net, opts)
% --------------------------------------------------------------------
%THIS IS WHERE YOU WILL SPECIFY YOUR NETWORK ARCHITECTURE FOR PROBLEM 3

% --------------------------------------------------------------------
%function net = mini(net, opts)
% --------------------------------------------------------------------
%THIS IS WHERE YOU WILL SPECIFY YOUR MINI NETWORK ARCHITECTURE FOR PROBLEM
%4
