/home/travis/build/jimfinnis/uesmanncpp/net.hpp

Go to the documentation of this file.

#ifndef __NET_HPP
#define __NET_HPP

#include <math.h>

#include "netType.hpp"
#include "data.hpp"

inline double sigmoid(double x){
    return 1.0/(1.0+exp(-x));
}

inline double sigmoidDiff(double x){
    double s = sigmoid(x);
    return (1.0-s)*s;
}

class Net {
    friend class OutputBlendingNet;
    friend class HInputNet;
public:
    
    virtual ~Net() {} 
    
    NetType type; 
    drand48_data rd; 
    
    void setSeed(long seed){
        srand48_r(seed,&rd);
    }
    
    virtual int getLayerSize(int n) const =0;
    
    virtual int getLayerCount() const =0;
    
    int getInputCount() const {
        return getLayerSize(0);
    }
    
    int getOutputCount() const {
        return getLayerSize(getLayerCount()-1);
    }
        
    
    
    virtual void setInputs(double *d) = 0;
    
    virtual double *getOutputs() const = 0;
    
    double *run(double *in) {
        setInputs(in);
        update();
        return getOutputs();
    }
    
    virtual void setH(double h)=0;
    
    virtual double getH() const =0;
    
    double test(ExampleSet& examples,int start=0,int num=-1){
        double mseSum = 0;
        // have to do this here, too, although runExamples does it, so we can
        // get the denominator for the mse.
        if(num<0)num=examples.getCount()-start;
        
        // for each example, run it and accumulate the sum of squared errors
        // on all outputs 
        
        for(int i=0;i<num;i++){
            int idx = start+i;
            setH(examples.getH(idx));
            double *netout = run(examples.getInputs(idx));
            double *exout = examples.getOutputs(idx);
            for(int j=0;j<examples.getOutputCount();j++){
                double d = netout[j]-exout[j];
                mseSum += d*d;
            }
        }
        
        // we then divide by the number of examples and the output count.
        return mseSum / (num * examples.getOutputCount());
    }
    
    struct SGDParams {
        friend class Net;
        
        int iterations;
        
        double eta;
        
        
        int nSlices;
        
        int nPerSlice;
        
        int cvInterval;
        
        SGDParams& crossValidationManual(int slices,int nperslice,int interval){
            nSlices = slices;
            nPerSlice = nperslice;
            cvInterval = interval;
            return *this;
        }
        
        ExampleSet::ShuffleMode shuffleMode;
        
        SGDParams& setShuffle(ExampleSet::ShuffleMode m){
            shuffleMode = m;
            return *this;
        }
        
        
        
        bool selectBestWithCV;
        
        SGDParams& setSelectBestWithCV(bool v=true){
            selectBestWithCV=v;
            return *this;
        }
        
        
        bool cvShuffle;
        
        SGDParams& setCVShuffle(bool v=true){
            cvShuffle=v;
            return *this;
        }
        
        int initrange;
        
        SGDParams& setInitRange(double range=-1){
            initrange = range;
            return *this;
        }
        
        long seed;
        
        SGDParams& setSeed(long v){
            seed = v;
            return *this;
        }
        
        double *bestNetBuffer;
        
        bool storeBestNet;
        
    private:
        void init(double _eta,int _iters){
            seed = 0L;
            eta = _eta;
            iterations = _iters;
            initrange = -1;
            bestNetBuffer = NULL;
            ownsBestNetBuffer = false;
            storeBestNet = false;
            nSlices=0;
            nPerSlice=0;
            cvInterval=1;
            shuffleMode = ExampleSet::STRIDE;
            selectBestWithCV=false; // there might not be CV!
            cvShuffle = true; // do shuffle CV at the end of an epoch
        }
    public:        
        SGDParams(double _eta, int _iters) {
            init(_eta,_iters);
        }
        
        SGDParams(double _eta,const ExampleSet& examples,int _iters){
            init(_eta,examples.getCount()*_iters);
        }
        
        ~SGDParams(){
            if(ownsBestNetBuffer)delete[] bestNetBuffer;
        }
        
        SGDParams &crossValidation(const ExampleSet& examples,
                                   double propCV,
                                   int cvCount,
                                   int cvSlices,
                                   bool cvShuf=true
                                   ){
            cvShuffle = cvShuf;
            // calculate the number of CV examples
            int nCV = (int)round(propCV*examples.getCount());
            if(nCV==0 || nCV>examples.getCount())
                throw std::out_of_range("Bad cross-validation count");
            if(cvSlices<=0)
                throw std::out_of_range("Zero (or fewer) CV slices is a bad thing");
            // calculate the number of examples per slice and check it's not zero.
            // The resulting number of CV examples may not agree with nCV above due
            // to the integer division
            nPerSlice = nCV/cvSlices;
            nSlices = cvSlices;
            if(!nPerSlice)
                throw std::logic_error("Too many slices");
            // calculate the cvInterval
            cvInterval = iterations/cvCount;
            if(cvInterval<=0)
                throw std::logic_error("Too many CV events");
            // we want to pick the best network by CV rather than training error
            selectBestWithCV=true;
            
            printf("Cross-validation: %d slices, %d items per slice, %d total\n",
                   nSlices,nPerSlice,nSlices*nPerSlice);
            return *this;
        }
        
        SGDParams &storeBest(){
            ownsBestNetBuffer = true;
            storeBestNet = true;
            return *this;
        }
    private:
        bool ownsBestNetBuffer;
    };
    
    
    
    double trainSGD(ExampleSet &examples,SGDParams& params){
        
        // set seed for PRNG
        setSeed(params.seed);
        
        // separate out the training examples from the cross-validation examples
        int nCV = params.nSlices*params.nPerSlice;
        // it's an error if there are too many CV examples
        if(nCV>=examples.getCount())
            throw std::out_of_range("Too many cross-validation examples");
        
        if(!nCV && params.selectBestWithCV)
            throw std::logic_error("cannot use CV to select best when no CV is done");
        
        // get the number of actual training examples
        int nExamples = examples.getCount() - nCV;
        
        // initialise the network
        initWeights(params.initrange);
        
        // initialise minimum error to rogue value
        double minError = -1;
        
        // We don't shuffle before getting the cross-validation examples,
        // because in some cases there's a kind of "fake" cv going on where the
        // training portion and cv portion have to have similar (or identical)
        // distributions. See the boolean test code for an example.
        //        examples.shuffle(&rd,params.shuffleMode);
        
        // build a temporary subset for the CV examples. This still needs to exist
        // even if we're not using CV, so in that case we'll just
        // use a dummy of one example.
        
        ExampleSet cvExamples(examples,nCV?examples.getCount()-nCV:0,nCV?nCV:1);
        
        
        // setup a countdown for when we cross-validate
        int cvCountdown = params.cvInterval;
        // and which slice we are doing
        int cvSlice = 0;
        
        // now actually do the training
        
        FILE *log = fopen("foo","w");
        fprintf(log,"x,slice,y\n");
        for(int i=0;i<params.iterations;i++){
            // find the example number
            int exampleIndex = i % nExamples;
            
            // at the start of each epoch, reshuffle. This will effectively do an extra shuffle
            // as we've already done it once at the start, before splitting out the CV examples.
            
            if(exampleIndex == 0)
                examples.shuffle(&rd,params.shuffleMode,nExamples);
                
            // train here, just one example, no batching.
            double trainingError = trainBatch(examples,exampleIndex,1,params.eta);
            
            if(!params.selectBestWithCV){
                // now test the error and keep the best net. This works differently
                // if we're doing this by cross-validation or training error. Here
                // we're using the training error.
                if(minError < 0 || trainingError < minError){
                    if(params.storeBestNet){
                        if(!params.bestNetBuffer)
                            params.bestNetBuffer = new double[getDataSize()];
                        save(params.bestNetBuffer);
                    }
                    minError = trainingError;
                }
            }
            
            // is there cross-validation? If so, do it.
            
            if(nCV && !--cvCountdown){
                cvCountdown = params.cvInterval; // reset
                
                // test the appropriate slice, from example cvSlice*nPerSlice, length nPerSlice,
                // and get the MSE
                double error = test(cvExamples,cvSlice*params.nPerSlice,
                                    params.nPerSlice);
                fprintf(log,"%d,%d,%f\n",i,cvSlice,error);
                
                // test this against the min error as was done above
                if(params.selectBestWithCV){
                    if(minError < 0 || trainingError < minError){
                        if(params.storeBestNet){
                        if(!params.bestNetBuffer)
                            params.bestNetBuffer = new double[getDataSize()];
                            save(params.bestNetBuffer);
                        }
                        minError = trainingError;
                    }
                }
                
                // increment the slice index
                cvSlice = (cvSlice+1)%params.nSlices;
                // if we are now on the first slice, shuffle the entire CV set
                if(!cvSlice && params.cvShuffle)
                    cvExamples.shuffle(&rd,params.shuffleMode);
            }
        }
        
        fclose(log);
        
        // at the end, finalise the network to the best found if we can
        if(params.bestNetBuffer)
            load(params.bestNetBuffer);
        
        // test on either the entire CV set or the training set and return result
        return test(nCV?cvExamples:examples);
    }
    
    virtual int getDataSize() const = 0;
    
    virtual void save(double *buf) const = 0;
    
    virtual void load(double *buf) = 0;
    
protected:
    
    
    
    virtual void update() = 0;
    
    Net(NetType tp){
        type = tp;
        setSeed(0);
    }
    
    inline double drand(double mn,double mx){
        double res;
        drand48_r(&rd,&res);
        return res*(mx-mn)+mn;
    }
    
    virtual void initWeights(double initr) = 0;
    
    virtual double trainBatch(ExampleSet& ex,int start,int num,double eta) = 0;
    
};    


#endif /* __NET_HPP */