/home/travis/build/jimfinnis/uesmanncpp/data.hpp

Go to the documentation of this file.

#ifndef __DATA_H
#define __DATA_H

#include <assert.h>
#include <stdint.h>

#include "mnist.hpp"

template <class T,class TestFunc> void alternate(T *arr,int nitems,int cycle,TestFunc f){
    // for each item, if it is not the appropriate value,
    // scan forward until we find one which is and swap with that.
    // Leave if we can't find one.
    for(int i=0;i<nitems;i++){
        if(f(arr[i])%cycle!=(i%cycle)){
            // doesn't match; swap.
            for(int j=i;;j++){
                if(j>=nitems)return; // can't find a match, exit.
                // scan for one that does
                if(f(arr[j])%cycle==i%cycle){
                    // and swap and leave loop
                    T v=arr[i];
                    arr[i]=arr[j];
                    arr[j]=v;
                    break;
                }
            }
        }
    }
}


class ExampleSet {
    double **examples; 
    double *data; 
    
    int ninputs; 
    int noutputs; 
    int ct; 
    
    uint32_t outputOffset; 
    uint32_t hOffset; 
    
    bool ownsData;
    
    int numHLevels;
    
    double minH;
    double maxH;
    
    
public:
    
    
    ExampleSet(int n,int nin,int nout,int levels){
        ninputs=nin;
        noutputs=nout;
        ct=n;
        numHLevels = levels;
        minH=0;
        maxH=1;
        
//        printf("Allocating new set %d*(%d,%d)\n",
//               n,ninputs,noutputs);
        
        // size of a single example: number of inputs plus number of outputs
        // plus one for the modulator.
        
        uint32_t exampleSize = ninputs+noutputs+1;
        
        // calculate the offsets
        outputOffset = ninputs;
        hOffset = ninputs+noutputs;
        
        data = new double[exampleSize*ct]; // allocate data
        examples = new double*[ct]; // allocate example pointers
        
        for(int i=0;i<ct;i++){
            // work out and store the example pointer
            examples[i] = data+i*exampleSize;
            
        }
        ownsData = true;
    }
    
    ExampleSet(const ExampleSet &parent,int start,int length){
        if(length > parent.ct - start || start<0 || length<1)
            throw std::out_of_range("subset out of range");
        ownsData = false;
        ninputs = parent.ninputs;
        noutputs = parent.noutputs;
        outputOffset = ninputs;
        hOffset = ninputs+noutputs;
        data = parent.data;
        examples = new double*[length];
        ct = length;
        numHLevels = parent.numHLevels;
        minH = parent.minH;
        maxH = parent.maxH;
        
        for(int i=0;i<ct;i++){
            examples[i] = parent.examples[start+i];
        }
    }
    
    ExampleSet(const MNIST& mnist) : ExampleSet(
                                                mnist.getCount(), // number of examples
                                                mnist.r()*mnist.c(), // input count
                                                mnist.getMaxLabel()+1, // output count
                                                1 // single modulation level
                                                ){
        // fill in the data
        for(int i=0;i<ct;i++){
            // convert each pixel into a 0-1 double and store
            uint8_t *imgpix = mnist.getImg(i);
            double *inpix = getInputs(i);
            for(int i=0;i<ninputs;i++){
                double pixval = *imgpix++;
                pixval /= 255.0;
                *inpix++ = pixval; 
            }
            // fill in the one-hot encoded output
            double *out = getOutputs(i);
            for(int outIdx=0;outIdx<noutputs;outIdx++){
                out[outIdx] = mnist.getLabel(i)==outIdx?1:0;
            }
            setH(i,0); // set nominal modulator value
        }
        ownsData=true;
    }
    
    ~ExampleSet(){
        if(ownsData){ // only delete the data if we aren't a subset
            delete [] data;
        }
    }
    
public:
    
    enum ShuffleMode { 
        STRIDE,
              ALTERNATE,
              SINGLE,
              NONE
    };
        
    
    void shuffle(drand48_data *rd,ShuffleMode mode,int nExamples=0){
        if(mode == NONE) // this means we don't shuffle
            return;
        
        if(!nExamples)
            nExamples=ct;
        
        int blockSize; // size of the blocks we are shuffling, in bytes
        if(mode == STRIDE)
            blockSize = numHLevels;
        else
            blockSize = 1;
        double **tmp = new double*[blockSize]; // temporary storage for swapping
        
        for(int i=(nExamples/blockSize)-1;i>=1;i--){
            long lr;
            lrand48_r(rd,&lr);
            int j = lr%(i+1);
            memcpy(tmp,examples+i*blockSize,blockSize*sizeof(double*));
            memcpy(examples+i*blockSize,examples+j*blockSize,blockSize*sizeof(double*));
            memcpy(examples+j*blockSize,tmp,blockSize*sizeof(double*));
        }
        // if this mode is set, rearrange the shuffled data so that the h-levels cycle
        if(mode == ALTERNATE){
            alternate<double*>(examples, nExamples, numHLevels,
                               // abominations like this are why I used an overcomplicated
                               // example system at first...
                               [this](double *e){
                               double d = (e[hOffset]-minH)/(maxH-minH);
                               int i = (int)(d*(numHLevels-1));
                               return i;
                           });
        }
        delete [] tmp;
    }
    
    ExampleSet& setHRange(double mn,double mx){
        minH = mn;
        maxH = mx;
        return *this;
    }
        
    
    int getInputCount() const {
        return ninputs;
    }
    
    int getOutputCount() const {
        return noutputs;
    }
    
    int getCount() const {
        return ct;
    }
    
    
    double *getInputs(int example) {
        assert(example<ct);
        return examples[example]; // inputs are first in each block
    }
    
    double *getOutputs(int example) {
        assert(example<ct);
        return examples[example] + outputOffset;
    }
    
    double getH(int example) const {
        assert(example<ct);
        return *(examples[example] + hOffset);
    }
    
    int getNumHLevels(){
        return numHLevels;
    }
          
    
    void setH(int example, double h){
        assert(example<ct);
        *(examples[example] + hOffset) = h;
    }
    
    void dump(int start=0,int end=-1){
        if(end<0)end=ct;
        for(int i=start;i<end;i++){
            double *ins = getInputs(i);
            double *outs = getOutputs(i);
            for(int j=0;j<ninputs;j++){
                printf("%f ",ins[j]);
            }
            printf(" modulator %f --> ",getH(i));
            for(int j=0;j<noutputs;j++){
                printf("%f ",outs[j]);
            }
            printf("\n");
        }
    }
    
};



#endif /* __DATA_H */