Full Code of harthur/brain for AI

Repository: harthur/brain
Branch: master
Commit: 05bd28721c52
Files: 22
Total size: 46.4 KB

Directory structure:
gitextract_7dmu9uf3/

├── .gitignore
├── CONTRIBUTING.md
├── Gruntfile.js
├── LICENSE
├── README.md
├── bower.json
├── browser.js
├── lib/
│   ├── brain.js
│   ├── cross-validate.js
│   ├── lookup.js
│   └── neuralnetwork.js
├── package.json
├── stream-example.js
└── test/
    ├── README.md
    ├── cross-validation/
    │   └── ocr.js
    └── unit/
        ├── bitwise.js
        ├── hash.js
        ├── json.js
        ├── lookup.js
        ├── options.js
        ├── stream-bitwise.js
        └── trainopts.js

================================================
FILE CONTENTS
================================================

================================================
FILE: .gitignore
================================================
# Mac.
.DS_STORE

# Node.
node_modules
npm-debug.log


================================================
FILE: CONTRIBUTING.md
================================================
Thanks for taking the time to contribute to brain.js. Follow these guidelines to make the process smoother:

1. One feature per pull request. Each PR should have one focus, and all the code changes should be supporting that one feature or bug fix. Using a [separate branch](https://guides.github.com/introduction/flow/index.html) for each feature should help you manage developing multiple features at once.

2. Follow the style of the file when it comes to syntax like curly braces and indents.

3. Add a test for the feature or fix, if possible. See the `test` directory for existing tests and README describing how to run these tests.


================================================
FILE: Gruntfile.js
================================================
/*
 * To run this file:
 *  `npm install --dev`
 *  `npm install -g grunt`
 *
 *  `grunt --help`
 */

var fs = require("fs"),
    browserify = require("browserify"),
    pkg = require("./package.json");

module.exports = function(grunt) {
  grunt.initConfig({
    mochaTest: {
      test: {
        options: {
          style: 'bdd',
          reporter: 'spec'
        },
        src: ['test/unit/*.js']
      }
    },
    pkg: grunt.file.readJSON('package.json'),
    uglify: {
      options: {
        banner: "/*\n" + grunt.file.read('LICENSE') + "*/"
      },
      dist: {
        files: {
          '<%=pkg.name%>-<%=pkg.version%>.min.js': ['<%=pkg.name%>-<%=pkg.version%>.js']
        }
      }
    }
  });

  grunt.registerTask('build', 'build a browser file', function() {
    var done = this.async();

    var outfile = './brain-' + pkg.version + '.js';

    var bundle = browserify('./browser.js').bundle(function(err, src) {
      console.log("> " + outfile);

      // prepend license
      var license = fs.readFileSync("./LICENSE");
      src = "/*\n" + license + "*/" + src;

      // write out the browser file
      fs.writeFileSync(outfile, src);
      done();
    });
  });
  grunt.registerTask('test', 'mochaTest');

  grunt.loadNpmTasks('grunt-mocha-test');
  grunt.loadNpmTasks('grunt-contrib-uglify');
};


================================================
FILE: LICENSE
================================================
Copyright (c) 2010 Heather Arthur

Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:

The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.



================================================
FILE: README.md
================================================
*This project has reached the end of its development as a simple neural network library. Feel free to browse the code, but please use other JavaScript neural network libraries in development like [brain.js](https://github.com/BrainJS/brain.js) and [convnetjs](https://github.com/karpathy/convnetjs).*

# brain

`brain` is a JavaScript [neural network](http://neuralnetworksanddeeplearning.com/) library. Here's an example of using it to approximate the XOR function:

```javascript
var net = new brain.NeuralNetwork();

net.train([{input: [0, 0], output: [0]},
           {input: [0, 1], output: [1]},
           {input: [1, 0], output: [1]},
           {input: [1, 1], output: [0]}]);

var output = net.run([1, 0]);  // [0.987]
```

There's no reason to use a neural network to figure out XOR however (-: so here's a more involved, realistic example:
[Demo: training a neural network to recognize color contrast](http://harthur.github.com/brain/)

## Using in node
If you have [node](http://nodejs.org/) you can install with [npm](http://npmjs.org):

```
npm install brain
```

## Using in the browser
Download the latest [brain.js](https://github.com/harthur/brain/tree/gh-pages). Training is computationally expensive, so you should try to train the network offline (or on a Worker) and use the `toFunction()` or `toJSON()` options to plug the pre-trained network in to your website.

## Training
Use `train()` to train the network with an array of training data. The network has to be trained with all the data in bulk in one call to `train()`. The more training patterns, the longer it will probably take to train, but the better the network will be at classifiying new patterns.

#### Data format
Each training pattern should have an `input` and an `output`, both of which can be either an array of numbers from `0` to `1` or a hash of numbers from `0` to `1`. For the [color constrast demo](http://harthur.github.com/brain/) it looks something like this:

```javascript
var net = new brain.NeuralNetwork();

net.train([{input: { r: 0.03, g: 0.7, b: 0.5 }, output: { black: 1 }},
           {input: { r: 0.16, g: 0.09, b: 0.2 }, output: { white: 1 }},
           {input: { r: 0.5, g: 0.5, b: 1.0 }, output: { white: 1 }}]);

var output = net.run({ r: 1, g: 0.4, b: 0 });  // { white: 0.99, black: 0.002 }
```

#### Options
`train()` takes a hash of options as its second argument:

```javascript
net.train(data, {
  errorThresh: 0.005,  // error threshold to reach
  iterations: 20000,   // maximum training iterations
  log: true,           // console.log() progress periodically
  logPeriod: 10,       // number of iterations between logging
  learningRate: 0.3    // learning rate
})
```

The network will train until the training error has gone below the threshold (default `0.005`) or the max number of iterations (default `20000`) has been reached, whichever comes first.

By default training won't let you know how its doing until the end, but set `log` to `true` to get periodic updates on the current training error of the network. The training error should decrease every time. The updates will be printed to console. If you set `log` to a function, this function will be called with the updates instead of printing to the console.

The learning rate is a parameter that influences how quickly the network trains. It's a number from `0` to `1`. If the learning rate is close to `0` it will take longer to train. If the learning rate is closer to `1` it will train faster but it's in danger of training to a local minimum and performing badly on new data. The default learning rate is `0.3`.

#### Output
The output of `train()` is a hash of information about how the training went:

```javascript
{
  error: 0.0039139985510105032,  // training error
  iterations: 406                // training iterations
}
```

#### Failing
If the network failed to train, the error will be above the error threshold. This could happen because the training data is too noisy (most likely), the network doesn't have enough hidden layers or nodes to handle the complexity of the data, or it hasn't trained for enough iterations.

If the training error is still something huge like `0.4` after 20000 iterations, it's a good sign that the network can't make sense of the data you're giving it.

## JSON
Serialize or load in the state of a trained network with JSON:

```javascript
var json = net.toJSON();

net.fromJSON(json);
```

You can also get a custom standalone function from a trained network that acts just like `run()`:

```javascript
var run = net.toFunction();

var output = run({ r: 1, g: 0.4, b: 0 });

console.log(run.toString()); // copy and paste! no need to import brain.js
```

## Options
`NeuralNetwork()` takes a hash of options:

```javascript
var net = new brain.NeuralNetwork({
  hiddenLayers: [4],
  learningRate: 0.6 // global learning rate, useful when training using streams
});
```

#### hiddenLayers
Specify the number of hidden layers in the network and the size of each layer. For example, if you want two hidden layers - the first with 3 nodes and the second with 4 nodes, you'd give:

```
hiddenLayers: [3, 4]
```

By default `brain` uses one hidden layer with size proportionate to the size of the input array.

## Streams
The network now has a [WriteStream](http://nodejs.org/api/stream.html#stream_class_stream_writable). You can train the network by using `pipe()` to send the training data to the network.

#### Example
Refer to `stream-example.js` for an example on how to train the network with a stream.

#### Initialization
To train the network using a stream you must first create the stream by calling `net.createTrainStream()` which takes the following options:

* `floodCallback()` - the callback function to re-populate the stream. This gets called on every training iteration.
* `doneTrainingCallback(info)` - the callback function to execute when the network is done training. The `info` param will contain a hash of information about how the training went:

```javascript
{
  error: 0.0039139985510105032,  // training error
  iterations: 406                // training iterations
}
```

#### Transform
Use a [Transform](http://nodejs.org/api/stream.html#stream_class_stream_transform) to coerce the data into the correct format. You might also use a Transform stream to normalize your data on the fly.


================================================
FILE: bower.json
================================================
{
  "name": "brain",
  "version": "0.7.0",
  "homepage": "https://github.com/harthur/brain",
  "authors": [
    "Heather Arthur <fayearthur@gmail.com>"
  ],
  "description": "Neural network library",
  "keywords": [
    "neural-networks",
    "machine-learning",
    "classifier"
  ],
  "main": "lib/brain.js",
  "ignore": [
    "node_modules",
    "test"
  ],
  "dependencies": {
    "underscore": ">=1.5.1"
  }
}


================================================
FILE: browser.js
================================================
// this file is the entrypoint for building a browser file with browserify

brain = require("./lib/brain");

================================================
FILE: lib/brain.js
================================================
exports.NeuralNetwork = require("./neuralnetwork").NeuralNetwork;
exports.crossValidate = require("./cross-validate");


================================================
FILE: lib/cross-validate.js
================================================
var _ = require("underscore")._;

function testPartition(classifierConst, opts, trainOpts, trainSet, testSet) {
  var classifier = new classifierConst(opts);

  var beginTrain = Date.now();

  var trainingStats = classifier.train(trainSet, trainOpts);

  var beginTest = Date.now();

  var testStats = classifier.test(testSet);

  var endTest = Date.now();

  var stats = _(testStats).extend({
    trainTime : beginTest - beginTrain,
    testTime : endTest - beginTest,
    iterations: trainingStats.iterations,
    trainError: trainingStats.error,
    learningRate: trainOpts.learningRate,
    hidden: classifier.hiddenSizes,
    network: classifier.toJSON()
  });

  return stats;
}

module.exports = function crossValidate(classifierConst, data, opts, trainOpts, k) {
  k = k || 4;
  var size = data.length / k;

  data = _(data).sortBy(function() {
    return Math.random();
  });

  var avgs = {
    error : 0,
    trainTime : 0,
    testTime : 0,
    iterations: 0,
    trainError: 0
  };

  var stats = {
    truePos: 0,
    trueNeg: 0,
    falsePos: 0,
    falseNeg: 0,
    total: 0
  };

  var misclasses = [];

  var results = _.range(k).map(function(i) {
    var dclone = _(data).clone();
    var testSet = dclone.splice(i * size, size);
    var trainSet = dclone;

    var result = testPartition(classifierConst, opts, trainOpts, trainSet, testSet);

    _(avgs).each(function(sum, stat) {
      avgs[stat] = sum + result[stat];
    });

    _(stats).each(function(sum, stat) {
      stats[stat] = sum + result[stat];
    })

    misclasses.push(result.misclasses);

    return result;
  });

  _(avgs).each(function(sum, i) {
    avgs[i] = sum / k;
  });

  stats.precision = stats.truePos / (stats.truePos + stats.falsePos);
  stats.recall = stats.truePos / (stats.truePos + stats.falseNeg);
  stats.accuracy = (stats.trueNeg + stats.truePos) / stats.total;

  stats.testSize = size;
  stats.trainSize = data.length - size;

  return {
    avgs: avgs,
    stats: stats,
    sets: results,
    misclasses: _(misclasses).flatten()
  };
}


================================================
FILE: lib/lookup.js
================================================
var _ = require("underscore");

/* Functions for turning sparse hashes into arrays and vice versa */

function buildLookup(hashes) {
  // [{a: 1}, {b: 6, c: 7}] -> {a: 0, b: 1, c: 2}
  var hash = _(hashes).reduce(function(memo, hash) {
    return _(memo).extend(hash);
  }, {});
  return lookupFromHash(hash);
}

function lookupFromHash(hash) {
  // {a: 6, b: 7} -> {a: 0, b: 1}
  var lookup = {};
  var index = 0;
  for (var i in hash) {
    lookup[i] = index++;
  }
  return lookup;
}

function toArray(lookup, hash) {
  // {a: 0, b: 1}, {a: 6} -> [6, 0]
  var array = [];
  for (var i in lookup) {
    array[lookup[i]] = hash[i] || 0;
  }
  return array;
}

function toHash(lookup, array) {
  // {a: 0, b: 1}, [6, 7] -> {a: 6, b: 7}
  var hash = {};
  for (var i in lookup) {
    hash[i] = array[lookup[i]];
  }
  return hash;
}

function lookupFromArray(array) {
  var lookup = {};
  // super fast loop
  var z = 0;
  var i = array.length;
  while (i-- > 0) {
    lookup[array[i]] = z++;
  };
  return lookup;
}

module.exports = {
  buildLookup: buildLookup,
  lookupFromHash: lookupFromHash,
  toArray: toArray,
  toHash: toHash,
  lookupFromArray: lookupFromArray
};

================================================
FILE: lib/neuralnetwork.js
================================================
var _ = require("underscore"),
    lookup = require("./lookup"),
    Writable = require('stream').Writable,
    inherits = require('inherits');

var NeuralNetwork = function(options) {
  options = options || {};
  this.learningRate = options.learningRate || 0.3;
  this.momentum = options.momentum || 0.1;
  this.hiddenSizes = options.hiddenLayers;

  this.binaryThresh = options.binaryThresh || 0.5;
}

NeuralNetwork.prototype = {
  initialize: function(sizes) {
    this.sizes = sizes;
    this.outputLayer = this.sizes.length - 1;

    this.biases = []; // weights for bias nodes
    this.weights = [];
    this.outputs = [];

    // state for training
    this.deltas = [];
    this.changes = []; // for momentum
    this.errors = [];

    for (var layer = 0; layer <= this.outputLayer; layer++) {
      var size = this.sizes[layer];
      this.deltas[layer] = zeros(size);
      this.errors[layer] = zeros(size);
      this.outputs[layer] = zeros(size);

      if (layer > 0) {
        this.biases[layer] = randos(size);
        this.weights[layer] = new Array(size);
        this.changes[layer] = new Array(size);

        for (var node = 0; node < size; node++) {
          var prevSize = this.sizes[layer - 1];
          this.weights[layer][node] = randos(prevSize);
          this.changes[layer][node] = zeros(prevSize);
        }
      }
    }
  },

  run: function(input) {
    if (this.inputLookup) {
      input = lookup.toArray(this.inputLookup, input);
    }

    var output = this.runInput(input);

    if (this.outputLookup) {
      output = lookup.toHash(this.outputLookup, output);
    }
    return output;
  },

  runInput: function(input) {
    this.outputs[0] = input;  // set output state of input layer

    for (var layer = 1; layer <= this.outputLayer; layer++) {
      for (var node = 0; node < this.sizes[layer]; node++) {
        var weights = this.weights[layer][node];

        var sum = this.biases[layer][node];
        for (var k = 0; k < weights.length; k++) {
          sum += weights[k] * input[k];
        }
        this.outputs[layer][node] = 1 / (1 + Math.exp(-sum));
      }
      var output = input = this.outputs[layer];
    }
    return output;
  },

  train: function(data, options) {
    data = this.formatData(data);

    options = options || {};
    var iterations = options.iterations || 20000;
    var errorThresh = options.errorThresh || 0.005;
    var log = options.log ? (_.isFunction(options.log) ? options.log : console.log) : false;
    var logPeriod = options.logPeriod || 10;
    var learningRate = options.learningRate || this.learningRate || 0.3;
    var callback = options.callback;
    var callbackPeriod = options.callbackPeriod || 10;

    var inputSize = data[0].input.length;
    var outputSize = data[0].output.length;

    var hiddenSizes = this.hiddenSizes;
    if (!hiddenSizes) {
      hiddenSizes = [Math.max(3, Math.floor(inputSize / 2))];
    }
    var sizes = _([inputSize, hiddenSizes, outputSize]).flatten();
    this.initialize(sizes);

    var error = 1;
    for (var i = 0; i < iterations && error > errorThresh; i++) {
      var sum = 0;
      for (var j = 0; j < data.length; j++) {
        var err = this.trainPattern(data[j].input, data[j].output, learningRate);
        sum += err;
      }
      error = sum / data.length;

      if (log && (i % logPeriod == 0)) {
        log("iterations:", i, "training error:", error);
      }
      if (callback && (i % callbackPeriod == 0)) {
        callback({ error: error, iterations: i });
      }
    }

    return {
      error: error,
      iterations: i
    };
  },

  trainPattern : function(input, target, learningRate) {
    learningRate = learningRate || this.learningRate;

    // forward propogate
    this.runInput(input);

    // back propogate
    this.calculateDeltas(target);
    this.adjustWeights(learningRate);

    var error = mse(this.errors[this.outputLayer]);
    return error;
  },

  calculateDeltas: function(target) {
    for (var layer = this.outputLayer; layer >= 0; layer--) {
      for (var node = 0; node < this.sizes[layer]; node++) {
        var output = this.outputs[layer][node];

        var error = 0;
        if (layer == this.outputLayer) {
          error = target[node] - output;
        }
        else {
          var deltas = this.deltas[layer + 1];
          for (var k = 0; k < deltas.length; k++) {
            error += deltas[k] * this.weights[layer + 1][k][node];
          }
        }
        this.errors[layer][node] = error;
        this.deltas[layer][node] = error * output * (1 - output);
      }
    }
  },

  adjustWeights: function(learningRate) {
    for (var layer = 1; layer <= this.outputLayer; layer++) {
      var incoming = this.outputs[layer - 1];

      for (var node = 0; node < this.sizes[layer]; node++) {
        var delta = this.deltas[layer][node];

        for (var k = 0; k < incoming.length; k++) {
          var change = this.changes[layer][node][k];

          change = (learningRate * delta * incoming[k])
                   + (this.momentum * change);

          this.changes[layer][node][k] = change;
          this.weights[layer][node][k] += change;
        }
        this.biases[layer][node] += learningRate * delta;
      }
    }
  },

  formatData: function(data) {
    if (!_.isArray(data)) { // turn stream datum into array
      var tmp = [];
      tmp.push(data);
      data = tmp;
    }
    // turn sparse hash input into arrays with 0s as filler
    var datum = data[0].input;
    if (!_(datum).isArray() && !(datum instanceof Float64Array)) {
      if (!this.inputLookup) {
        this.inputLookup = lookup.buildLookup(_(data).pluck("input"));
      }
      data = data.map(function(datum) {
        var array = lookup.toArray(this.inputLookup, datum.input)
        return _(_(datum).clone()).extend({ input: array });
      }, this);
    }

    if (!_(data[0].output).isArray()) {
      if (!this.outputLookup) {
        this.outputLookup = lookup.buildLookup(_(data).pluck("output"));
      }
      data = data.map(function(datum) {
        var array = lookup.toArray(this.outputLookup, datum.output);
        return _(_(datum).clone()).extend({ output: array });
      }, this);
    }
    return data;
  },

  test : function(data) {
    data = this.formatData(data);

    // for binary classification problems with one output node
    var isBinary = data[0].output.length == 1;
    var falsePos = 0,
        falseNeg = 0,
        truePos = 0,
        trueNeg = 0;

    // for classification problems
    var misclasses = [];

    // run each pattern through the trained network and collect
    // error and misclassification statistics
    var sum = 0;
    for (var i = 0; i < data.length; i++) {
      var output = this.runInput(data[i].input);
      var target = data[i].output;

      var actual, expected;
      if (isBinary) {
        actual = output[0] > this.binaryThresh ? 1 : 0;
        expected = target[0];
      }
      else {
        actual = output.indexOf(_(output).max());
        expected = target.indexOf(_(target).max());
      }

      if (actual != expected) {
        var misclass = data[i];
        _(misclass).extend({
          actual: actual,
          expected: expected
        })
        misclasses.push(misclass);
      }

      if (isBinary) {
        if (actual == 0 && expected == 0) {
          trueNeg++;
        }
        else if (actual == 1 && expected == 1) {
          truePos++;
        }
        else if (actual == 0 && expected == 1) {
          falseNeg++;
        }
        else if (actual == 1 && expected == 0) {
          falsePos++;
        }
      }

      var errors = output.map(function(value, i) {
        return target[i] - value;
      });
      sum += mse(errors);
    }
    var error = sum / data.length;

    var stats = {
      error: error,
      misclasses: misclasses
    };

    if (isBinary) {
      _(stats).extend({
        trueNeg: trueNeg,
        truePos: truePos,
        falseNeg: falseNeg,
        falsePos: falsePos,
        total: data.length,
        precision: truePos / (truePos + falsePos),
        recall: truePos / (truePos + falseNeg),
        accuracy: (trueNeg + truePos) / data.length
      })
    }
    return stats;
  },

  toJSON: function() {
    /* make json look like:
      {
        layers: [
          { x: {},
            y: {}},
          {'0': {bias: -0.98771313, weights: {x: 0.8374838, y: 1.245858},
           '1': {bias: 3.48192004, weights: {x: 1.7825821, y: -2.67899}}},
          { f: {bias: 0.27205739, weights: {'0': 1.3161821, '1': 2.00436}}}
        ]
      }
    */
    var layers = [];
    for (var layer = 0; layer <= this.outputLayer; layer++) {
      layers[layer] = {};

      var nodes;
      // turn any internal arrays back into hashes for readable json
      if (layer == 0 && this.inputLookup) {
        nodes = _(this.inputLookup).keys();
      }
      else if (layer == this.outputLayer && this.outputLookup) {
        nodes = _(this.outputLookup).keys();
      }
      else {
        nodes = _.range(0, this.sizes[layer]);
      }

      for (var j = 0; j < nodes.length; j++) {
        var node = nodes[j];
        layers[layer][node] = {};

        if (layer > 0) {
          layers[layer][node].bias = this.biases[layer][j];
          layers[layer][node].weights = {};
          for (var k in layers[layer - 1]) {
            var index = k;
            if (layer == 1 && this.inputLookup) {
              index = this.inputLookup[k];
            }
            layers[layer][node].weights[k] = this.weights[layer][j][index];
          }
        }
      }
    }
    return { layers: layers, outputLookup:!!this.outputLookup, inputLookup:!!this.inputLookup };
  },

  fromJSON: function(json) {
    var size = json.layers.length;
    this.outputLayer = size - 1;

    this.sizes = new Array(size);
    this.weights = new Array(size);
    this.biases = new Array(size);
    this.outputs = new Array(size);

    for (var i = 0; i <= this.outputLayer; i++) {
      var layer = json.layers[i];
      if (i == 0 && (!layer[0] || json.inputLookup)) {
        this.inputLookup = lookup.lookupFromHash(layer);
      }
      else if (i == this.outputLayer && (!layer[0] || json.outputLookup)) {
        this.outputLookup = lookup.lookupFromHash(layer);
      }

      var nodes = _(layer).keys();
      this.sizes[i] = nodes.length;
      this.weights[i] = [];
      this.biases[i] = [];
      this.outputs[i] = [];

      for (var j in nodes) {
        var node = nodes[j];
        this.biases[i][j] = layer[node].bias;
        this.weights[i][j] = _(layer[node].weights).toArray();
      }
    }
    return this;
  },

   toFunction: function() {
    var json = this.toJSON();
    // return standalone function that mimics run()
    return new Function("input",
'  var net = ' + JSON.stringify(json) + ';\n\n\
  for (var i = 1; i < net.layers.length; i++) {\n\
    var layer = net.layers[i];\n\
    var output = {};\n\
    \n\
    for (var id in layer) {\n\
      var node = layer[id];\n\
      var sum = node.bias;\n\
      \n\
      for (var iid in node.weights) {\n\
        sum += node.weights[iid] * input[iid];\n\
      }\n\
      output[id] = (1 / (1 + Math.exp(-sum)));\n\
    }\n\
    input = output;\n\
  }\n\
  return output;');
  },

  // This will create a TrainStream (WriteStream)
  //  for us to send the training data to.
  //  param: opts - the training options
  createTrainStream: function(opts) {
    opts = opts || {};
    opts.neuralNetwork = this;
    this.trainStream = new TrainStream(opts);
    return this.trainStream;
  }
}

function randomWeight() {
  return Math.random() * 0.4 - 0.2;
}

function zeros(size) {
  var array = new Array(size);
  for (var i = 0; i < size; i++) {
    array[i] = 0;
  }
  return array;
}

function randos(size) {
  var array = new Array(size);
  for (var i = 0; i < size; i++) {
    array[i] = randomWeight();
  }
  return array;
}

function mse(errors) {
  // mean squared error
  var sum = 0;
  for (var i = 0; i < errors.length; i++) {
    sum += Math.pow(errors[i], 2);
  }
  return sum / errors.length;
}

exports.NeuralNetwork = NeuralNetwork;

function TrainStream(opts) {
  Writable.call(this, {
    objectMode: true
  });

  opts = opts || {};

  // require the neuralNetwork
  if (!opts.neuralNetwork) {
    throw new Error('no neural network specified');
  }

  this.neuralNetwork = opts.neuralNetwork;
  this.dataFormatDetermined = false;

  this.inputKeys = [];
  this.outputKeys = []; // keeps track of keys seen
  this.i = 0; // keep track of the for loop i variable that we got rid of
  this.iterations = opts.iterations || 20000;
  this.errorThresh = opts.errorThresh || 0.005;
  this.log = opts.log ? (_.isFunction(opts.log) ? opts.log : console.log) : false;
  this.logPeriod = opts.logPeriod || 10;
  this.callback = opts.callback;
  this.callbackPeriod = opts.callbackPeriod || 10;
  this.floodCallback = opts.floodCallback;
  this.doneTrainingCallback = opts.doneTrainingCallback;

  this.size = 0;
  this.count = 0;

  this.sum = 0;

  this.on('finish', this.finishStreamIteration);

  return this;
}

inherits(TrainStream, Writable);

/*
  _write expects data to be in the form of a datum.
  ie. {input: {a: 1 b: 0}, output: {z: 0}}
 */
TrainStream.prototype._write = function(chunk, enc, next) {
  if (!chunk) { // check for the end of one interation of the stream
    this.emit('finish');
    return next();
  }

  if (!this.dataFormatDetermined) {
    this.size++;
    this.inputKeys = _.union(this.inputKeys, _.keys(chunk.input));
    this.outputKeys = _.union(this.outputKeys, _.keys(chunk.output));
    this.firstDatum = this.firstDatum || chunk;
    return next();
  }

  this.count++;

  var data = this.neuralNetwork.formatData(chunk);
  this.trainDatum(data[0]);

  // tell the Readable Stream that we are ready for more data
  next();
}

TrainStream.prototype.trainDatum = function(datum) {
  var err = this.neuralNetwork.trainPattern(datum.input, datum.output);
  this.sum += err;
}

TrainStream.prototype.finishStreamIteration = function() {
  if (this.dataFormatDetermined && this.size !== this.count) {
    console.log("This iteration's data length was different from the first.");
  }

  if (!this.dataFormatDetermined) {
    // create the lookup
    this.neuralNetwork.inputLookup = lookup.lookupFromArray(this.inputKeys);
    if(!_.isArray(this.firstDatum.output)){
      this.neuralNetwork.outputLookup = lookup.lookupFromArray(this.outputKeys);
    }

    var data = this.neuralNetwork.formatData(this.firstDatum);
    var inputSize = data[0].input.length;
    var outputSize = data[0].output.length;

    var hiddenSizes = this.hiddenSizes;
    if (!hiddenSizes) {
      hiddenSizes = [Math.max(3, Math.floor(inputSize / 2))];
    }
    var sizes = _([inputSize, hiddenSizes, outputSize]).flatten();
    this.dataFormatDetermined = true;
    this.neuralNetwork.initialize(sizes);

    if (typeof this.floodCallback === 'function') {
      this.floodCallback();
    }
    return;
  }

  var error = this.sum / this.size;

  if (this.log && (this.i % this.logPeriod == 0)) {
    this.log("iterations:", this.i, "training error:", error);
  }
  if (this.callback && (this.i % this.callbackPeriod == 0)) {
    this.callback({
      error: error,
      iterations: this.i
    });
  }

  this.sum = 0;
  this.count = 0;
  // update the iterations
  this.i++;

  // do a check here to see if we need the stream again
  if (this.i < this.iterations && error > this.errorThresh) {
    if (typeof this.floodCallback === 'function') {
      return this.floodCallback();
    }
  } else {
    // done training
    if (typeof this.doneTrainingCallback === 'function') {
      return this.doneTrainingCallback({
        error: error,
        iterations: this.i
      });
    }
  }
}


================================================
FILE: package.json
================================================
{
    "name": "brain",
    "description": "Neural network library",
    "version": "0.7.0",
    "author": "Heather Arthur <fayearthur@gmail.com>",
    "repository": {
        "type": "git",
        "url": "http://github.com/harthur/brain.git"
    },
    "scripts": {
        "test-unit": "mocha test/unit",
        "test-cv": "mocha test/cross-validation --timeout 10000",
        "test": "npm run test-unit && npm run test-cv"
    },
    "main": "./lib/brain",
    "dependencies": {
        "underscore": ">=1.5.1",
        "inherits": "~2.0.1"
    },
    "devDependencies": {
        "mocha": ">=1.0.0",
        "canvas": ">=0.10.0",
        "grunt": "~0.4.3",
        "grunt-contrib-uglify": "~0.2.0",
        "grunt-mocha-test": "~0.11.0",
        "browserify": "~3.32.0"
    },
    "keywords": ["neural network", "classifier", "machine learning"]
}


================================================
FILE: stream-example.js
================================================
var assert = require("assert"),
    brain = require("./lib/brain");

var net = new brain.NeuralNetwork();

var xor = [
  { input: [0, 0], output: [0]},
  { input: [0, 1], output: [1]},
  { input: [1, 0], output: [1]},
  { input: [1, 1], output: [0]}];

var trainStream = net.createTrainStream({
  /**
   * Write training data to the stream. Called on each training iteration.
   */
  floodCallback: function() {
    flood(trainStream, xor);
  },

  /**
   * Called when the network is done training.
   */
  doneTrainingCallback: function(obj) {
    console.log("trained in " + obj.iterations + " iterations with error: "
                + obj.error);

    var result = net.run([0, 1]);

    console.log("0 XOR 1: ", result);  // 0.987
  }
});

// kick it off
flood(trainStream, xor);


function flood(stream, data) {
  for (var i = 0; i < data.length; i++) {
    stream.write(data[i]);
  }
  // let it know we've reached the end of the data
  stream.write(null);
}


================================================
FILE: test/README.md
================================================
# Tests

To run the tests in this directory, make sure you've installed the dev dependencies with this command from the top-level directory:

```
npm install
```

Then you can run all tests (unit and cross-validation) using `npm test`.

# Unit tests
Run the unit tests with:

```
grunt test
```

or

`npm run test-unit`

# Cross-validation tests
The cross-validation tests will actually test how good the neural network is a training by getting a bunch of training data, training it with some, and using the rest as verification.

Cross-validation tests will take a long time to run, and in the end will give you a printout with the average error of the test data.

Run these with:

```
mocha test/cross-validation/* --timeout 10000
```

or

`npm run test-cv`


================================================
FILE: test/cross-validation/ocr.js
================================================
var canvas = require("canvas"),
    _ = require("underscore"),
    brain = require("../../lib/brain"),
    crossValidate = require("../../lib/cross-validate");

var dim = 24;

function getSampling(context, letter, font) {
  context.clearRect(0, 0, dim, dim);
  context.font = dim + "px " + font;
  context.fillText(letter, 0, dim);

  var data = context.getImageData(0, 0, dim, dim);
  var lumas = extractPoints(data);
  return lumas;
}

function extractPoints(imageData) {
  var points = [];
  for (var x = 0; x < imageData.width; x = x + 2) {
    for (var y = 0; y < imageData.height; y = y + 2) {
      var i = x * 4 + y * 4 * imageData.width;
      var r = imageData.data[i],
          g = imageData.data[i + 1],
          b = imageData.data[i + 2],
          a = imageData.data[i + 3];

      var luma = a == 0 ? 1 : (r * 299/1000 + g * 587/1000
        + b * 114/1000 ) / 255;

      points.push(luma);
    }
  }
  return points;
}

describe('OCR cross-validation', function() {
  it('recognize characters in different fonts', function() {
    var canvas = require("canvas");
    var canvas = new canvas(dim, dim);
    var context = canvas.getContext("2d");

    var letters = ["A", "B", "C", "D", "E",
                   "K", "O", "Z"];
    var fonts = ["Arial", "Courier", "Georgia", "Menlo", "Optima",
                 "Copperplate", "American Typewriter", "Comic Sans",
                 "Baskerville", "Verdana", "Helvetica", "Didot",
                 "Geneva", "Cracked", "Impact", "Cooper"];

    var data = [];

    letters.forEach(function(letter) {
       fonts.forEach(function(font) {
          var input = getSampling(context, letter, font);

          var output = {};
          output[letter] = 1;
          data.push({ input: input, output: output });
       });
    });

    console.log("Cross validating");
    var result = crossValidate(brain.NeuralNetwork, data, {});

    console.log("\nMisclassifications:");
    result.misclasses.forEach(function(misclass) {
      console.log("input: " + misclass.input
        + " actual: " + letters[misclass.actual]
        + " expected: " + letters[misclass.expected] + "\n")
    })

    console.log("\nCross-validation of OCR data:\n");
    console.log(result.avgs);

    console.log("\nMisclassification rate: "
      + result.misclasses.length / data.length);

    console.log("\nMean squared error: "
      + result.avgs.error);

    var perf = result.avgs.iterations / (result.avgs.trainTime / 1000);
    console.log("\nTraining iterations per second: " + perf);

    assert.ok(result.avgs.error < .1);
  })
})


================================================
FILE: test/unit/bitwise.js
================================================
var assert = require("assert"),
    brain = require("../../lib/brain");

var wiggle = 0.1;

function testBitwise(data, op) {
  var net = new brain.NeuralNetwork();
  net.train(data, { errorThresh: 0.003 });

  for(var i in data) {
    var output = net.run(data[i].input);
    var target = data[i].output;
    assert.ok(output < (target + wiggle) && output > (target - wiggle),
     "failed to train " + op + " - output: " + output + " target: " + target);
  }
}

describe('bitwise functions', function() {

  it('NOT function', function() {
    var not = [{input: [0], output: [1]},
               {input: [1], output: [0]}];
    testBitwise(not, "not");
  })

  it('XOR function', function() {
    var xor = [{input: [0, 0], output: [0]},
               {input: [0, 1], output: [1]},
               {input: [1, 0], output: [1]},
               {input: [1, 1], output: [0]}];
    testBitwise(xor, "xor");
  })

  it('OR function', function() {
    var or = [{input: [0, 0], output: [0]},
              {input: [0, 1], output: [1]},
              {input: [1, 0], output: [1]},
              {input: [1, 1], output: [1]}];
    testBitwise(or, "or");
  });

  it('AND function', function() {
    var and = [{input: [0, 0], output: [0]},
               {input: [0, 1], output: [0]},
               {input: [1, 0], output: [0]},
               {input: [1, 1], output: [1]}];
    testBitwise(and, "and");
  })
})


================================================
FILE: test/unit/hash.js
================================================
var assert = require("assert"),
    brain = require("../../lib/brain");

describe('hash input and output', function() {
  it('runs correctly with array input and output', function() {
    var net = new brain.NeuralNetwork();

    net.train([{input: [0, 0], output: [0]},
               {input: [0, 1], output: [1]},
               {input: [1, 0], output: [1]},
               {input: [1, 1], output: [0]}]);
    var output = net.run([1, 0]);

    assert.ok(output[0] > 0.9, "output: " + output[0]);
  })

 it('runs correctly with hash input', function() {
    var net = new brain.NeuralNetwork();

    var info = net.train([{input: { x: 0, y: 0 }, output: [0]},
               {input: { x: 0, y: 1 }, output: [1]},
               {input: { x: 1, y: 0 }, output: [1]},
               {input: { x: 1, y: 1 }, output: [0]}]);
    var output = net.run({x: 1, y: 0});

    assert.ok(output[0] > 0.9, "output: " + output[0]);
  })

 it('runs correctly with hash output', function() {
    var net = new brain.NeuralNetwork();

    net.train([{input: [0, 0], output: { answer: 0 }},
               {input: [0, 1], output: { answer: 1 }},
               {input: [1, 0], output: { answer: 1 }},
               {input: [1, 1], output: { answer: 0 }}]);

    var output = net.run([1, 0]);

    assert.ok(output.answer > 0.9, "output: " + output.answer);
  })

  it('runs correctly with hash input and output', function() {
    var net = new brain.NeuralNetwork();

    net.train([{input: { x: 0, y: 0 }, output: { answer: 0 }},
               {input: { x: 0, y: 1 }, output: { answer: 1 }},
               {input: { x: 1, y: 0 }, output: { answer: 1 }},
               {input: { x: 1, y: 1 }, output: { answer: 0 }}]);

    var output = net.run({x: 1, y: 0});

    assert.ok(output.answer > 0.9, "output: " + output.answer);
  })

  it('runs correctly with sparse hashes', function() {
      var net = new brain.NeuralNetwork();

      net.train([{input: {}, output: {}},
                 {input: { y: 1 }, output: { answer: 1 }},
                 {input: { x: 1 }, output: { answer: 1 }},
                 {input: { x: 1, y: 1 }, output: {}}]);


      var output = net.run({x: 1});

      assert.ok(output.answer > 0.9);
  })

  it('runs correctly with unseen input', function() {
      var net = new brain.NeuralNetwork();

      net.train([{input: {}, output: {}},
                 {input: { y: 1 }, output: { answer: 1 }},
                 {input: { x: 1 }, output: { answer: 1 }},
                 {input: { x: 1, y: 1 }, output: {}}]);

      var output = net.run({x: 1, z: 1});
      assert.ok(output.answer > 0.9);
  })
})


================================================
FILE: test/unit/json.js
================================================
var assert = require("assert"),
    brain = require("../../lib/brain");

describe('JSON', function() {
  var net = new brain.NeuralNetwork();

  net.train([{input:  {"0": Math.random(), b: Math.random()},
              output: {c: Math.random(), "0": Math.random()}},
             {input:  {"0": Math.random(), b: Math.random()},
              output: {c: Math.random(), "0": Math.random()}}]);

  var serialized = net.toJSON();
  var net2 = new brain.NeuralNetwork().fromJSON(serialized);

  var input = {"0" : Math.random(), b: Math.random()};

  it('toJSON()/fromJSON()', function() {
    var output1 = net.run(input);
    var output2 = net2.run(input);

    assert.equal(JSON.stringify(output1), JSON.stringify(output2),
                  "loading json serialized network failed");
  })


  it('toFunction()', function() {
    var output1 = net.run(input);
    var output2 = net.toFunction()(input);

    assert.equal(JSON.stringify(output1), JSON.stringify(output2),
                   "standalone network function failed");
  })
})


================================================
FILE: test/unit/lookup.js
================================================
var assert = require("assert"),
    lookup = require("../../lib/lookup");


describe('lookup', function() {
  it('lookupFromHash()', function() {
    var lup = lookup.lookupFromHash({ a: 6, b: 7, c: 8 });

    assert.deepEqual(lup, { a: 0, b: 1, c: 2 });
  })

  it('buildLookup()', function() {
    var lup = lookup.buildLookup([{ x: 0, y: 0 },
      { x: 1, z: 0 },
      { q: 0 },
      { x: 1, y: 1 }]);

    assert.deepEqual(lup, { x: 0, y: 1, z: 2, q: 3 })
  })

  it('toArray()', function() {
    var lup = { a: 0, b: 1, c: 2 };

    var array = lookup.toArray(lup, { b: 8, notinlookup: 9 });

    assert.deepEqual(array, [0, 8, 0])
  })

  it('toHash()', function() {
    var lup = { b: 1, a: 0, c: 2 };

    var hash = lookup.toHash(lup, [0, 9, 8]);

    assert.deepEqual(hash, {a: 0, b: 9, c: 8})
  })
})


================================================
FILE: test/unit/options.js
================================================
var assert = require("assert"),
    _ = require("underscore"),
    brain = require("../../lib/brain");

describe('neural network options', function() {
  it('hiddenLayers', function() {
    var net = new brain.NeuralNetwork({ hiddenLayers: [8, 7] });

    net.train([{input: [0, 0], output: [0]},
               {input: [0, 1], output: [1]},
               {input: [1, 0], output: [1]},
               {input: [1, 1], output: [0]}]);

    var json = net.toJSON();

    assert.equal(json.layers.length, 4);
    assert.equal(_(json.layers[1]).keys().length, 8);
    assert.equal(_(json.layers[2]).keys().length, 7);
  })

  it('hiddenLayers default expand to input size', function() {
    var net = new brain.NeuralNetwork();

    net.train([{input: [0, 0, 1, 1, 1, 1, 1, 1, 1], output: [0]},
               {input: [0, 1, 1, 1, 1, 1, 1, 1, 1], output: [1]},
               {input: [1, 0, 1, 1, 1, 1, 1, 1, 1], output: [1]},
               {input: [1, 1, 1, 1, 1, 1, 1, 1, 1], output: [0]}]);

    var json = net.toJSON();

    assert.equal(json.layers.length, 3);
    assert.equal(_(json.layers[1]).keys().length, 4, "9 input units means 4 hidden");
  })


  it('learningRate - higher learning rate should train faster', function() {
    var data = [{input: [0, 0], output: [0]},
                {input: [0, 1], output: [1]},
                {input: [1, 0], output: [1]},
                {input: [1, 1], output: [1]}];

    var net1 = new brain.NeuralNetwork();
    var iters1 = net1.train(data, { learningRate: 0.5 }).iterations;

    var net2 = new brain.NeuralNetwork();
    var iters2 = net2.train(data, { learningRate: 0.8 }).iterations;

    assert.ok(iters1 > (iters2 * 1.1), iters1 + " !> " + iters2 * 1.1);
  })

  it('learningRate - backwards compatibility', function() {
    var data = [{input: [0, 0], output: [0]},
                {input: [0, 1], output: [1]},
                {input: [1, 0], output: [1]},
                {input: [1, 1], output: [1]}];

    var net1 = new brain.NeuralNetwork({ learningRate: 0.5 });
    var iters1 = net1.train(data).iterations;

    var net2 = new brain.NeuralNetwork( { learningRate: 0.8 });
    var iters2 = net2.train(data).iterations;

    assert.ok(iters1 > (iters2 * 1.1), iters1 + " !> " + iters2 * 1.1);
  })

  it('momentum - higher momentum should train faster', function() {
    var data = [{input: [0, 0], output: [0]},
                {input: [0, 1], output: [1]},
                {input: [1, 0], output: [1]},
                {input: [1, 1], output: [1]}];

    var net1 = new brain.NeuralNetwork({ momentum: 0.1 });
    var iters1 = net1.train(data).iterations;

    var net2 = new brain.NeuralNetwork({ momentum: 0.5 });
    var iters2 = net2.train(data).iterations;

    assert.ok(iters1 > (iters2 * 1.1), iters1 + " !> " + (iters2 * 1.1));
  })

  describe('log', function () {
    var logCalled;

    beforeEach(function () {
      logCalled = false;
    });

    function logFunction() {
      logCalled = true;
    }

    function trainWithLog(log) {
      var net = new brain.NeuralNetwork();
      net.train([{input: [0], output: [0]}],
        {
          log: log,
          logPeriod: 1
        });
      }

      it('should call console.log if log === true', function () {
        var originalLog = console.log;
        console.log = logFunction;

        trainWithLog(true);

        console.log = originalLog;
        assert.equal(logCalled, true);
      })

      it('should call the given log function', function () {
        trainWithLog(logFunction);

        assert.equal(logCalled, true);
      })
  })
})


================================================
FILE: test/unit/stream-bitwise.js
================================================
var assert = require("assert"),
    brain = require("../../lib/brain");

function StreamTester(opts) {
  if (!(this instanceof StreamTester)) return new StreamTester(opts);

  var self = this;

  this.wiggle = opts.wiggle || 0.1;
  this.op = opts.op;

  this.testData = opts.testData;
  this.fakeBuffer = [];
  this.errorThresh = opts.errorThresh || 0.004;

  this.net = new brain.NeuralNetwork();

  this.trainStream = this.net.createTrainStream({
    floodCallback: self.flood.bind(self),
    doneTrainingCallback: self.doneTraining.bind(self),
    errorThresh: self.errorThresh // error threshold to reach
  });
  this.flood();
}

/*
  Every time you finish an epoch of flood,
  you must write null to the stream
  to let it know we have reached the end of the epoch
 */
StreamTester.prototype.flood = function() {
  var self = this;

  for (var i = self.testData.length - 1; i >= 0; i--) {
    self.trainStream.write(self.testData[i]);
  }
  self.trainStream.write(null);
}

StreamTester.prototype.doneTraining = function(info) {
  var self = this;

  for (var i in self.testData) {
    var output = self.net.run(self.testData[i].input)[0];
    var target = self.testData[i].output;
    assert.ok(output < (target + self.wiggle) && output > (target - self.wiggle),
      "failed to train " + self.op + " - output: " + output + " target: " + target);
  }
}


function testBitwise(data, op) {
  var st = StreamTester({
    testData: data,
    op: op,
    wiggle: 0.1,
    errorThresh: 0.003
  });
}

describe('bitwise functions', function() {

  it('NOT function', function() {
    var not = [{
      input: [0],
      output: [1]
    }, {
      input: [1],
      output: [0]
    }];
    testBitwise(not, "not");
  })

  it('XOR function', function() {
    var xor = [{
      input: [0, 0],
      output: [0]
    }, {
      input: [0, 1],
      output: [1]
    }, {
      input: [1, 0],
      output: [1]
    }, {
      input: [1, 1],
      output: [0]
    }];
    testBitwise(xor, "xor");
  })

  it('OR function', function() {
    var or = [{
      input: [0, 0],
      output: [0]
    }, {
      input: [0, 1],
      output: [1]
    }, {
      input: [1, 0],
      output: [1]
    }, {
      input: [1, 1],
      output: [1]
    }];
    testBitwise(or, "or");
  });

  it('AND function', function() {
    var and = [{
      input: [0, 0],
      output: [0]
    }, {
      input: [0, 1],
      output: [0]
    }, {
      input: [1, 0],
      output: [0]
    }, {
      input: [1, 1],
      output: [1]
    }];
    testBitwise(and, "and");
  })
})


================================================
FILE: test/unit/trainopts.js
================================================
var assert = require("assert"),
    brain = require("../../lib/brain");

var data = [{input: [0, 0], output: [0]},
            {input: [0, 1], output: [1]},
            {input: [1, 0], output: [1]},
            {input: [1, 1], output: [1]}];

describe('train() options', function() {
  it('train until error threshold reached', function() {
    var net = new brain.NeuralNetwork();
    var error = net.train(data, {
      errorThresh: 0.2,
      iterations: 100000
    }).error;

    assert.ok(error < 0.2, "network did not train until error threshold was reached");
  });

  it('train until max iterations reached', function() {
    var net = new brain.NeuralNetwork();
    var stats = net.train(data, {
      errorThresh: 0.001,
      iterations: 1
    });

    assert.equal(stats.iterations, 1);
  })

  it('training callback called with training stats', function(done) {
    var iters = 100;
    var period = 20;
    var target = iters / 20;

    var calls = 0;

    var net = new brain.NeuralNetwork();
    net.train(data, {
      iterations: iters,
      callback: function(stats) {
        assert.ok(stats.iterations % period == 0);

        calls++;
        if (calls == target) {
          done();
        }
      },
      callbackPeriod: 20
    });
  });
})