Repository: karpathy/convnetjs Branch: master Commit: 4c3358a315b4 Files: 54 Total size: 505.7 KB Directory structure: gitextract_qh9a6ot7/ ├── LICENSE ├── Readme.md ├── bower.json ├── build/ │ ├── deepqlearn.js │ ├── util.js │ └── vis.js ├── compile/ │ ├── build.xml │ └── yuicompressor-2.4.8.jar ├── demo/ │ ├── autoencoder.html │ ├── automatic.html │ ├── cifar10.html │ ├── classify2d.html │ ├── css/ │ │ ├── automatic.css │ │ └── style.css │ ├── image_regression.html │ ├── js/ │ │ ├── autoencoder.js │ │ ├── automatic.js │ │ ├── classify2d.js │ │ ├── image-helpers.js │ │ ├── image_regression.js │ │ ├── images-demo.js │ │ ├── npgmain.js │ │ ├── pica.js │ │ ├── regression.js │ │ ├── rldemo.js │ │ └── trainers.js │ ├── mnist.html │ ├── regression.html │ ├── rldemo.html │ ├── speedtest.html │ └── trainers.html ├── src/ │ ├── convnet_export.js │ ├── convnet_init.js │ ├── convnet_layers_dotproducts.js │ ├── convnet_layers_dropout.js │ ├── convnet_layers_input.js │ ├── convnet_layers_loss.js │ ├── convnet_layers_nonlinearities.js │ ├── convnet_layers_normalization.js │ ├── convnet_layers_pool.js │ ├── convnet_magicnet.js │ ├── convnet_net.js │ ├── convnet_trainers.js │ ├── convnet_util.js │ ├── convnet_vol.js │ └── convnet_vol_util.js └── test/ └── jasmine/ ├── MIT.LICENSE ├── SpecRunner.html ├── lib/ │ └── jasmine-2.0.0/ │ ├── boot.js │ ├── console.js │ ├── jasmine-html.js │ ├── jasmine.css │ └── jasmine.js └── spec/ └── NeuralNetSpec.js ================================================ FILE CONTENTS ================================================ ================================================ FILE: LICENSE ================================================ The MIT License Copyright (c) 2014 Andrej Karpathy Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: Readme.md ================================================ # ConvNetJS ConvNetJS is a Javascript implementation of Neural networks, together with nice browser-based demos. It currently supports: - Common **Neural Network modules** (fully connected layers, non-linearities) - Classification (SVM/Softmax) and Regression (L2) **cost functions** - Ability to specify and train **Convolutional Networks** that process images - An experimental **Reinforcement Learning** module, based on Deep Q Learning For much more information, see the main page at [convnetjs.com](http://convnetjs.com) **Note**: I am not actively maintaining ConvNetJS anymore because I simply don't have time. I think the npm repo might not work at this point. ## Online Demos - [Convolutional Neural Network on MNIST digits](http://cs.stanford.edu/~karpathy/convnetjs/demo/mnist.html) - [Convolutional Neural Network on CIFAR-10](http://cs.stanford.edu/~karpathy/convnetjs/demo/cifar10.html) - [Toy 2D data](http://cs.stanford.edu/~karpathy/convnetjs/demo/classify2d.html) - [Toy 1D regression](http://cs.stanford.edu/~karpathy/convnetjs/demo/regression.html) - [Training an Autoencoder on MNIST digits](http://cs.stanford.edu/~karpathy/convnetjs/demo/autoencoder.html) - [Deep Q Learning Reinforcement Learning demo](http://cs.stanford.edu/people/karpathy/convnetjs/demo/rldemo.html) - [Image Regression ("Painting")](http://cs.stanford.edu/~karpathy/convnetjs/demo/image_regression.html) - [Comparison of SGD/Adagrad/Adadelta on MNIST](http://cs.stanford.edu/people/karpathy/convnetjs/demo/trainers.html) ## Example Code Here's a minimum example of defining a **2-layer neural network** and training it on a single data point: ```javascript // species a 2-layer neural network with one hidden layer of 20 neurons var layer_defs = []; // input layer declares size of input. here: 2-D data // ConvNetJS works on 3-Dimensional volumes (sx, sy, depth), but if you're not dealing with images // then the first two dimensions (sx, sy) will always be kept at size 1 layer_defs.push({type:'input', out_sx:1, out_sy:1, out_depth:2}); // declare 20 neurons, followed by ReLU (rectified linear unit non-linearity) layer_defs.push({type:'fc', num_neurons:20, activation:'relu'}); // declare the linear classifier on top of the previous hidden layer layer_defs.push({type:'softmax', num_classes:10}); var net = new convnetjs.Net(); net.makeLayers(layer_defs); // forward a random data point through the network var x = new convnetjs.Vol([0.3, -0.5]); var prob = net.forward(x); // prob is a Vol. Vols have a field .w that stores the raw data, and .dw that stores gradients console.log('probability that x is class 0: ' + prob.w[0]); // prints 0.50101 var trainer = new convnetjs.SGDTrainer(net, {learning_rate:0.01, l2_decay:0.001}); trainer.train(x, 0); // train the network, specifying that x is class zero var prob2 = net.forward(x); console.log('probability that x is class 0: ' + prob2.w[0]); // now prints 0.50374, slightly higher than previous 0.50101: the networks // weights have been adjusted by the Trainer to give a higher probability to // the class we trained the network with (zero) ``` and here is a small **Convolutional Neural Network** if you wish to predict on images: ```javascript var layer_defs = []; layer_defs.push({type:'input', out_sx:32, out_sy:32, out_depth:3}); // declare size of input // output Vol is of size 32x32x3 here layer_defs.push({type:'conv', sx:5, filters:16, stride:1, pad:2, activation:'relu'}); // the layer will perform convolution with 16 kernels, each of size 5x5. // the input will be padded with 2 pixels on all sides to make the output Vol of the same size // output Vol will thus be 32x32x16 at this point layer_defs.push({type:'pool', sx:2, stride:2}); // output Vol is of size 16x16x16 here layer_defs.push({type:'conv', sx:5, filters:20, stride:1, pad:2, activation:'relu'}); // output Vol is of size 16x16x20 here layer_defs.push({type:'pool', sx:2, stride:2}); // output Vol is of size 8x8x20 here layer_defs.push({type:'conv', sx:5, filters:20, stride:1, pad:2, activation:'relu'}); // output Vol is of size 8x8x20 here layer_defs.push({type:'pool', sx:2, stride:2}); // output Vol is of size 4x4x20 here layer_defs.push({type:'softmax', num_classes:10}); // output Vol is of size 1x1x10 here net = new convnetjs.Net(); net.makeLayers(layer_defs); // helpful utility for converting images into Vols is included var x = convnetjs.img_to_vol(document.getElementById('some_image')) var output_probabilities_vol = net.forward(x) ``` ## Getting Started A [Getting Started](http://cs.stanford.edu/people/karpathy/convnetjs/started.html) tutorial is available on main page. The full [Documentation](http://cs.stanford.edu/people/karpathy/convnetjs/docs.html) can also be found there. See the **releases** page for this project to get the minified, compiled library, and a direct link to is also available below for convenience (but please host your own copy) - [convnet.js](http://cs.stanford.edu/people/karpathy/convnetjs/build/convnet.js) - [convnet-min.js](http://cs.stanford.edu/people/karpathy/convnetjs/build/convnet-min.js) ## Compiling the library from src/ to build/ If you would like to add features to the library, you will have to change the code in `src/` and then compile the library into the `build/` directory. The compilation script simply concatenates files in `src/` and then minifies the result. The compilation is done using an ant task: it compiles `build/convnet.js` by concatenating the source files in `src/` and then minifies the result into `build/convnet-min.js`. Make sure you have **ant** installed (on Ubuntu you can simply *sudo apt-get install* it), then cd into `compile/` directory and run: $ ant -lib yuicompressor-2.4.8.jar -f build.xml The output files will be in `build/` ## Use in Node The library is also available on *node.js*: 1. Install it: `$ npm install convnetjs` 2. Use it: `var convnetjs = require("convnetjs");` ## License MIT ================================================ FILE: bower.json ================================================ { "name": "convnetjs", "version": "0.0.0", "authors": [ "Andrej Karpathy " ], "description": "Deep Learning in Javascript. Train Convolutional Neural Networks (or ordinary ones) in your browser.", "main": "build/convnet.js", "moduleType": [ "amd", "es6", "globals", "node", "yui" ], "keywords": [ "machine", "learning", "AI", "convnet" ], "license": "MIT", "homepage": "http://cs.stanford.edu/people/karpathy/convnetjs/", "ignore": [ "**/.*", "node_modules", "bower_components", "test", "tests" ] } ================================================ FILE: build/deepqlearn.js ================================================ var deepqlearn = deepqlearn || { REVISION: 'ALPHA' }; (function(global) { "use strict"; // An agent is in state0 and does action0 // environment then assigns reward0 and provides new state, state1 // Experience nodes store all this information, which is used in the // Q-learning update step var Experience = function(state0, action0, reward0, state1) { this.state0 = state0; this.action0 = action0; this.reward0 = reward0; this.state1 = state1; } // A Brain object does all the magic. // over time it receives some inputs and some rewards // and its job is to set the outputs to maximize the expected reward var Brain = function(num_states, num_actions, opt) { var opt = opt || {}; // in number of time steps, of temporal memory // the ACTUAL input to the net will be (x,a) temporal_window times, and followed by current x // so to have no information from previous time step going into value function, set to 0. this.temporal_window = typeof opt.temporal_window !== 'undefined' ? opt.temporal_window : 1; // size of experience replay memory this.experience_size = typeof opt.experience_size !== 'undefined' ? opt.experience_size : 30000; // number of examples in experience replay memory before we begin learning this.start_learn_threshold = typeof opt.start_learn_threshold !== 'undefined'? opt.start_learn_threshold : Math.floor(Math.min(this.experience_size*0.1, 1000)); // gamma is a crucial parameter that controls how much plan-ahead the agent does. In [0,1] this.gamma = typeof opt.gamma !== 'undefined' ? opt.gamma : 0.8; // number of steps we will learn for this.learning_steps_total = typeof opt.learning_steps_total !== 'undefined' ? opt.learning_steps_total : 100000; // how many steps of the above to perform only random actions (in the beginning)? this.learning_steps_burnin = typeof opt.learning_steps_burnin !== 'undefined' ? opt.learning_steps_burnin : 3000; // what epsilon value do we bottom out on? 0.0 => purely deterministic policy at end this.epsilon_min = typeof opt.epsilon_min !== 'undefined' ? opt.epsilon_min : 0.05; // what epsilon to use at test time? (i.e. when learning is disabled) this.epsilon_test_time = typeof opt.epsilon_test_time !== 'undefined' ? opt.epsilon_test_time : 0.01; // advanced feature. Sometimes a random action should be biased towards some values // for example in flappy bird, we may want to choose to not flap more often if(typeof opt.random_action_distribution !== 'undefined') { // this better sum to 1 by the way, and be of length this.num_actions this.random_action_distribution = opt.random_action_distribution; if(this.random_action_distribution.length !== num_actions) { console.log('TROUBLE. random_action_distribution should be same length as num_actions.'); } var a = this.random_action_distribution; var s = 0.0; for(var k=0;k0.0001) { console.log('TROUBLE. random_action_distribution should sum to 1!'); } } else { this.random_action_distribution = []; } // states that go into neural net to predict optimal action look as // x0,a0,x1,a1,x2,a2,...xt // this variable controls the size of that temporal window. Actions are // encoded as 1-of-k hot vectors this.net_inputs = num_states * this.temporal_window + num_actions * this.temporal_window + num_states; this.num_states = num_states; this.num_actions = num_actions; this.window_size = Math.max(this.temporal_window, 2); // must be at least 2, but if we want more context even more this.state_window = new Array(this.window_size); this.action_window = new Array(this.window_size); this.reward_window = new Array(this.window_size); this.net_window = new Array(this.window_size); // create [state -> value of all possible actions] modeling net for the value function var layer_defs = []; if(typeof opt.layer_defs !== 'undefined') { // this is an advanced usage feature, because size of the input to the network, and number of // actions must check out. This is not very pretty Object Oriented programming but I can't see // a way out of it :( layer_defs = opt.layer_defs; if(layer_defs.length < 2) { console.log('TROUBLE! must have at least 2 layers'); } if(layer_defs[0].type !== 'input') { console.log('TROUBLE! first layer must be input layer!'); } if(layer_defs[layer_defs.length-1].type !== 'regression') { console.log('TROUBLE! last layer must be input regression!'); } if(layer_defs[0].out_depth * layer_defs[0].out_sx * layer_defs[0].out_sy !== this.net_inputs) { console.log('TROUBLE! Number of inputs must be num_states * temporal_window + num_actions * temporal_window + num_states!'); } if(layer_defs[layer_defs.length-1].num_neurons !== this.num_actions) { console.log('TROUBLE! Number of regression neurons should be num_actions!'); } } else { // create a very simple neural net by default layer_defs.push({type:'input', out_sx:1, out_sy:1, out_depth:this.net_inputs}); if(typeof opt.hidden_layer_sizes !== 'undefined') { // allow user to specify this via the option, for convenience var hl = opt.hidden_layer_sizes; for(var k=0;k maxval) { maxk = k; maxval = action_values.w[k]; } } return {action:maxk, value:maxval}; }, getNetInput: function(xt) { // return s = (x,a,x,a,x,a,xt) state vector. // It's a concatenation of last window_size (x,a) pairs and current state x var w = []; w = w.concat(xt); // start with current state // and now go backwards and append states and actions from history temporal_window times var n = this.window_size; for(var k=0;k this.temporal_window) { // we have enough to actually do something reasonable var net_input = this.getNetInput(input_array); if(this.learning) { // compute epsilon for the epsilon-greedy policy this.epsilon = Math.min(1.0, Math.max(this.epsilon_min, 1.0-(this.age - this.learning_steps_burnin)/(this.learning_steps_total - this.learning_steps_burnin))); } else { this.epsilon = this.epsilon_test_time; // use test-time value } var rf = convnetjs.randf(0,1); if(rf < this.epsilon) { // choose a random action with epsilon probability action = this.random_action(); } else { // otherwise use our policy to make decision var maxact = this.policy(net_input); action = maxact.action; } } else { // pathological case that happens first few iterations // before we accumulate window_size inputs var net_input = []; action = this.random_action(); } // remember the state and action we took for backward pass this.net_window.shift(); this.net_window.push(net_input); this.state_window.shift(); this.state_window.push(input_array); this.action_window.shift(); this.action_window.push(action); return action; }, backward: function(reward) { this.latest_reward = reward; this.average_reward_window.add(reward); this.reward_window.shift(); this.reward_window.push(reward); if(!this.learning) { return; } // various book-keeping this.age += 1; // it is time t+1 and we have to store (s_t, a_t, r_t, s_{t+1}) as new experience // (given that an appropriate number of state measurements already exist, of course) if(this.forward_passes > this.temporal_window + 1) { var e = new Experience(); var n = this.window_size; e.state0 = this.net_window[n-2]; e.action0 = this.action_window[n-2]; e.reward0 = this.reward_window[n-2]; e.state1 = this.net_window[n-1]; if(this.experience.length < this.experience_size) { this.experience.push(e); } else { // replace. finite memory! var ri = convnetjs.randi(0, this.experience_size); this.experience[ri] = e; } } // learn based on experience, once we have some samples to go on // this is where the magic happens... if(this.experience.length > this.start_learn_threshold) { var avcost = 0.0; for(var k=0;k < this.tdtrainer.batch_size;k++) { var re = convnetjs.randi(0, this.experience.length); var e = this.experience[re]; var x = new convnetjs.Vol(1, 1, this.net_inputs); x.w = e.state0; var maxact = this.policy(e.state1); var r = e.reward0 + this.gamma * maxact.value; var ystruct = {dim: e.action0, val: r}; var loss = this.tdtrainer.train(x, ystruct); avcost += loss.loss; } avcost = avcost/this.tdtrainer.batch_size; this.average_loss_window.add(avcost); } }, visSelf: function(elt) { elt.innerHTML = ''; // erase elt first // elt is a DOM element that this function fills with brain-related information var brainvis = document.createElement('div'); // basic information var desc = document.createElement('div'); var t = ''; t += 'experience replay size: ' + this.experience.length + '
'; t += 'exploration epsilon: ' + this.epsilon + '
'; t += 'age: ' + this.age + '
'; t += 'average Q-learning loss: ' + this.average_loss_window.get_average() + '
'; t += 'smooth-ish reward: ' + this.average_reward_window.get_average() + '
'; desc.innerHTML = t; brainvis.appendChild(desc); elt.appendChild(brainvis); } } global.Brain = Brain; })(deepqlearn); (function(lib) { "use strict"; if (typeof module === "undefined" || typeof module.exports === "undefined") { window.deepqlearn = lib; // in ordinary browser attach library to window } else { module.exports = lib; // in nodejs } })(deepqlearn); ================================================ FILE: build/util.js ================================================ // contains various utility functions var cnnutil = (function(exports){ // a window stores _size_ number of values // and returns averages. Useful for keeping running // track of validation or training accuracy during SGD var Window = function(size, minsize) { this.v = []; this.size = typeof(size)==='undefined' ? 100 : size; this.minsize = typeof(minsize)==='undefined' ? 20 : minsize; this.sum = 0; } Window.prototype = { add: function(x) { this.v.push(x); this.sum += x; if(this.v.length>this.size) { var xold = this.v.shift(); this.sum -= xold; } }, get_average: function() { if(this.v.length < this.minsize) return -1; else return this.sum/this.v.length; }, reset: function(x) { this.v = []; this.sum = 0; } } // returns min, max and indeces of an array var maxmin = function(w) { if(w.length === 0) { return {}; } // ... ;s var maxv = w[0]; var minv = w[0]; var maxi = 0; var mini = 0; for(var i=1;i maxv) { maxv = w[i]; maxi = i; } if(w[i] < minv) { minv = w[i]; mini = i; } } return {maxi: maxi, maxv: maxv, mini: mini, minv: minv, dv:maxv-minv}; } // returns string representation of float // but truncated to length of d digits var f2t = function(x, d) { if(typeof(d)==='undefined') { var d = 5; } var dd = 1.0 * Math.pow(10, d); return '' + Math.floor(x*dd)/dd; } exports = exports || {}; exports.Window = Window; exports.maxmin = maxmin; exports.f2t = f2t; return exports; })(typeof module != 'undefined' && module.exports); // add exports to module.exports if in node.js ================================================ FILE: build/vis.js ================================================ // contains various utility functions var cnnvis = (function(exports){ // can be used to graph loss, or accuract over time var Graph = function(options) { var options = options || {}; this.step_horizon = options.step_horizon || 1000; this.pts = []; this.maxy = -9999; this.miny = 9999; } Graph.prototype = { // canv is the canvas we wish to update with this new datapoint add: function(step, y) { var time = new Date().getTime(); // in ms if(y>this.maxy*0.99) this.maxy = y*1.05; if(y this.step_horizon) this.step_horizon *= 2; }, // elt is a canvas we wish to draw into drawSelf: function(canv) { var pad = 25; var H = canv.height; var W = canv.width; var ctx = canv.getContext('2d'); ctx.clearRect(0, 0, W, H); ctx.font="10px Georgia"; var f2t = function(x) { var dd = 1.0 * Math.pow(10, 2); return '' + Math.floor(x*dd)/dd; } // draw guidelines and values ctx.strokeStyle = "#999"; ctx.beginPath(); var ng = 10; for(var i=0;i<=ng;i++) { var xpos = i/ng*(W-2*pad)+pad; ctx.moveTo(xpos, pad); ctx.lineTo(xpos, H-pad); ctx.fillText(f2t(i/ng*this.step_horizon/1000)+'k',xpos,H-pad+14); } for(var i=0;i<=ng;i++) { var ypos = i/ng*(H-2*pad)+pad; ctx.moveTo(pad, ypos); ctx.lineTo(W-pad, ypos); ctx.fillText(f2t((ng-i)/ng*(this.maxy-this.miny) + this.miny), 0, ypos); } ctx.stroke(); var N = this.pts.length; if(N<2) return; // draw the actual curve var t = function(x, y, s) { var tx = x / s.step_horizon * (W-pad*2) + pad; var ty = H - ((y-s.miny) / (s.maxy-s.miny) * (H-pad*2) + pad); return {tx:tx, ty:ty} } ctx.strokeStyle = "red"; ctx.beginPath() for(var i=0;ithis.maxy*0.99) this.maxy = y*1.05; if(y this.step_horizon) this.step_horizon *= 2; }, // elt is a canvas we wish to draw into drawSelf: function(canv) { var pad = 25; var H = canv.height; var W = canv.width; var ctx = canv.getContext('2d'); ctx.clearRect(0, 0, W, H); ctx.font="10px Georgia"; var f2t = function(x) { var dd = 1.0 * Math.pow(10, 2); return '' + Math.floor(x*dd)/dd; } // draw guidelines and values ctx.strokeStyle = "#999"; ctx.beginPath(); var ng = 10; for(var i=0;i<=ng;i++) { var xpos = i/ng*(W-2*pad)+pad; ctx.moveTo(xpos, pad); ctx.lineTo(xpos, H-pad); ctx.fillText(f2t(i/ng*this.step_horizon/1000)+'k',xpos,H-pad+14); } for(var i=0;i<=ng;i++) { var ypos = i/ng*(H-2*pad)+pad; ctx.moveTo(pad, ypos); ctx.lineTo(W-pad, ypos); ctx.fillText(f2t((ng-i)/ng*(this.maxy-this.miny) + this.miny), 0, ypos); } ctx.stroke(); var N = this.pts.length; if(N<2) return; // draw legend for(var k=0;k ================================================ FILE: demo/autoencoder.html ================================================ ConvNetJS MNIST demo

ConvNetJS Denoising Autoencoder demo

Description

All the other demos are examples of Supervised Learning, so in this demo I wanted to show an example of Unsupervised Learning. We are going to train an autoencoder on MNIST digits.

An autoencoder is a regression task where the network is asked to predict its input (in other words, model the identity function). Sounds simple enough, except the network has a tight bottleneck of a few neurons in the middle (in the default example only two!), forcing it to create effective representations that compress the input into a low-dimensional code that can be used by the decoder to reproduce the original input.

Report questions/bugs/suggestions to @karpathy.

Training Stats

Current image:
Learning rate:


Loss:


Network Visualization

================================================ FILE: demo/automatic.html ================================================ ConvNetJS Automatic

ConvNetJS Automatic Prediction Demo

Introduction

This demo illustrates the usage of ConvNetJS' MagicNet class, which performs fully automatic prediction given your arbitrary data. Internally, the MagicNet tries out many different types of networks, performs cross-validations of network hyper-parameters across folds of your data, and creates a final classifier by model averaging the best architectures. The API for MagicNet looks as follows: 

var opts = {}; // options struct
opts.train_ratio = 0.7;
opts.num_folds = 10; // number of folds to eval per candidate
opts.num_candidates = 10; // number of candidates to eval in parallel
opts.num_epochs = 50; // epochs to make through data per fold
// below, train_data is a list of input Vols and train_labels is a 
// list of integer correct labels (in 0...K).
var magicNet = new convnetjs.MagicNet(train_data, train_labels, opts);
magicNet.onFinishBatch(finishedBatch); // example of setting callback for events

// start training magicNet. Every step() call all candidates train on one example
setInterval(function(){ magicNet.step() }, 0);

// once at least one batch of candidates is evaluated on all folds we can do prediction!
function finishedBatch() {
  // prediction example. xout is Vol of scores
  // there is also predict_soft(), which returns the full score volume for all labels
  var predicted_label = magicNet.predict(some_test_vol);
}

Your data

Currently made input data assumptions:

  • Provide data as CSV (comma-separated) values. Leave out any header rows.
  • Every row is a data point.
  • No missing values.
  • Last column is the class (only classification is currently supported).

The text area is pre-filled with a Car Quality Evaluation dataset to show you example input, but there are a few buttons that load some example datasets (more details on these: Iris data, Car Eval data, Yeast Data). A nice place to find more datasets are UCI Repository or mldata.org.

(and send % of imported data randomly into Test Set below)

Cross-Validation

Index of column to classify as target. (e.g. 0 = first column, -1 = last column)
Percent of data to use for training (rest will be validation)
Number of data folds to evaluate per candidate
Number of candidates in a batch, to evaluate in parallel
Number of epochs to make over each fold
Number of Neurons in each layer: Min Max

Below: graph of the validation accuracy for current batch of candidate models as a function of the number of training points they have seen during training. Good networks will rise up as high as possible and stay there. The best performer is printed in detail below the graph. The graph is less wiggly if there is more data.

Evaluate on Test Set

Paste a test set in box below to evaluate the final test accuracy, which is based on a model-averaged ensemble of the best discovered network from the training data above. The CSV pasted below should be in the same format as the one used for training data above. The text field is pre-filled with the training data.

Number of best models to average in the ensemble network

Export Best Network



Above you can export a trained MagicNet in JSON format. The exported MagicNet is simply a thin wrapper around a list of the best networks that were discovered during cross-validation and it can be loaded and used again as follows: 
var magicNet = new convnetjs.MagicNet();
magicNet.fromJSON(json);
magicNet.predict(some_vol); // ready to use!








================================================ FILE: demo/cifar10.html ================================================ ConvNetJS CIFAR-10 demo

ConvNetJS CIFAR-10 demo

Description

This demo trains a Convolutional Neural Network on the CIFAR-10 dataset in your browser, with nothing but Javascript. The state of the art on this dataset is about 90% accuracy and human performance is at about 94% (not perfect as the dataset can be a bit ambiguous). I used this python script to parse the original files (python version) into batches of images that can be easily loaded into page DOM with img tags.

This dataset is more difficult and it takes longer to train a network. Data augmentation includes random flipping and random image shifts by up to 2px horizontally and verically.

By default, in this demo we're using Adadelta which is one of per-parameter adaptive step size methods, so we don't have to worry about changing learning rates or momentum over time. However, I still included the text fields for changing these if you'd like to play around with SGD+Momentum trainer.

Report questions/bugs/suggestions to @karpathy.

Training Stats

Learning rate:
Momentum:
Batch size:
Weight decay:




Loss:

Test an image from your computer:

Instantiate a Network and Trainer


Network Visualization

Example predictions on Test set

================================================ FILE: demo/classify2d.html ================================================ ConvNetJS demo: Classify toy 2D data

ConvnetJS demo: toy 2d classification with 2-layer neural network

The simulation below shows a toy binary problem with a few data points of class 0 (red) and 1 (green). The network is set up as:


Feel free to change this, the text area above gets eval()'d when you hit the button and the network gets reloaded. Every 10th of a second, all points are fed to the network multiple times through the trainer class to train the network. The resulting predictions of the network are then "painted" under the data points to show you the generalization.

On the right we visualize the transformed representation of all grid points in the original space and the data, for a given layer and only for 2 neurons at a time. The number in the bracket shows the total number of neurons at that level of representation. If the number is more than 2, you will only see the two visualized but you can cycle through all of them with the cycle button.

Browser not supported for Canvas. Get a real browser.

Controls:
CLICK: Add red data point
SHIFT+CLICK: Add green data point
CTRL+CLICK: Remove closest data point

Browser not supported for Canvas. Get a real browser.

Go back to ConvNetJS

================================================ FILE: demo/css/automatic.css ================================================ #wrap { width:800px; margin-left: auto; margin-right: auto; } h2 { text-align: center; font-size: 34px; font-weight: 300; margin-bottom: 50px; } h1 { font-size: 26px; font-weight: 400; border-bottom: 1px #999 solid; } #datamsg{ background-color: white; margin-top: 2px; padding: 10px; } #prepromsg{ margin-top: 10px; padding: 10px; } .msg{ padding: 2px; } body { font-family: 'Lato', sans-serif; color: #333; font-size: 20px; font-weight: 300; } input[type=text] { border: 1px solid #999; padding: 3px; font-size: 18px; color: #333; } .clouds-flat-button { position: relative; vertical-align: top; width: 100%; height: 80px; padding: 0; color:#454545; text-align: center; font-size: 16px; background: #ecf0f1; border: 0; border-bottom: 2px solid #dadedf; cursor: pointer; -webkit-box-shadow: inset 0 -2px #dadedf; box-shadow: inset 0 -2px #dadedf; } .clouds-flat-button:active { top: 1px; outline: none; -webkit-box-shadow: none; box-shadow: none; } #bestmodel { font-size: 16px; background-color: #FAFAFA; padding: 10px; } #bestmodeloverall { font-size: 16px; background-color: #FAFAFA; padding: 10px; margin-top: 10px; } .syntaxhighlighter { font-size: 14px !important; overflow-y: hidden !important; } .sopts { box-shadow: 0px 0px 2px 0px #555; margin-bottom: 5px; padding: 10px; } ================================================ FILE: demo/css/style.css ================================================ .layer { border: 1px solid #999; margin-bottom: 5px; text-align: left; padding: 10px; } .layer_act { width: 500px; float: right; } .ltconv { background-color: #FDD; } .ltrelu { background-color: #FDF; } .ltpool { background-color: #DDF; } .ltsoftmax { background-color: #FFD; } .ltfc { background-color: #DFF; } .ltlrn { background-color: #DFD; } .ltdropout { background-color: #AAA; } .ltitle { color: #333; font-size: 18px; } .actmap { margin: 1px; } #trainstats { text-align: left; } .clear { clear: both; } #wrap { width: 800px; margin-left: auto; margin-right: auto; } h1 { font-size: 16px; color: #333; background-color: #DDD; border-bottom: 1px #999 solid; text-align: center; } h2 { text-align: center; } .secpart { width: 400px; float: left; } #lossgraph { /*border: 1px solid #F0F;*/ width: 100%; } .testdiv canvas { float: left; width: 64px; } .testdiv { background-color: rgba(255,255,255,0.65); height: auto; width: auto; display: inline-block; font-size: 12px; box-shadow: 0px 0px 2px 2px #EEE; margin: 5px; padding: 5px; color: black; } .probsdiv { float: left; width: 100px; margin-left: 1px; } .pp { margin: 1px; padding: 1px; } #testset_acc { /* margin-bottom: 200px; */ } #testset_vis { margin-bottom: 200px; } body { font-family: Arial, "Helvetica Neue", Helvetica, sans-serif; /* color: #333; */ /* padding: 20px; */ } ================================================ FILE: demo/image_regression.html ================================================ ConvNetJS demo: Image Painting

ConvnetJS demo: Image "Painting"

This demo that treats the pixels of an image as a learning problem: it takes the (x,y) position on a grid and learns to predict the color at that point using regression to (r,g,b). It's a bit like compression, since the image information is encoded in the weights of the network, but almost certainly not of practical kind :)

Note that the entire ConvNetJS definition is shown in textbox below and it gets eval()'d to create the network, so feel free to fiddle with the parameters and hit "reload". I found that, empirically and interestingly, deeper networks tend to work much better on this task given a fixed parameter budget.

Report questions/bugs/suggestions to @karpathy.



Choose your own image:

Original Image
Neural Network output


Learning rate:
The learning rate should probably be decreased over time (slide left) to let the network better overfit the training data. It's nice to not have to worry about overfitting.


You can upload your own image above (click Choose File), or you can click on any of the images below to load them.


Go back to ConvNetJS

================================================ FILE: demo/js/autoencoder.js ================================================ // globals var layer_defs, net, trainer; var t = "\ layer_defs = [];\n\ layer_defs.push({type:'input', out_sx:28, out_sy:28, out_depth:1});\n\ layer_defs.push({type:'fc', num_neurons:50, activation:'tanh'});\n\ layer_defs.push({type:'fc', num_neurons:50, activation:'tanh'});\n\ layer_defs.push({type:'fc', num_neurons:2});\n\ layer_defs.push({type:'fc', num_neurons:50, activation:'tanh'});\n\ layer_defs.push({type:'fc', num_neurons:50, activation:'tanh'});\n\ layer_defs.push({type:'regression', num_neurons:28*28});\n\ \n\ net = new convnetjs.Net();\n\ net.makeLayers(layer_defs);\n\ \n\ trainer = new convnetjs.SGDTrainer(net, {learning_rate:1, method:'adadelta', batch_size:50, l2_decay:0.001, l1_decay:0.001});\n\ "; // ------------------------ // BEGIN MNIST SPECIFIC STUFF // ------------------------ var sample_training_instance = function() { // find an unloaded batch var bi = Math.floor(Math.random()*loaded_train_batches.length); var b = loaded_train_batches[bi]; var k = Math.floor(Math.random()*3000); // sample within the batch var n = b*3000+k; // load more batches over time if(step_num%5000===0 && step_num>0) { for(var i=0;i3) { // actual weights filters_div.appendChild(document.createTextNode('Weights:')); filters_div.appendChild(document.createElement('br')); for(var j=0;j= selected_layer.out_depth) d1 = 0; // and wrap if(d0 >= selected_layer.out_depth) d0 = 0; // and wrap $("#cyclestatus").html('drawing neurons ' + d0 + ' and ' + d1 + ' of layer #' + lix + ' (' + net.layers[lix].layer_type + ')'); } function updateLix(newlix) { $("#button"+lix).css('background-color', ''); // erase highlight lix = newlix; d0 = 0; d1 = 1; // reset these $("#button"+lix).css('background-color', '#FFA'); $("#cyclestatus").html('drawing neurons ' + d0 + ' and ' + d1 + ' of layer with index ' + lix + ' (' + net.layers[lix].layer_type + ')'); } var lossGraph = new cnnvis.Graph(); var xLossWindow = new cnnutil.Window(100); var w2LossWindow = new cnnutil.Window(100); var w1LossWindow = new cnnutil.Window(100); var step_num = 0; var colors = ["red", "blue", "green", "orange", "magenta", "cyan", "purple", "silver", "olive", "lime", "yellow"]; var step = function(sample) { // train on it with network var stats = trainer.train(sample.x, sample.x.w); // keep track of stats such as the average training error and loss xLossWindow.add(stats.cost_loss); w1LossWindow.add(stats.l1_decay_loss); w2LossWindow.add(stats.l2_decay_loss); // visualize training status var train_elt = document.getElementById("trainstats"); train_elt.innerHTML = ''; var t = 'Forward time per example: ' + stats.fwd_time + 'ms'; train_elt.appendChild(document.createTextNode(t)); train_elt.appendChild(document.createElement('br')); var t = 'Backprop time per example: ' + stats.bwd_time + 'ms'; train_elt.appendChild(document.createTextNode(t)); train_elt.appendChild(document.createElement('br')); var t = 'Regression loss: ' + f2t(xLossWindow.get_average()); train_elt.appendChild(document.createTextNode(t)); train_elt.appendChild(document.createElement('br')); var t = 'L2 Weight decay loss: ' + f2t(w2LossWindow.get_average()); train_elt.appendChild(document.createTextNode(t)); train_elt.appendChild(document.createElement('br')); var t = 'L1 Weight decay loss: ' + f2t(w1LossWindow.get_average()); train_elt.appendChild(document.createTextNode(t)); train_elt.appendChild(document.createElement('br')); var t = 'Examples seen: ' + step_num; train_elt.appendChild(document.createTextNode(t)); train_elt.appendChild(document.createElement('br')); // visualize activations if(step_num % 100 === 0) { var vis_elt = document.getElementById("visnet"); visualize_activations(net, vis_elt); } // visualize embedding if(step_num % 100 === 0) { var embcanvas = document.getElementById('embedding'); var ctx = embcanvas.getContext("2d"); var EW = embcanvas.width; var EH = embcanvas.height; // propagate a few training examples through the network and grab codes var xcodes = []; var ycodes = []; var ns = embed_samples.length; // number of samples for(var k=0;k= 0 && xw1 >= 0 && xw2 >= 0) { // if they are -1 it means not enough data was accumulated yet for estimates lossGraph.add(step_num, xa + xw1 + xw2); lossGraph.drawSelf(document.getElementById("lossgraph")); } } step_num++; } // user settings var change_lr = function() { trainer.learning_rate = parseFloat(document.getElementById("lr_input").value); update_net_param_display(); } var update_net_param_display = function() { document.getElementById('lr_input').value = trainer.learning_rate; } var toggle_pause = function() { paused = !paused; var btn = document.getElementById('buttontp'); if(paused) { btn.value = 'resume' } else { btn.value = 'pause'; } } var dump_json = function() { document.getElementById("dumpjson").value = JSON.stringify(net.toJSON()); } var clear_graph = function() { lossGraph = new cnnvis.Graph(); // reinit graph too } var reset_all = function() { update_net_param_display(); // reinit windows that keep track of val/train accuracies lossGraph = new cnnvis.Graph(); // reinit graph too step_num = 0; // enter buttons for layers var t = ''; for(var i=1;i"; } $("#layer_ixes").html(t); $("#button"+lix).css('background-color', '#FFA'); $("#cyclestatus").html('drawing neurons ' + d0 + ' and ' + d1 + ' of layer with index ' + lix + ' (' + net.layers[lix].layer_type + ')'); } var load_from_json = function() { var jsonString = document.getElementById("dumpjson").value; var json = JSON.parse(jsonString); net = new convnetjs.Net(); net.fromJSON(json); reset_all(); } var change_net = function() { eval($("#newnet").val()); reset_all(); } ================================================ FILE: demo/js/automatic.js ================================================ // utility functions Array.prototype.contains = function(v) { for(var i = 0; i < this.length; i++) { if(this[i] === v) return true; } return false; }; Array.prototype.unique = function() { var arr = []; for(var i = 0; i < this.length; i++) { if(!arr.contains(this[i])) { arr.push(this[i]); } } return arr; } function FAIL(outdivid, msg) { $(outdivid).prepend("
"+msg+"
") } function SUCC(outdivid, msg) { $(outdivid).prepend("
"+msg+"
") } // looks at a column i of data and guesses what's in it // returns results of analysis: is column numeric? How many unique entries and what are they? function guessColumn(data, c) { var numeric = true; var vs = []; for(var i=0,n=data.length;i 20 && i>3 && i < D-3) { if(i==4) { SUCC(outdivid, "..."); // suppress output for too many columns } } else { SUCC(outdivid, "column " + i + " looks " + (res.numeric ? "numeric" : "NOT numeric") + " and has " + res.num + " unique elements"); } } return {arr: arr, colstats: colstats}; } // process input mess into vols and labels function makeDataset(arr, colstats) { var labelix = parseInt($("#labelix").val()); if(labelix < 0) labelix = D + labelix; // -1 should turn to D-1 var data = []; var labels = []; for(var i=0;i 0) { t += 'Results based on ' + c.acc.length + ' folds:'; t += 'best model in current batch (validation accuracy ' + mm.maxv + '):
'; t += 'Net layer definitions:
'; t += JSON.stringify(cm.layer_defs); t += '
Trainer definition:
'; t += JSON.stringify(cm.trainer_def); t += '
'; } $('#bestmodel').html(t); // also print out the best model so far var t = ''; if(magicNet.evaluated_candidates.length > 0) { var cm = magicNet.evaluated_candidates[0]; t += 'validation accuracy of best model so far, overall: ' + cm.accv / cm.acc.length + '
'; t += 'Net layer definitions:
'; t += JSON.stringify(cm.layer_defs); t += '
Trainer definition:
'; t += JSON.stringify(cm.trainer_def); t += '
'; } $('#bestmodeloverall').html(t); } } // TODO: MOVE TO CONVNETJS UTILS var randperm = function(n) { var i = n, j = 0, temp; var array = []; for(var q=0;q"; } $("#layer_ixes").html(t); $("#button"+lix).css('background-color', '#FFA'); $("#cyclestatus").html('drawing neurons ' + d0 + ' and ' + d1 + ' of layer with index ' + lix + ' (' + net.layers[lix].layer_type + ')'); } function updateLix(newlix) { $("#button"+lix).css('background-color', ''); // erase highlight lix = newlix; d0 = 0; d1 = 1; // reset these $("#button"+lix).css('background-color', '#FFA'); $("#cyclestatus").html('drawing neurons ' + d0 + ' and ' + d1 + ' of layer with index ' + lix + ' (' + net.layers[lix].layer_type + ')'); } function myinit() { } function random_data(){ data = []; labels = []; for(var k=0;k<40;k++) { data.push([convnetjs.randf(-3,3), convnetjs.randf(-3,3)]); labels.push(convnetjs.randf(0,1) > 0.5 ? 1 : 0); } N = labels.length; } function original_data(){ data = []; labels = []; data.push([-0.4326 , 1.1909 ]); labels.push(1); data.push([3.0, 4.0]); labels.push(1); data.push([0.1253 , -0.0376 ]); labels.push(1); data.push([0.2877 , 0.3273 ]); labels.push(1); data.push([-1.1465 , 0.1746 ]); labels.push(1); data.push([1.8133 , 1.0139 ]); labels.push(0); data.push([2.7258 , 1.0668 ]); labels.push(0); data.push([1.4117 , 0.5593 ]); labels.push(0); data.push([4.1832 , 0.3044 ]); labels.push(0); data.push([1.8636 , 0.1677 ]); labels.push(0); data.push([0.5 , 3.2 ]); labels.push(1); data.push([0.8 , 3.2 ]); labels.push(1); data.push([1.0 , -2.2 ]); labels.push(1); N = labels.length; } function circle_data() { data = []; labels = []; for(var i=0;i<50;i++) { var r = convnetjs.randf(0.0, 2.0); var t = convnetjs.randf(0.0, 2*Math.PI); data.push([r*Math.sin(t), r*Math.cos(t)]); labels.push(1); } for(var i=0;i<50;i++) { var r = convnetjs.randf(3.0, 5.0); //var t = convnetjs.randf(0.0, 2*Math.PI); var t = 2*Math.PI*i/50.0 data.push([r*Math.sin(t), r*Math.cos(t)]); labels.push(0); } N = data.length; } function spiral_data() { data = []; labels = []; var n = 100; for(var i=0;i= selected_layer.out_depth) d1 = 0; // and wrap if(d0 >= selected_layer.out_depth) d0 = 0; // and wrap $("#cyclestatus").html('drawing neurons ' + d0 + ' and ' + d1 + ' of layer #' + lix + ' (' + net.layers[lix].layer_type + ')'); } var lix = 4; // layer id to track first 2 neurons of var d0 = 0; // first dimension to show visualized var d1 = 1; // second dimension to show visualized function draw(){ ctx.clearRect(0,0,WIDTH,HEIGHT); var netx = new convnetjs.Vol(1,1,2); // draw decisions in the grid var density= 5.0; var gridstep = 2; var gridx = []; var gridy = []; var gridl = []; for(var x=0.0, cx=0; x<=WIDTH; x+= density, cx++) { for(var y=0.0, cy=0; y<=HEIGHT; y+= density, cy++) { //var dec= svm.marginOne([(x-WIDTH/2)/ss, (y-HEIGHT/2)/ss]); netx.w[0] = (x-WIDTH/2)/ss; netx.w[1] = (y-HEIGHT/2)/ss; var a = net.forward(netx, false); if(a.w[0] > a.w[1]) ctx.fillStyle = 'rgb(250, 150, 150)'; else ctx.fillStyle = 'rgb(150, 250, 150)'; //ctx.fillStyle = 'rgb(150,' + Math.floor(a.w[0]*105)+150 + ',150)'; //ctx.fillStyle = 'rgb(' + Math.floor(a.w[0]*255) + ',' + Math.floor(a.w[1]*255) + ', 0)'; ctx.fillRect(x-density/2-1, y-density/2-1, density+2, density+2); if(cx%gridstep === 0 && cy%gridstep===0) { // record the transformation information var xt = net.layers[lix].out_act.w[d0]; // in screen coords var yt = net.layers[lix].out_act.w[d1]; // in screen coords gridx.push(xt); gridy.push(yt); gridl.push(a.w[0] > a.w[1]); // remember final label as well } } } // draw axes ctx.beginPath(); ctx.strokeStyle = 'rgb(50,50,50)'; ctx.lineWidth = 1; ctx.moveTo(0, HEIGHT/2); ctx.lineTo(WIDTH, HEIGHT/2); ctx.moveTo(WIDTH/2, 0); ctx.lineTo(WIDTH/2, HEIGHT); ctx.stroke(); // draw representation transformation axes for two neurons at some layer var mmx = cnnutil.maxmin(gridx); var mmy = cnnutil.maxmin(gridy); visctx.clearRect(0,0,visWIDTH,visHEIGHT); visctx.strokeStyle = 'rgb(0, 0, 0)'; var n = Math.floor(Math.sqrt(gridx.length)); // size of grid. Should be fine? var ng = gridx.length; var c = 0; // counter visctx.beginPath() for(var x=0;x= 0 && ix2 >= 0 && ix1 < ng && ix2 < ng && y= 0 && ix2 >= 0 && ix1 < ng && ix2 < ng && x =0) { console.log('splicing ' + mink); data.splice(mink, 1); labels.splice(mink, 1); N -= 1; } } else { // add datapoint at location of click data.push([xt, yt]); labels.push(shiftPressed ? 1 : 0); N += 1; } } function keyDown(key){ } function keyUp(key) { } $(function() { // note, globals viscanvas = document.getElementById('viscanvas'); visctx = viscanvas.getContext('2d'); visWIDTH = viscanvas.width; visHEIGHT = viscanvas.height; circle_data(); $("#layerdef").val(t); reload(); NPGinit(20); }); ================================================ FILE: demo/js/image-helpers.js ================================================ var loadFile = function(event) { var reader = new FileReader(); reader.onload = function(){ var preview = document.getElementById('preview_img'); preview.src = centerCrop(reader.result); preview.src = resize(preview.src); }; reader.readAsDataURL(event.target.files[0]); }; function centerCrop(src){ var image = new Image(); image.src = src; var max_width = Math.min(image.width, image.height); var max_height = Math.min(image.width, image.height); var canvas = document.createElement('canvas'); var ctx = canvas.getContext("2d"); ctx.clearRect(0, 0, canvas.width, canvas.height); canvas.width = max_width; canvas.height = max_height; ctx.drawImage(image, (max_width - image.width)/2, (max_height - image.height)/2, image.width, image.height); return canvas.toDataURL("image/png"); } function resize(src){ var image = new Image(); image.src = src; var canvas = document.createElement('canvas'); canvas.width = image.width; canvas.height = image.height; var ctx = canvas.getContext("2d"); ctx.clearRect(0, 0, canvas.width, canvas.height); ctx.drawImage(image, 0, 0, image.width, image.height); var dst = document.createElement('canvas'); dst.width = image_dimension; dst.height = image_dimension; window.pica.WW = false; window.pica.resizeCanvas(canvas, dst, { quality: 2, unsharpAmount: 500, unsharpThreshold: 100, transferable: false }, function (err) { }); window.pica.WW = true; return dst.toDataURL("image/png"); } ================================================ FILE: demo/js/image_regression.js ================================================ var data, labels; var layer_defs, net, trainer; // create neural net var t = "layer_defs = [];\n\ layer_defs.push({type:'input', out_sx:1, out_sy:1, out_depth:2}); // 2 inputs: x, y \n\ layer_defs.push({type:'fc', num_neurons:20, activation:'relu'});\n\ layer_defs.push({type:'fc', num_neurons:20, activation:'relu'});\n\ layer_defs.push({type:'fc', num_neurons:20, activation:'relu'});\n\ layer_defs.push({type:'fc', num_neurons:20, activation:'relu'});\n\ layer_defs.push({type:'fc', num_neurons:20, activation:'relu'});\n\ layer_defs.push({type:'fc', num_neurons:20, activation:'relu'});\n\ layer_defs.push({type:'fc', num_neurons:20, activation:'relu'});\n\ layer_defs.push({type:'regression', num_neurons:3}); // 3 outputs: r,g,b \n\ \n\ net = new convnetjs.Net();\n\ net.makeLayers(layer_defs);\n\ \n\ trainer = new convnetjs.SGDTrainer(net, {learning_rate:0.01, momentum:0.9, batch_size:5, l2_decay:0.0});\n\ "; var batches_per_iteration = 100; var mod_skip_draw = 100; var smooth_loss = -1; function update(){ // forward prop the data var W = nn_canvas.width; var H = nn_canvas.height; var p = oridata.data; var v = new convnetjs.Vol(1,1,2); var loss = 0; var lossi = 0; var N = batches_per_iteration; for(var iters=0;iters0) { for(var i=0;i 0 ? dwq : 0.0; } draw_activations_COLOR(activations_div, dd, scale); for(var q=0;q3) { // actual weights filters_div.appendChild(document.createTextNode('Weights:')); filters_div.appendChild(document.createElement('br')); for(var j=0;j' + classes_txt[preds[k].k] + '' } probsdiv.innerHTML = t; probsdiv.className = 'probsdiv'; div.appendChild(probsdiv); // add it into DOM $(div).prependTo($("#testset_vis")).hide().fadeIn('slow').slideDown('slow'); if($(".probsdiv").length>200) { $("#testset_vis > .probsdiv").last().remove(); // pop to keep upper bound of shown items } } testAccWindow.add(num_correct/num_total); $("#testset_acc").text('test accuracy based on last 200 test images: ' + testAccWindow.get_average()); } var testImage = function(img) { var x = convnetjs.img_to_vol(img); var out_p = net.forward(x); var vis_elt = document.getElementById("visnet"); visualize_activations(net, vis_elt); var preds =[] for(var k=0;k' + classes_txt[preds[k].k] + '' } probsdiv.innerHTML = t; probsdiv.className = 'probsdiv'; div.appendChild(probsdiv); // add it into DOM $(div).prependTo($("#testset_vis")).hide().fadeIn('slow').slideDown('slow'); if($(".probsdiv").length>200) { $("#testset_vis > .probsdiv").last().remove(); // pop to keep upper bound of shown items } } var lossGraph = new cnnvis.Graph(); var xLossWindow = new cnnutil.Window(100); var wLossWindow = new cnnutil.Window(100); var trainAccWindow = new cnnutil.Window(100); var valAccWindow = new cnnutil.Window(100); var testAccWindow = new cnnutil.Window(50, 1); var step_num = 0; var step = function(sample) { var x = sample.x; var y = sample.label; if(sample.isval) { // use x to build our estimate of validation error net.forward(x); var yhat = net.getPrediction(); var val_acc = yhat === y ? 1.0 : 0.0; valAccWindow.add(val_acc); return; // get out } // train on it with network var stats = trainer.train(x, y); var lossx = stats.cost_loss; var lossw = stats.l2_decay_loss; // keep track of stats such as the average training error and loss var yhat = net.getPrediction(); var train_acc = yhat === y ? 1.0 : 0.0; xLossWindow.add(lossx); wLossWindow.add(lossw); trainAccWindow.add(train_acc); // visualize training status var train_elt = document.getElementById("trainstats"); train_elt.innerHTML = ''; var t = 'Forward time per example: ' + stats.fwd_time + 'ms'; train_elt.appendChild(document.createTextNode(t)); train_elt.appendChild(document.createElement('br')); var t = 'Backprop time per example: ' + stats.bwd_time + 'ms'; train_elt.appendChild(document.createTextNode(t)); train_elt.appendChild(document.createElement('br')); var t = 'Classification loss: ' + f2t(xLossWindow.get_average()); train_elt.appendChild(document.createTextNode(t)); train_elt.appendChild(document.createElement('br')); var t = 'L2 Weight decay loss: ' + f2t(wLossWindow.get_average()); train_elt.appendChild(document.createTextNode(t)); train_elt.appendChild(document.createElement('br')); var t = 'Training accuracy: ' + f2t(trainAccWindow.get_average()); train_elt.appendChild(document.createTextNode(t)); train_elt.appendChild(document.createElement('br')); var t = 'Validation accuracy: ' + f2t(valAccWindow.get_average()); train_elt.appendChild(document.createTextNode(t)); train_elt.appendChild(document.createElement('br')); var t = 'Examples seen: ' + step_num; train_elt.appendChild(document.createTextNode(t)); train_elt.appendChild(document.createElement('br')); // visualize activations if(step_num % 100 === 0) { var vis_elt = document.getElementById("visnet"); visualize_activations(net, vis_elt); } // log progress to graph, (full loss) if(step_num % 200 === 0) { var xa = xLossWindow.get_average(); var xw = wLossWindow.get_average(); if(xa >= 0 && xw >= 0) { // if they are -1 it means not enough data was accumulated yet for estimates lossGraph.add(step_num, xa + xw); lossGraph.drawSelf(document.getElementById("lossgraph")); } } // run prediction on test set if((step_num % 100 === 0 && step_num > 0) || step_num===100) { test_predict(); } step_num++; } // user settings var change_lr = function() { trainer.learning_rate = parseFloat(document.getElementById("lr_input").value); update_net_param_display(); } var change_momentum = function() { trainer.momentum = parseFloat(document.getElementById("momentum_input").value); update_net_param_display(); } var change_batch_size = function() { trainer.batch_size = parseFloat(document.getElementById("batch_size_input").value); update_net_param_display(); } var change_decay = function() { trainer.l2_decay = parseFloat(document.getElementById("decay_input").value); update_net_param_display(); } var update_net_param_display = function() { document.getElementById('lr_input').value = trainer.learning_rate; document.getElementById('momentum_input').value = trainer.momentum; document.getElementById('batch_size_input').value = trainer.batch_size; document.getElementById('decay_input').value = trainer.l2_decay; } var toggle_pause = function() { paused = !paused; var btn = document.getElementById('buttontp'); if(paused) { btn.value = 'resume' } else { btn.value = 'pause'; } } var dump_json = function() { document.getElementById("dumpjson").value = JSON.stringify(this.net.toJSON()); } var clear_graph = function() { lossGraph = new cnnvis.Graph(); // reinit graph too } var reset_all = function() { // reinit trainer trainer = new convnetjs.SGDTrainer(net, {learning_rate:trainer.learning_rate, momentum:trainer.momentum, batch_size:trainer.batch_size, l2_decay:trainer.l2_decay}); update_net_param_display(); // reinit windows that keep track of val/train accuracies xLossWindow.reset(); wLossWindow.reset(); trainAccWindow.reset(); valAccWindow.reset(); testAccWindow.reset(); lossGraph = new cnnvis.Graph(); // reinit graph too step_num = 0; } var load_from_json = function() { var jsonString = document.getElementById("dumpjson").value; var json = JSON.parse(jsonString); net = new convnetjs.Net(); net.fromJSON(json); reset_all(); } var load_pretrained = function() { $.getJSON(dataset_name + "_snapshot.json", function(json){ net = new convnetjs.Net(); net.fromJSON(json); trainer.learning_rate = 0.0001; trainer.momentum = 0.9; trainer.batch_size = 2; trainer.l2_decay = 0.00001; reset_all(); }); } var change_net = function() { eval($("#newnet").val()); reset_all(); } ================================================ FILE: demo/js/npgmain.js ================================================ //Simple game engine //Author: Andrej Karpathy //License: BSD //This function does all the boring canvas stuff. To use it, just create functions: //update() gets called every frame //draw() gets called every frame //myinit() gets called once in beginning //mouseClick(x, y) gets called on mouse click //keyUp(keycode) gets called when key is released //keyDown(keycode) gets called when key is pushed var canvas; var ctx; var WIDTH; var HEIGHT; var FPS; function drawBubble(x, y, w, h, radius) { var r = x + w; var b = y + h; ctx.beginPath(); ctx.strokeStyle="black"; ctx.lineWidth="2"; ctx.moveTo(x+radius, y); ctx.lineTo(x+radius/2, y-10); ctx.lineTo(x+radius * 2, y); ctx.lineTo(r-radius, y); ctx.quadraticCurveTo(r, y, r, y+radius); ctx.lineTo(r, y+h-radius); ctx.quadraticCurveTo(r, b, r-radius, b); ctx.lineTo(x+radius, b); ctx.quadraticCurveTo(x, b, x, b-radius); ctx.lineTo(x, y+radius); ctx.quadraticCurveTo(x, y, x+radius, y); ctx.stroke(); } function drawRect(x, y, w, h){ ctx.beginPath(); ctx.rect(x,y,w,h); ctx.closePath(); ctx.fill(); ctx.stroke(); } function drawCircle(x, y, r){ ctx.beginPath(); ctx.arc(x, y, r, 0, Math.PI*2, true); ctx.closePath(); ctx.stroke(); ctx.fill(); } //uniform distribution integer function randi(s, e) { return Math.floor(Math.random()*(e-s) + s); } //uniform distribution function randf(s, e) { return Math.random()*(e-s) + s; } //normal distribution random number function randn(mean, variance) { var V1, V2, S; do { var U1 = Math.random(); var U2 = Math.random(); V1 = 2 * U1 - 1; V2 = 2 * U2 - 1; S = V1 * V1 + V2 * V2; } while (S > 1); X = Math.sqrt(-2 * Math.log(S) / S) * V1; X = mean + Math.sqrt(variance) * X; return X; } function eventClick(e) { //get position of cursor relative to top left of canvas var x; var y; if (e.pageX || e.pageY) { x = e.pageX; y = e.pageY; } else { x = e.clientX + document.body.scrollLeft + document.documentElement.scrollLeft; y = e.clientY + document.body.scrollTop + document.documentElement.scrollTop; } x -= canvas.offsetLeft; y -= canvas.offsetTop; //call user-defined callback mouseClick(x, y, e.shiftKey, e.ctrlKey); } //event codes can be found here: //http://www.aspdotnetfaq.com/Faq/What-is-the-list-of-KeyCodes-for-JavaScript-KeyDown-KeyPress-and-KeyUp-events.aspx function eventKeyUp(e) { var keycode = ('which' in e) ? e.which : e.keyCode; keyUp(keycode); } function eventKeyDown(e) { var keycode = ('which' in e) ? e.which : e.keyCode; keyDown(keycode); } function NPGinit(FPS){ //takes frames per secont to run at canvas = document.getElementById('NPGcanvas'); ctx = canvas.getContext('2d'); WIDTH = canvas.width; HEIGHT = canvas.height; canvas.addEventListener('click', eventClick, false); //canvas element cannot get focus by default. Requires to either set //tabindex to 1 so that it's focusable, or we need to attach listeners //to the document. Here we do the latter document.addEventListener('keyup', eventKeyUp, true); document.addEventListener('keydown', eventKeyDown, true); setInterval(NPGtick, 1000/FPS); myinit(); } function NPGtick() { update(); draw(); } ================================================ FILE: demo/js/pica.js ================================================ /* pica 1.0.7 nodeca/pica */!function(e){if("object"==typeof exports&&"undefined"!=typeof module)module.exports=e();else if("function"==typeof define&&define.amd)define([],e);else{var f;"undefined"!=typeof window?f=window:"undefined"!=typeof global?f=global:"undefined"!=typeof self&&(f=self),f.pica=e()}}(function(){var define,module,exports;return (function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);var f=new Error("Cannot find module '"+o+"'");throw f.code="MODULE_NOT_FOUND",f}var l=n[o]={exports:{}};t[o][0].call(l.exports,function(e){var n=t[o][1][e];return s(n?n:e)},l,l.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o= bkHalf && y >= bkHalf && x + bkHalf < srcW && y + bkHalf < srcH) { for (by = 0; by < 3; by++) { for (bx = 0; bx < 3; bx++) { sx = x + bx - bkHalf; sy = y + by - bkHalf; br += gs[sx + sy * srcW] * blurKernel[bPtr++]; } } return (br - (br % _bkWsum)) / _bkWsum; } for (by = 0; by < 3; by++) { for (bx = 0; bx < 3; bx++) { sx = x + bx - bkHalf; sy = y + by - bkHalf; if (sx >= 0 && sx < srcW && sy >= 0 && sy < srcH) { w = blurKernel[bPtr]; wsum += w; br += gs[sx + sy * srcW] * w; } bPtr++; } } return ((br - (br % wsum)) / wsum)|0; } function blur(src, srcW, srcH/*, radius*/) { var x, y, output = new Uint16Array(src.length); for (x = 0; x < srcW; x++) { for (y = 0; y < srcH; y++) { output[y * srcW + x] = blurPoint(src, x, y, srcW, srcH); } } return output; } module.exports = blur; },{}],2:[function(require,module,exports){ // High speed resize with tuneable speed/quality ratio 'use strict'; var unsharp = require('./unsharp'); // Precision of fixed FP values var FIXED_FRAC_BITS = 14; var FIXED_FRAC_VAL = 1 << FIXED_FRAC_BITS; // // Presets for quality 0..3. Filter functions + window size // var FILTER_INFO = [ { // Nearest neibor (Box) win: 0.5, filter: function (x) { return (x >= -0.5 && x < 0.5) ? 1.0 : 0.0; } }, { // Hamming win: 1.0, filter: function (x) { if (x <= -1.0 || x >= 1.0) { return 0.0; } if (x > -1.19209290E-07 && x < 1.19209290E-07) { return 1.0; } var xpi = x * Math.PI; return ((Math.sin(xpi) / xpi) * (0.54 + 0.46 * Math.cos(xpi / 1.0))); } }, { // Lanczos, win = 2 win: 2.0, filter: function (x) { if (x <= -2.0 || x >= 2.0) { return 0.0; } if (x > -1.19209290E-07 && x < 1.19209290E-07) { return 1.0; } var xpi = x * Math.PI; return (Math.sin(xpi) / xpi) * Math.sin(xpi / 2.0) / (xpi / 2.0); } }, { // Lanczos, win = 3 win: 3.0, filter: function (x) { if (x <= -3.0 || x >= 3.0) { return 0.0; } if (x > -1.19209290E-07 && x < 1.19209290E-07) { return 1.0; } var xpi = x * Math.PI; return (Math.sin(xpi) / xpi) * Math.sin(xpi / 3.0) / (xpi / 3.0); } } ]; function clampTo8(i) { return i < 0 ? 0 : (i > 255 ? 255 : i); } function toFixedPoint(num) { return Math.floor(num * FIXED_FRAC_VAL); } // Calculate convolution filters for each destination point, // and pack data to Int16Array: // // [ shift, length, data..., shift2, length2, data..., ... ] // // - shift - offset in src image // - length - filter length (in src points) // - data - filter values sequence // function createFilters(quality, srcSize, destSize) { var filterFunction = FILTER_INFO[quality].filter; var scale = destSize / srcSize; var scaleInverted = 1.0 / scale; var scaleClamped = Math.min(1.0, scale); // For upscale // Filter window (averaging interval), scaled to src image var srcWindow = FILTER_INFO[quality].win / scaleClamped; var destPixel, srcPixel, srcFirst, srcLast, filterElementSize, floatFilter, fxpFilter, total, fixedTotal, pxl, idx, floatVal, fixedVal; var leftNotEmpty, rightNotEmpty, filterShift, filterSize; var maxFilterElementSize = Math.floor((srcWindow + 1) * 2 ); var packedFilter = new Int16Array((maxFilterElementSize + 2) * destSize); var packedFilterPtr = 0; // For each destination pixel calculate source range and built filter values for (destPixel = 0; destPixel < destSize; destPixel++) { // Scaling should be done relative to central pixel point srcPixel = (destPixel + 0.5) * scaleInverted; srcFirst = Math.max(0, Math.floor(srcPixel - srcWindow)); srcLast = Math.min(srcSize - 1, Math.ceil(srcPixel + srcWindow)); filterElementSize = srcLast - srcFirst + 1; floatFilter = new Float32Array(filterElementSize); fxpFilter = new Int16Array(filterElementSize); total = 0.0; // Fill filter values for calculated range for (pxl = srcFirst, idx = 0; pxl <= srcLast; pxl++, idx++) { floatVal = filterFunction(((pxl + 0.5) - srcPixel) * scaleClamped); total += floatVal; floatFilter[idx] = floatVal; } // Normalize filter, convert to fixed point and accumulate conversion error fixedTotal = 0; for (idx = 0; idx < floatFilter.length; idx++) { fixedVal = toFixedPoint(floatFilter[idx] / total); fixedTotal += fixedVal; fxpFilter[idx] = fixedVal; } // Compensate normalization error, to minimize brightness drift fxpFilter[destSize >> 1] += toFixedPoint(1.0) - fixedTotal; // // Now pack filter to useable form // // 1. Trim heading and tailing zero values, and compensate shitf/length // 2. Put all to single array in this format: // // [ pos shift, data length, value1, value2, value3, ... ] // leftNotEmpty = 0; while (leftNotEmpty < fxpFilter.length && fxpFilter[leftNotEmpty] === 0) { leftNotEmpty++; } if (leftNotEmpty < fxpFilter.length) { rightNotEmpty = fxpFilter.length - 1; while (rightNotEmpty > 0 && fxpFilter[rightNotEmpty] === 0) { rightNotEmpty--; } filterShift = srcFirst + leftNotEmpty; filterSize = rightNotEmpty - leftNotEmpty + 1; packedFilter[packedFilterPtr++] = filterShift; // shift packedFilter[packedFilterPtr++] = filterSize; // size packedFilter.set(fxpFilter.subarray(leftNotEmpty, rightNotEmpty + 1), packedFilterPtr); packedFilterPtr += filterSize; } else { // zero data, write header only packedFilter[packedFilterPtr++] = 0; // shift packedFilter[packedFilterPtr++] = 0; // size } } return packedFilter; } // Convolve image in horizontal directions and transpose output. In theory, // transpose allow: // // - use the same convolver for both passes (this fails due different // types of input array and temporary buffer) // - making vertical pass by horisonltal lines inprove CPU cache use. // // But in real life this doesn't work :) // function convolveHorizontally(src, dest, srcW, srcH, destW, filters) { var r, g, b, a; var filterPtr, filterShift, filterSize; var srcPtr, srcY, destX, filterVal; var srcOffset = 0, destOffset = 0; // For each row for (srcY = 0; srcY < srcH; srcY++) { filterPtr = 0; // Apply precomputed filters to each destination row point for (destX = 0; destX < destW; destX++) { // Get the filter that determines the current output pixel. filterShift = filters[filterPtr++]; filterSize = filters[filterPtr++]; srcPtr = (srcOffset + (filterShift * 4))|0; r = g = b = a = 0; // Apply the filter to the row to get the destination pixel r, g, b, a for (; filterSize > 0; filterSize--) { filterVal = filters[filterPtr++]; // Use reverse order to workaround deopts in old v8 (node v.10) // Big thanks to @mraleph (Vyacheslav Egorov) for the tip. a = (a + filterVal * src[srcPtr + 3])|0; b = (b + filterVal * src[srcPtr + 2])|0; g = (g + filterVal * src[srcPtr + 1])|0; r = (r + filterVal * src[srcPtr])|0; srcPtr = (srcPtr + 4)|0; } // Bring this value back in range. All of the filter scaling factors // are in fixed point with FIXED_FRAC_BITS bits of fractional part. dest[destOffset + 3] = clampTo8(a >> 14/*FIXED_FRAC_BITS*/); dest[destOffset + 2] = clampTo8(b >> 14/*FIXED_FRAC_BITS*/); dest[destOffset + 1] = clampTo8(g >> 14/*FIXED_FRAC_BITS*/); dest[destOffset] = clampTo8(r >> 14/*FIXED_FRAC_BITS*/); destOffset = (destOffset + srcH * 4)|0; } destOffset = ((srcY + 1) * 4)|0; srcOffset = ((srcY + 1) * srcW * 4)|0; } } // Technically, convolvers are the same. But input array and temporary // buffer can be of different type (especially, in old browsers). So, // keep code in separate functions to avoid deoptimizations & speed loss. function convolveVertically(src, dest, srcW, srcH, destW, filters) { var r, g, b, a; var filterPtr, filterShift, filterSize; var srcPtr, srcY, destX, filterVal; var srcOffset = 0, destOffset = 0; // For each row for (srcY = 0; srcY < srcH; srcY++) { filterPtr = 0; // Apply precomputed filters to each destination row point for (destX = 0; destX < destW; destX++) { // Get the filter that determines the current output pixel. filterShift = filters[filterPtr++]; filterSize = filters[filterPtr++]; srcPtr = (srcOffset + (filterShift * 4))|0; r = g = b = a = 0; // Apply the filter to the row to get the destination pixel r, g, b, a for (; filterSize > 0; filterSize--) { filterVal = filters[filterPtr++]; // Use reverse order to workaround deopts in old v8 (node v.10) // Big thanks to @mraleph (Vyacheslav Egorov) for the tip. a = (a + filterVal * src[srcPtr + 3])|0; b = (b + filterVal * src[srcPtr + 2])|0; g = (g + filterVal * src[srcPtr + 1])|0; r = (r + filterVal * src[srcPtr])|0; srcPtr = (srcPtr + 4)|0; } // Bring this value back in range. All of the filter scaling factors // are in fixed point with FIXED_FRAC_BITS bits of fractional part. dest[destOffset + 3] = clampTo8(a >> 14/*FIXED_FRAC_BITS*/); dest[destOffset + 2] = clampTo8(b >> 14/*FIXED_FRAC_BITS*/); dest[destOffset + 1] = clampTo8(g >> 14/*FIXED_FRAC_BITS*/); dest[destOffset] = clampTo8(r >> 14/*FIXED_FRAC_BITS*/); destOffset = (destOffset + srcH * 4)|0; } destOffset = ((srcY + 1) * 4)|0; srcOffset = ((srcY + 1) * srcW * 4)|0; } } function resetAlpha(dst, width, height) { var ptr = 3, len = (width * height * 4)|0; while (ptr < len) { dst[ptr] = 0xFF; ptr = (ptr + 4)|0; } } function resize(options) { var src = options.src; var srcW = options.width; var srcH = options.height; var destW = options.toWidth; var destH = options.toHeight; var dest = options.dest || new Uint8Array(destW * destH * 4); var quality = options.quality === undefined ? 3 : options.quality; var alpha = options.alpha || false; var unsharpAmount = options.unsharpAmount === undefined ? 0 : (options.unsharpAmount|0); var unsharpThreshold = options.unsharpThreshold === undefined ? 0 : (options.unsharpThreshold|0); if (srcW < 1 || srcH < 1 || destW < 1 || destH < 1) { return []; } var filtersX = createFilters(quality, srcW, destW), filtersY = createFilters(quality, srcH, destH); var tmp = new Uint8Array(destW * srcH * 4); // To use single function we need src & tmp of the same type. // But src can be CanvasPixelArray, and tmp - Uint8Array. So, keep // vertical and horizontal passes separately to avoid deoptimization. convolveHorizontally(src, tmp, srcW, srcH, destW, filtersX); convolveVertically(tmp, dest, srcH, destW, destH, filtersY); // That's faster than doing checks in convolver. // !!! Note, canvas data is not premultipled. We don't need other // alpha corrections. if (!alpha) { resetAlpha(dest, destW, destH); } if (unsharpAmount) { unsharp(dest, destW, destH, unsharpAmount, 1.0, unsharpThreshold); } return dest; } module.exports = resize; },{"./unsharp":3}],3:[function(require,module,exports){ // Unsharp mask filter // // http://stackoverflow.com/a/23322820/1031804 // USM(O) = O + (2 * (Amount / 100) * (O - GB)) // GB - gaussial blur. // // brightness = 0.299*R + 0.587*G + 0.114*B // http://stackoverflow.com/a/596243/1031804 // // To simplify math, normalize brighness mutipliers to 2^16: // // brightness = (19595*R + 38470*G + 7471*B) / 65536 'use strict'; var blur = require('./blur'); function clampTo8(i) { return i < 0 ? 0 : (i > 255 ? 255 : i); } // Convert image to greyscale, 16bits FP result (8.8) // function greyscale(src, srcW, srcH) { var size = srcW * srcH; var result = new Uint16Array(size); // We don't use sign, but that helps to JIT var i, srcPtr; for (i = 0, srcPtr = 0; i < size; i++) { result[i] = (src[srcPtr + 2] * 7471 // blue + src[srcPtr + 1] * 38470 // green + src[srcPtr] * 19595) >>> 8; // red srcPtr = (srcPtr + 4)|0; } return result; } // Apply unsharp mask to src // // NOTE: radius is ignored to simplify gaussian blur calculation // on practice we need radius 0.3..2.0. Use 1.0 now. // function unsharp(src, srcW, srcH, amount, radius, threshold) { var x, y, c, diff = 0, corr, srcPtr; // Normalized delta multiplier. Expect that: var AMOUNT_NORM = Math.floor(amount * 256 / 50); // Convert to grayscale: // // - prevent color drift // - speedup blur calc // var gs = greyscale(src, srcW, srcH); var blured = blur(gs, srcW, srcH, 1); var fpThreshold = threshold << 8; var gsPtr = 0; for (y = 0; y < srcH; y++) { for (x = 0; x < srcW; x++) { // calculate brightness blur, difference & update source buffer diff = gs[gsPtr] - blured[gsPtr]; // Update source image if thresold exceeded if (Math.abs(diff) > fpThreshold) { // Calculate correction multiplier corr = 65536 + ((diff * AMOUNT_NORM) >> 8); srcPtr = gsPtr * 4; c = src[srcPtr]; src[srcPtr++] = clampTo8((c * corr) >> 16); c = src[srcPtr]; src[srcPtr++] = clampTo8((c * corr) >> 16); c = src[srcPtr]; src[srcPtr] = clampTo8((c * corr) >> 16); } gsPtr++; } // end row } // end column } module.exports = unsharp; },{"./blur":1}],4:[function(require,module,exports){ // Proxy to simplify split between webworker/plain calls 'use strict'; var resize = require('./pure/resize'); module.exports = function (options, callback) { var output = resize(options); callback(null, output); }; },{"./pure/resize":2}],5:[function(require,module,exports){ // Web Worker wrapper for image resize function 'use strict'; module.exports = function(self) { var resize = require('./resize'); self.onmessage = function (ev) { resize(ev.data, function(err, output) { if (err) { self.postMessage({ err: err }); return; } self.postMessage({ output: output }, [ output.buffer ]); }); }; }; },{"./resize":4}],6:[function(require,module,exports){ var bundleFn = arguments[3]; var sources = arguments[4]; var cache = arguments[5]; var stringify = JSON.stringify; module.exports = function (fn) { var keys = []; var wkey; var cacheKeys = Object.keys(cache); for (var i = 0, l = cacheKeys.length; i < l; i++) { var key = cacheKeys[i]; if (cache[key].exports === fn) { wkey = key; break; } } if (!wkey) { wkey = Math.floor(Math.pow(16, 8) * Math.random()).toString(16); var wcache = {}; for (var i = 0, l = cacheKeys.length; i < l; i++) { var key = cacheKeys[i]; wcache[key] = key; } sources[wkey] = [ Function(['require','module','exports'], '(' + fn + ')(self)'), wcache ]; } var skey = Math.floor(Math.pow(16, 8) * Math.random()).toString(16); var scache = {}; scache[wkey] = wkey; sources[skey] = [ Function(['require'],'require(' + stringify(wkey) + ')(self)'), scache ]; var src = '(' + bundleFn + ')({' + Object.keys(sources).map(function (key) { return stringify(key) + ':[' + sources[key][0] + ',' + stringify(sources[key][1]) + ']' ; }).join(',') + '},{},[' + stringify(skey) + '])' ; return new Worker(window.URL.createObjectURL( new Blob([src], { type: 'text/javascript' }) )); }; },{}]},{},[])("./") }); ================================================ FILE: demo/js/regression.js ================================================ var N, data, labels; var ss = 30.0; // scale for drawing var layer_defs, net, trainer; // create neural net var t = "layer_defs = [];\n\ layer_defs.push({type:'input', out_sx:1, out_sy:1, out_depth:1});\n\ layer_defs.push({type:'fc', num_neurons:20, activation:'relu'});\n\ layer_defs.push({type:'fc', num_neurons:20, activation:'sigmoid'});\n\ layer_defs.push({type:'regression', num_neurons:1});\n\ \n\ net = new convnetjs.Net();\n\ net.makeLayers(layer_defs);\n\ \n\ trainer = new convnetjs.SGDTrainer(net, {learning_rate:0.01, momentum:0.0, batch_size:1, l2_decay:0.001});\n\ "; var lix=2; // layer id of layer we'd like to draw outputs of function reload() { eval($("#layerdef").val()); // refresh buttons var t = ''; for(var i=1;i"; } $("#layer_ixes").html(t); $("#button"+lix).css('background-color', '#FFA'); } function updateLix(newlix) { $("#button"+lix).css('background-color', ''); // erase highlight lix = newlix; $("#button"+lix).css('background-color', '#FFA'); } function regen_data() { N = parseInt($("#num_data").val()); data = []; labels = []; for(var i=0;i0.0&&ua<1.0&&ub>0.0&&ub<1.0) { var up = new Vec(p1.x+ua*(p2.x-p1.x), p1.y+ua*(p2.y-p1.y)); return {ua:ua, ub:ub, up:up}; // up is intersection point } return false; } var line_point_intersect = function(p1,p2,p0,rad) { var v = new Vec(p2.y-p1.y,-(p2.x-p1.x)); // perpendicular vector var d = Math.abs((p2.x-p1.x)*(p1.y-p0.y)-(p1.x-p0.x)*(p2.y-p1.y)); d = d / v.length(); if(d > rad) { return false; } v.normalize(); v.scale(d); var up = p0.add(v); if(Math.abs(p2.x-p1.x)>Math.abs(p2.y-p1.y)) { var ua = (up.x - p1.x) / (p2.x - p1.x); } else { var ua = (up.y - p1.y) / (p2.y - p1.y); } if(ua>0.0&&ua<1.0) { return {ua:ua, up:up}; } return false; } // Wall is made up of two points var Wall = function(p1, p2) { this.p1 = p1; this.p2 = p2; } // World object contains many agents and walls and food and stuff var util_add_box = function(lst, x, y, w, h) { lst.push(new Wall(new Vec(x,y), new Vec(x+w,y))); lst.push(new Wall(new Vec(x+w,y), new Vec(x+w,y+h))); lst.push(new Wall(new Vec(x+w,y+h), new Vec(x,y+h))); lst.push(new Wall(new Vec(x,y+h), new Vec(x,y))); } // item is circle thing on the floor that agent can interact with (see or eat, etc) var Item = function(x, y, type) { this.p = new Vec(x, y); // position this.type = type; this.rad = 10; // default radius this.age = 0; this.cleanup_ = false; } var World = function() { this.agents = []; this.W = canvas.width; this.H = canvas.height; this.clock = 0; // set up walls in the world this.walls = []; var pad = 10; util_add_box(this.walls, pad, pad, this.W-pad*2, this.H-pad*2); util_add_box(this.walls, 100, 100, 200, 300); // inner walls this.walls.pop(); util_add_box(this.walls, 400, 100, 200, 300); this.walls.pop(); // set up food and poison this.items = [] for(var k=0;k<30;k++) { var x = convnetjs.randf(20, this.W-20); var y = convnetjs.randf(20, this.H-20); var t = convnetjs.randi(1, 3); // food or poison (1 and 2) var it = new Item(x, y, t); this.items.push(it); } } World.prototype = { // helper function to get closest colliding walls/items stuff_collide_: function(p1, p2, check_walls, check_items) { var minres = false; // collide with walls if(check_walls) { for(var i=0,n=this.walls.length;ieyep var eyep = new Vec(a.p.x + e.max_range * Math.sin(a.angle + e.angle), a.p.y + e.max_range * Math.cos(a.angle + e.angle)); var res = this.stuff_collide_(a.p, eyep, true, true); if(res) { // eye collided with wall e.sensed_proximity = res.up.dist_from(a.p); e.sensed_type = res.type; } else { e.sensed_proximity = e.max_range; e.sensed_type = -1; } } } // let the agents behave in the world based on their input for(var i=0,n=this.agents.length;i2*Math.PI)a.angle-=2*Math.PI; // agent is trying to move from p to op. Check walls var res = this.stuff_collide_(a.op, a.p, true, false); if(res) { // wall collision! reset position a.p = a.op; } // handle boundary conditions if(a.p.x<0)a.p.x=0; if(a.p.x>this.W)a.p.x=this.W; if(a.p.y<0)a.p.y=0; if(a.p.y>this.H)a.p.y=this.H; } // tick all items var update_items = false; for(var i=0,n=this.items.length;i 5000 && this.clock % 100 === 0 && convnetjs.randf(0,1)<0.1) { it.cleanup_ = true; // replace this one, has been around too long update_items = true; } } if(update_items) { var nt = []; for(var i=0,n=this.items.length;i 0.75) forward_reward = 0.1 * proximity_reward; // agents like to eat good things var digestion_reward = this.digestion_signal; this.digestion_signal = 0.0; var reward = proximity_reward + forward_reward + digestion_reward; // pass to brain for learning this.brain.backward(reward); } } function draw_net() { if(simspeed <=1) { // we will always draw at these speeds } else { if(w.clock % 50 !== 0) return; // do this sparingly } var canvas = document.getElementById("net_canvas"); var ctx = canvas.getContext("2d"); var W = canvas.width; var H = canvas.height; ctx.clearRect(0, 0, canvas.width, canvas.height); var L = w.agents[0].brain.value_net.layers; var dx = (W - 50)/L.length; var x = 10; var y = 40; ctx.font="12px Verdana"; ctx.fillStyle = "rgb(0,0,0)"; ctx.fillText("Value Function Approximating Neural Network:", 10, 14); for(var k=0;k= 0) ctx.fillStyle = "rgb(0,0," + v + ")"; if(v < 0) ctx.fillStyle = "rgb(" + (-v) + ",0,0)"; ctx.fillRect(x,y,10,10); y += 12; if(y>H-25) { y = 40; x += 12}; } x += 50; y = 40; } } var reward_graph = new cnnvis.Graph(); function draw_stats() { var canvas = document.getElementById("vis_canvas"); var ctx = canvas.getContext("2d"); var W = canvas.width; var H = canvas.height; ctx.clearRect(0, 0, canvas.width, canvas.height); var a = w.agents[0]; var b = a.brain; var netin = b.last_input_array; ctx.strokeStyle = "rgb(0,0,0)"; //ctx.font="12px Verdana"; //ctx.fillText("Current state:",10,10); ctx.lineWidth = 10; ctx.beginPath(); for(var k=0,n=netin.length;k255)r=255;if(r<0)r=0; ctx.fillStyle = "rgb(" + r + ", 150, 150)"; ctx.strokeStyle = "rgb(0,0,0)"; for(var i=0,n=agents.length;i0) { for(var i=0;i ConvNetJS MNIST demo

ConvNetJS MNIST demo

Description

This demo trains a Convolutional Neural Network on the MNIST digits dataset in your browser, with nothing but Javascript. The dataset is fairly easy and one should expect to get somewhere around 99% accuracy within few minutes. I used this python script to parse the original files into batches of images that can be easily loaded into page DOM with img tags.

This network takes a 28x28 MNIST image and crops a random 24x24 window before training on it (this technique is called data augmentation and improves generalization). Similarly to do prediction, 4 random crops are sampled and the probabilities across all crops are averaged to produce final predictions. The network runs at about 5ms for both forward and backward pass on my reasonably decent Ubuntu+Chrome machine.

By default, in this demo we're using Adadelta which is one of per-parameter adaptive step size methods, so we don't have to worry about changing learning rates or momentum over time. However, I still included the text fields for changing these if you'd like to play around with SGD+Momentum trainer.

Report questions/bugs/suggestions to @karpathy.

Training Stats

Learning rate:
Momentum:
Batch size:
Weight decay:


Loss:

Test an image from your computer:

Instantiate a Network and Trainer


Network Visualization

Example predictions on Test set

================================================ FILE: demo/regression.html ================================================ ConvNetJS demo: Classify toy 2D data

ConvnetJS demo: toy 1d regression

The simulation below is a 1-dimensional regression where a neural network is trained to regress to y coordinates for every given point x through an L2 loss. That is, the minimized cost function computes the squared difference between the predicted y-coordinate and the "correct" y coordinate. Every 10th of a second, all points are fed to the network multiple times through the trainer class to train the network.

The simulation below will eval() whatever you have in the text area and reload. Feel free to explore and use ConvNetJS to instantiate your own network!

Report questions/bugs/suggestions to @karpathy.



Number of points to generate:

Add data points by clicking!


Also draw outputs of a layer (click layer button below) in red.
Browser not supported for Canvas. Get a real browser.

Go back to ConvNetJS

================================================ FILE: demo/rldemo.html ================================================ ConvNetJS Deep Q Learning Reinforcement Learning with Neural Network demo

ConvNetJS Deep Q Learning Demo

Description

This demo follows the description of the Deep Q Learning algorithm described in Playing Atari with Deep Reinforcement Learning, a paper from NIPS 2013 Deep Learning Workshop from DeepMind. The paper is a nice demo of a fairly standard (model-free) Reinforcement Learning algorithm (Q Learning) learning to play Atari games.

In this demo, instead of Atari games, we'll start out with something more simple: a 2D agent that has 9 eyes pointing in different angles ahead and every eye senses 3 values along its direction (up to a certain maximum visibility distance): distance to a wall, distance to a green thing, or distance to a red thing. The agent navigates by using one of 5 actions that turn it different angles. The red things are apples and the agent gets reward for eating them. The green things are poison and the agent gets negative reward for eating them. The training takes a few tens of minutes with current parameter settings.

Over time, the agent learns to avoid states that lead to states with low rewards, and picks actions that lead to better states instead.

Q-Learner full specification and options

The textfield below gets eval()'d to produce the Q-learner for this demo. This allows you to fiddle with various parameters and settings and also shows how you can use the API for your own purposes. All of these settings are optional but are listed to give an idea of possibilities. Feel free to change things around and hit reload! Documentation for all options is the paper linked to above, and there are also comments for every option in the source code javascript file.

Q-Learner API

It's very simple to use deeqlearn.Brain: Initialize your network:

   var brain = new deepqlearn.Brain(num_inputs, num_actions);

And to train it proceed in loops as follows:

   var action = brain.forward(array_with_num_inputs_numbers);
   // action is a number in [0, num_actions) telling index of the action the agent chooses
   // here, apply the action on environment and observe some reward. Finally, communicate it:
   brain.backward(reward); // <-- learning magic happens here

That's it! Let the agent learn over time (it will take opt.learning_steps_total), and it will only get better and better at accumulating reward as it learns. Note that the agent will still take random actions with probability opt.epsilon_min even once it's fully trained. To completely disable this randomness, or change it, you can disable the learning and set epsilon_test_time to 0:

   brain.epsilon_test_time = 0.0; // don't make any random choices, ever
   brain.learning = false;
   var action = brain.forward(array_with_num_inputs_numbers); // get optimal action from learned policy

State Visualizations

Left: Current input state (quite a useless thing to look at). Right: Average reward over time (this should go up as agent becomes better on average at collecting rewards)


(Takes ~10 minutes to train with current settings. If you're impatient, scroll down and load an example pre-trained network from pre-filled JSON)

Controls


I/O

You can save and load a network from JSON here. Note that the textfield is prefilled with a pretrained network that works reasonable well, if you're impatient to let yours train enough. Just hit the load button!


================================================ FILE: demo/speedtest.html ================================================ Simple speed test
================================================ FILE: demo/trainers.html ================================================ ConvNetJS Trainer Comparison on MNIST

ConvNetJS Trainer demo on MNIST

Description

This demo lets you evaluate multiple trainers against each other on MNIST. By default I've set up a little benchmark that puts SGD/SGD with momentum/Adam/Adagrad/Adadelta/Nesterov against each other. For reference math and explanations on these refer to Matthew Zeiler's Adadelta paper (Windowgrad is Idea #1 in the paper). In my own experience, Adagrad/Adadelta are "safer" because they don't depend so strongly on setting of learning rates (with Adadelta being slightly better), but well-tuned SGD+Momentum almost always converges faster and at better final values.

Report questions/bugs/suggestions to @karpathy.



Loss vs. Number of examples seen

Testing Accuracy vs. Number of examples seen

Training Accuracy vs. Number of examples seen

================================================ FILE: src/convnet_export.js ================================================ (function(lib) { "use strict"; if (typeof module === "undefined" || typeof module.exports === "undefined") { window.convnetjs = lib; // in ordinary browser attach library to window } else { module.exports = lib; // in nodejs } })(convnetjs); ================================================ FILE: src/convnet_init.js ================================================ var convnetjs = convnetjs || { REVISION: 'ALPHA' }; ================================================ FILE: src/convnet_layers_dotproducts.js ================================================ (function(global) { "use strict"; var Vol = global.Vol; // convenience // This file contains all layers that do dot products with input, // but usually in a different connectivity pattern and weight sharing // schemes: // - FullyConn is fully connected dot products // - ConvLayer does convolutions (so weight sharing spatially) // putting them together in one file because they are very similar var ConvLayer = function(opt) { var opt = opt || {}; // required this.out_depth = opt.filters; this.sx = opt.sx; // filter size. Should be odd if possible, it's cleaner. this.in_depth = opt.in_depth; this.in_sx = opt.in_sx; this.in_sy = opt.in_sy; // optional this.sy = typeof opt.sy !== 'undefined' ? opt.sy : this.sx; this.stride = typeof opt.stride !== 'undefined' ? opt.stride : 1; // stride at which we apply filters to input volume this.pad = typeof opt.pad !== 'undefined' ? opt.pad : 0; // amount of 0 padding to add around borders of input volume this.l1_decay_mul = typeof opt.l1_decay_mul !== 'undefined' ? opt.l1_decay_mul : 0.0; this.l2_decay_mul = typeof opt.l2_decay_mul !== 'undefined' ? opt.l2_decay_mul : 1.0; // computed // note we are doing floor, so if the strided convolution of the filter doesnt fit into the input // volume exactly, the output volume will be trimmed and not contain the (incomplete) computed // final application. this.out_sx = Math.floor((this.in_sx + this.pad * 2 - this.sx) / this.stride + 1); this.out_sy = Math.floor((this.in_sy + this.pad * 2 - this.sy) / this.stride + 1); this.layer_type = 'conv'; // initializations var bias = typeof opt.bias_pref !== 'undefined' ? opt.bias_pref : 0.0; this.filters = []; for(var i=0;i=0 && oy=0 && ox=0 && oy=0 && ox amax) amax = as[i]; } // compute exponentials (carefully to not blow up) var es = global.zeros(this.out_depth); var esum = 0.0; for(var i=0;i 0) { // violating dimension, apply loss x.dw[i] += 1; x.dw[y] -= 1; loss += ydiff; } } return loss; }, getParamsAndGrads: function() { return []; }, toJSON: function() { var json = {}; json.out_depth = this.out_depth; json.out_sx = this.out_sx; json.out_sy = this.out_sy; json.layer_type = this.layer_type; json.num_inputs = this.num_inputs; return json; }, fromJSON: function(json) { this.out_depth = json.out_depth; this.out_sx = json.out_sx; this.out_sy = json.out_sy; this.layer_type = json.layer_type; this.num_inputs = json.num_inputs; } } global.RegressionLayer = RegressionLayer; global.SoftmaxLayer = SoftmaxLayer; global.SVMLayer = SVMLayer; })(convnetjs); ================================================ FILE: src/convnet_layers_nonlinearities.js ================================================ (function(global) { "use strict"; var Vol = global.Vol; // convenience // Implements ReLU nonlinearity elementwise // x -> max(0, x) // the output is in [0, inf) var ReluLayer = function(opt) { var opt = opt || {}; // computed this.out_sx = opt.in_sx; this.out_sy = opt.in_sy; this.out_depth = opt.in_depth; this.layer_type = 'relu'; } ReluLayer.prototype = { forward: function(V, is_training) { this.in_act = V; var V2 = V.clone(); var N = V.w.length; var V2w = V2.w; for(var i=0;i 1/(1+e^(-x)) // so the output is between 0 and 1. var SigmoidLayer = function(opt) { var opt = opt || {}; // computed this.out_sx = opt.in_sx; this.out_sy = opt.in_sy; this.out_depth = opt.in_depth; this.layer_type = 'sigmoid'; } SigmoidLayer.prototype = { forward: function(V, is_training) { this.in_act = V; var V2 = V.cloneAndZero(); var N = V.w.length; var V2w = V2.w; var Vw = V.w; for(var i=0;i max(x) // where x is a vector of size group_size. Ideally of course, // the input size should be exactly divisible by group_size var MaxoutLayer = function(opt) { var opt = opt || {}; // required this.group_size = typeof opt.group_size !== 'undefined' ? opt.group_size : 2; // computed this.out_sx = opt.in_sx; this.out_sy = opt.in_sy; this.out_depth = Math.floor(opt.in_depth / this.group_size); this.layer_type = 'maxout'; this.switches = global.zeros(this.out_sx*this.out_sy*this.out_depth); // useful for backprop } MaxoutLayer.prototype = { forward: function(V, is_training) { this.in_act = V; var N = this.out_depth; var V2 = new Vol(this.out_sx, this.out_sy, this.out_depth, 0.0); // optimization branch. If we're operating on 1D arrays we dont have // to worry about keeping track of x,y,d coordinates inside // input volumes. In convnets we do :( if(this.out_sx === 1 && this.out_sy === 1) { for(var i=0;i a) { a = a2; ai = j; } } V2.w[i] = a; this.switches[i] = ix + ai; } } else { var n=0; // counter for switches for(var x=0;x a) { a = a2; ai = j; } } V2.set(x,y,i,a); this.switches[n] = ix + ai; n++; } } } } this.out_act = V2; return this.out_act; }, backward: function() { var V = this.in_act; // we need to set dw of this var V2 = this.out_act; var N = this.out_depth; V.dw = global.zeros(V.w.length); // zero out gradient wrt data // pass the gradient through the appropriate switch if(this.out_sx === 1 && this.out_sy === 1) { for(var i=0;i tanh(x) // so the output is between -1 and 1. var TanhLayer = function(opt) { var opt = opt || {}; // computed this.out_sx = opt.in_sx; this.out_sy = opt.in_sy; this.out_depth = opt.in_depth; this.layer_type = 'tanh'; } TanhLayer.prototype = { forward: function(V, is_training) { this.in_act = V; var V2 = V.cloneAndZero(); var N = V.w.length; for(var i=0;i=0 && oy=0 && ox a) { a = v; winx=ox; winy=oy;} } } } this.switchx[n] = winx; this.switchy[n] = winy; n++; A.set(ax, ay, d, a); } } } this.out_act = A; return this.out_act; }, backward: function() { // pooling layers have no parameters, so simply compute // gradient wrt data here var V = this.in_act; V.dw = global.zeros(V.w.length); // zero out gradient wrt data var A = this.out_act; // computed in forward pass var n = 0; for(var d=0;d num_epochs * num_training_data this.foldix = 0; // index of active fold // callbacks this.finish_fold_callback = null; this.finish_batch_callback = null; // initializations if(this.data.length > 0) { this.sampleFolds(); this.sampleCandidates(); } }; MagicNet.prototype = { // sets this.folds to a sampling of this.num_folds folds sampleFolds: function() { var N = this.data.length; var num_train = Math.floor(this.train_ratio * N); this.folds = []; // flush folds, if any for(var i=0;i