Repository: BrainJS/brain.js Branch: master Commit: 7c9db32d9c38 Files: 220 Total size: 944.6 KB Directory structure: gitextract_h9w2eaqg/ ├── .editorconfig ├── .eslintignore ├── .eslintrc ├── .gitattributes ├── .github/ │ ├── CONTRIBUTING.md │ ├── ISSUE_TEMPLATE/ │ │ ├── bug_report.md │ │ └── feature_request.md │ ├── PULL_REQUEST_TEMPLATE.md │ └── workflows/ │ └── nodejs.yml ├── .gitignore ├── .npmignore ├── .npmrc ├── .nvmrc ├── .prettierrc ├── .travis.yml ├── .vscode/ │ └── launch.json ├── LICENSE ├── README.md ├── jest.config.json ├── package.json ├── rollup.config.browser.js ├── rollup.config.js ├── src/ │ ├── README.md │ ├── activation/ │ │ ├── README.md │ │ ├── index.test.ts │ │ ├── index.ts │ │ ├── leaky-relu.test.ts │ │ ├── leaky-relu.ts │ │ ├── relu.test.ts │ │ ├── relu.ts │ │ ├── sigmoid.test.ts │ │ ├── sigmoid.ts │ │ ├── tanh.test.ts │ │ └── tanh.ts │ ├── autoencoder.test.ts │ ├── autoencoder.ts │ ├── cross-validate.test.ts │ ├── cross-validate.ts │ ├── errors/ │ │ └── untrained-neural-network-error.ts │ ├── estimator/ │ │ └── mean-squared-error.ts │ ├── feed-forward.end-to-end.test.ts │ ├── feed-forward.ts │ ├── feed-forward.unit.test.ts │ ├── index.ts │ ├── layer/ │ │ ├── README.md │ │ ├── activation.test.ts │ │ ├── activation.ts │ │ ├── add.test.ts │ │ ├── add.ts │ │ ├── arthur-feed-forward.ts │ │ ├── base-layer.test.ts │ │ ├── base-layer.ts │ │ ├── convolution.test.ts │ │ ├── convolution.ts │ │ ├── dropout.test.ts │ │ ├── dropout.ts │ │ ├── feed-forward.test.ts │ │ ├── feed-forward.ts │ │ ├── filter.ts │ │ ├── fully-connected.test.ts │ │ ├── fully-connected.ts │ │ ├── gru.ts │ │ ├── index.ts │ │ ├── input.test.ts │ │ ├── input.ts │ │ ├── internal.ts │ │ ├── leaky-relu.test.ts │ │ ├── leaky-relu.ts │ │ ├── lstm-cell.test.ts │ │ ├── lstm-cell.ts │ │ ├── modifier.ts │ │ ├── multiply-element.test.ts │ │ ├── multiply-element.ts │ │ ├── multiply.test.ts │ │ ├── multiply.ts │ │ ├── negative.test.ts │ │ ├── negative.ts │ │ ├── ones.ts │ │ ├── operator.ts │ │ ├── output.ts │ │ ├── pool.test.ts │ │ ├── pool.ts │ │ ├── random.test.ts │ │ ├── random.ts │ │ ├── recurrent-connection.ts │ │ ├── recurrent-input.ts │ │ ├── recurrent-zeros.ts │ │ ├── regression.ts │ │ ├── relu.test.ts │ │ ├── relu.ts │ │ ├── rnn-cell.test.ts │ │ ├── rnn-cell.ts │ │ ├── sigmoid.test.ts │ │ ├── sigmoid.ts │ │ ├── soft-max.test.ts │ │ ├── soft-max.ts │ │ ├── svm.ts │ │ ├── tanh.test.ts │ │ ├── tanh.ts │ │ ├── target.test.ts │ │ ├── target.ts │ │ ├── transpose.ts │ │ ├── types.ts │ │ └── zeros.ts │ ├── likely.test.ts │ ├── likely.ts │ ├── lookup.test.ts │ ├── lookup.ts │ ├── neural-network-gpu.end-to-end.test.ts │ ├── neural-network-gpu.test.ts │ ├── neural-network-gpu.ts │ ├── neural-network-types.ts │ ├── neural-network.bitwise.test.ts │ ├── neural-network.json.test.ts │ ├── neural-network.options.test.ts │ ├── neural-network.test-method.test.ts │ ├── neural-network.to-function.test.ts │ ├── neural-network.trainopts.test.ts │ ├── neural-network.ts │ ├── neural-network.unit.test.ts │ ├── praxis/ │ │ ├── README.md │ │ ├── arthur-deviation-biases.end-to-end.test.ts │ │ ├── arthur-deviation-biases.ts │ │ ├── arthur-deviation-biases.unit.test.ts │ │ ├── arthur-deviation-weights.end-to-end.test.ts │ │ ├── arthur-deviation-weights.ts │ │ ├── arthur-deviation-weights.unit.test.ts │ │ ├── base-praxis.ts │ │ ├── index.ts │ │ ├── momentum-root-mean-squared-propagation.test.ts │ │ └── momentum-root-mean-squared-propagation.ts │ ├── recurrent/ │ │ ├── gru-time-step.test.ts │ │ ├── gru-time-step.ts │ │ ├── gru.test.ts │ │ ├── gru.ts │ │ ├── lstm-time-step.end-to-end.test.ts │ │ ├── lstm-time-step.test.ts │ │ ├── lstm-time-step.ts │ │ ├── lstm.test.ts │ │ ├── lstm.ts │ │ ├── matrix/ │ │ │ ├── add-b.ts │ │ │ ├── add.ts │ │ │ ├── all-ones.ts │ │ │ ├── clone-negative.ts │ │ │ ├── clone.ts │ │ │ ├── copy.ts │ │ │ ├── equation.test.ts │ │ │ ├── equation.ts │ │ │ ├── index.test.ts │ │ │ ├── index.ts │ │ │ ├── max-i.ts │ │ │ ├── multiply-b.ts │ │ │ ├── multiply-element-b.ts │ │ │ ├── multiply-element.ts │ │ │ ├── multiply.ts │ │ │ ├── ones-matrix.ts │ │ │ ├── random-matrix.ts │ │ │ ├── random-n-matrix.ts │ │ │ ├── relu-b.ts │ │ │ ├── relu.ts │ │ │ ├── row-pluck-b.ts │ │ │ ├── row-pluck.ts │ │ │ ├── sample-i.ts │ │ │ ├── sigmoid-b.ts │ │ │ ├── sigmoid.ts │ │ │ ├── softmax.ts │ │ │ ├── tanh-b.ts │ │ │ └── tanh.ts │ │ ├── rnn-data-types.ts │ │ ├── rnn-time-step.test.ts │ │ ├── rnn-time-step.ts │ │ ├── rnn.test.ts │ │ └── rnn.ts │ ├── recurrent.baseline.test.ts │ ├── recurrent.end-to-end.test.ts │ ├── recurrent.ts │ ├── recurrent.unit.test.ts │ ├── test-utils.ts │ └── utilities/ │ ├── array-lookup-table.ts │ ├── cast.test.ts │ ├── cast.ts │ ├── data-formatter.test.ts │ ├── data-formatter.ts │ ├── flatten-layers.test.ts │ ├── flatten-layers.ts │ ├── kernel.ts │ ├── layer-from-json.test.ts │ ├── layer-from-json.ts │ ├── layer-setup.ts │ ├── layer-size.test.ts │ ├── layer-size.ts │ ├── lookup-table.ts │ ├── max.test.ts │ ├── max.ts │ ├── mse.test.ts │ ├── mse.ts │ ├── ones.test.ts │ ├── ones.ts │ ├── random-weight.test.ts │ ├── random-weight.ts │ ├── random.test.ts │ ├── random.ts │ ├── randos.test.ts │ ├── randos.ts │ ├── range.test.ts │ ├── range.ts │ ├── to-array.test.ts │ ├── to-array.ts │ ├── to-svg.test.ts │ ├── to-svg.ts │ ├── traverse-layers-excluding-from.ts │ ├── traverse-layers-from.ts │ ├── values-2d.ts │ ├── values-3d.ts │ ├── values.ts │ ├── zeros-2d.ts │ ├── zeros-3d.ts │ ├── zeros.test.ts │ └── zeros.ts └── tsconfig.json ================================================ FILE CONTENTS ================================================ ================================================ FILE: .editorconfig ================================================ root = true [*] indent_style = space indent_size = 2 charset = utf-8 trim_trailing_whitespace = true insert_final_newline = true [*.md] trim_trailing_whitespace = false ================================================ FILE: .eslintignore ================================================ # Mac. .DS_STORE **.DS_STORE # Node. node_modules npm-debug.log # Yarn yarn.lock # parcel bundler cache .cache # code coverage __coverage__ test*.* test/unit/coverage/** test/unit/*.js test/e2e/*.js **.min.js dist/ __coverage__/ index.d.ts ================================================ FILE: .eslintrc ================================================ { "env": { "es6": true }, "extends": [ "eslint:recommended", "standard-with-typescript", "plugin:@typescript-eslint/eslint-recommended", "plugin:@typescript-eslint/recommended", "plugin:prettier/recommended", "prettier/@typescript-eslint" ], "parser": "@typescript-eslint/parser", "parserOptions": { "project": "./tsconfig.json", "sourceType": "module", "ecmaVersion": 2020 }, "plugins": ["prettier", "@typescript-eslint"], "root": true, "rules": { "@typescript-eslint/explicit-function-return-type": "off", "@typescript-eslint/interface-name-prefix": "off", "@typescript-eslint/member-delimiter-style": "off", "@typescript-eslint/no-empty-function": "off", "@typescript-eslint/no-var-requires": "off", "@typescript-eslint/semi": "off", "@typescript-eslint/space-before-function-paren": "off", "@typescript-eslint/strict-boolean-expressions": "off", "arrow-parens": "off", "class-methods-use-this": "off", "max-classes-per-file": "off", "no-continue": "off", "no-empty-function": "off", "no-multi-assign": "off", "no-param-reassign": "off", "no-plusplus": "off", "no-prototype-builtins": "off", "no-restricted-globals": "off", "no-underscore-dangle": "off", "prettier/prettier": "error", "semi": "off", "standard/no-callback-literal": "off", "no-implied-eval": "off" } } ================================================ FILE: .gitattributes ================================================ dist/* linguist-vendored examples/* linguist-documentation *.js linguist-detectable=false ================================================ FILE: .github/CONTRIBUTING.md ================================================ # Contribution Steps: Thanks for taking the time to contribute to brain.js. Follow these guidelines to make the process smoother: 1. Clone `master` branch 2. One feature per pull request. Each PR should have one focus, and all the code changes should be supporting that one feature or bug fix. Using a [separate branch](https://guides.github.com/introduction/flow/index.html) for each feature should help you manage developing multiple features at once. 3. Add/update a test for the feature or fix, if possible. To run these tests: ```bash npm run test # run tests ``` # Notes: - This repository uses `.editorconfig`, `eslint` (`airbnb`) and `prettier` for linting and formating to make coding style consistent throughout the repository, which will automatically run on git `commit`. - Please do not run build/dist script and do not bump version number for the script. These things will be handled by the maintainers when necessary. ================================================ FILE: .github/ISSUE_TEMPLATE/bug_report.md ================================================ --- name: Bug Report 🐞 about: Create a report to help us improve. labels: bug --- ![A GIF or MEME to give some spice of the internet](url) ## _What_ is wrong? ## _Where_ does it happen? ## _How_ do we replicate the issue? - Step 1 - Step 2 - Step 3 ## Expected behavior (i.e. solution) ## Version information ### Nodejs: ### Browser: ### Brain.js: ## _How_ important is this (1-5)? ## Other Comments ================================================ FILE: .github/ISSUE_TEMPLATE/feature_request.md ================================================ --- name: Feature Request 💡 about: Suggest a new idea for the project. labels: enhancement --- ![A GIF or MEME to give some spice of the internet](url) ## Summary Brief explanation of the feature. ### Basic example If the proposal involves a new or changed API, include a basic code example. Omit this section if it's not applicable. ### Motivation Why are we doing this? What use cases does it support? What is the expected outcome? ================================================ FILE: .github/PULL_REQUEST_TEMPLATE.md ================================================ ![A GIF or MEME to give some spice of the internet](url) ## Description ## Motivation and Context [issue](https://github.com/BrainJS/brain.js/issues/###) ## How Has This Been Tested? ## Screenshots (if appropriate): ## Types of changes - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) ## Author's Checklist: - [ ] My code focuses on the main motivation and avoids scope creep. - [ ] My code passes current tests and adds new tests where possible. - [ ] My code is [SOLID](https://en.wikipedia.org/wiki/SOLID_(object-oriented_design)) and [DRY](https://en.wikipedia.org/wiki/Don%27t_repeat_yourself). - [ ] I have updated the documentation as needed. ## Reviewer's Checklist: - [ ] I kept my comments to the author positive, specific, and productive. - [ ] I tested the code and didn't find any new problems. - [ ] I think the motivation is good for the project. - [ ] I think the code works to satisfies the motivation. ================================================ FILE: .github/workflows/nodejs.yml ================================================ name: CI on: pull_request: branches: [master] push: branches: [master] jobs: build: name: Build, lint, and test on Node ${{ matrix.node }} and ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: node: [18.x, 20.x, 22.x] os: [ubuntu-latest, windows-latest, macOS-latest] exclude: # TODO: Get macOS tests to pass by upgrading to https://github.com/nodejs/node-gyp/releases - os: macos-latest node: 18.x - os: macos-latest node: 20.x - os: macos-latest node: 22.x steps: - name: Checkout repo uses: actions/checkout@v3 - name: Use Node ${{ matrix.node }} uses: actions/setup-node@v3 with: node-version: ${{ matrix.node }} cache: npm - if: startsWith(matrix.os, 'ubuntu') run: | sudo apt-get install -y build-essential libglew-dev libglu1-mesa-dev libxi-dev pkg-config ls /usr/include/c++/ # export CXXFLAGS='-include /usr/include/c++/11.2.0/limits' - name: Install deps and build (with cache) run: | touch ./dist rm package-lock.json npm i - name: Lint run: npm run lint - name: Test run: | # TODO: Remove the disabling of the following files. mv src/neural-network-gpu.end-to-end.test.ts src/neural-network-gpu.end-to-end.test.ts.DISABLED mv src/neural-network.bitwise.test.ts src/neural-network.bitwise.test.ts.DISABLED mv src/neural-network.trainopts.test.ts src/neural-network.trainopts.test.ts.DISABLED mv src/recurrent.end-to-end.test.ts src/recurrent.end-to-end.test.ts.DISABLED mv src/recurrent/gru.test.ts src/recurrent/gru.test.ts.DISABLED mv src/recurrent/lstm-time-step.end-to-end.test.ts src/recurrent/lstm-time-step.end-to-end.test.ts.DISABLED mv src/recurrent/lstm.test.ts src/recurrent/lstm.test.ts.DISABLED mv src/recurrent/rnn-time-step.test.ts src/recurrent/rnn-time-step.test.ts.DISABLED mv src/recurrent/rnn.test.ts src/recurrent/rnn.test.ts.DISABLED npm run test --ci --coverage --maxWorkers=2 - name: Build run: npm run build - name: Codecov uses: codecov/codecov-action@v3 ================================================ FILE: .gitignore ================================================ # Mac. .DS_STORE **.DS_STORE # Node. node_modules npm-debug.log # Yarn yarn.lock yarn-error.log # parcel bundler cache .cache # code coverage __coverage__ # webstorm .idea # distribution dist ================================================ FILE: .npmignore ================================================ # Mac. .DS_STORE **.DS_STORE # Node. node_modules npm-debug.log # parcel bundler cache .cache # code coverage __coverage__ test*.* .editorconfig .jshintrc .travis.yml .istanbul.yml .babelrc .idea/ .vscode/ test/ coverage/ .github/ .cache/ __coverage__ .cache .dist/ ================================================ FILE: .npmrc ================================================ registry=https://registry.npmjs.org ================================================ FILE: .nvmrc ================================================ v22.4.3 ================================================ FILE: .prettierrc ================================================ { "semi": true, "singleQuote": true, "tabWidth": 2, "trailingComma": "es5" } ================================================ FILE: .travis.yml ================================================ language: node_js node_js: - '10' - '12' cache: directories: - node_modules install: - npm ci script: - npm run lint - npm run build - npm run test sudo: false ================================================ FILE: .vscode/launch.json ================================================ { // Use IntelliSense to learn about possible attributes. // Hover to view descriptions of existing attributes. // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ { "name": "Launch via NPM", "request": "launch", "runtimeArgs": ["run-script", "debug"], "runtimeExecutable": "npm", "skipFiles": ["/**"], "type": "pwa-node" }, { "type": "node", "name": "vscode-jest-tests", "request": "launch", "args": ["--runInBand"], "cwd": "${workspaceFolder}", "console": "integratedTerminal", "internalConsoleOptions": "neverOpen", "disableOptimisticBPs": true, "program": "${workspaceFolder}/node_modules/jest/bin/jest" } ] } ================================================ FILE: LICENSE ================================================ Copyright (c) 2010-2019 Heather Arthur Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: README.md ================================================

Logo

# brain.js GPU accelerated Neural networks in JavaScript for Browsers and Node.js

GitHub [![npm](https://img.shields.io/npm/dt/brain.js.svg?style=flat-square)](https://npmjs.com/package/brain.js) [![js-standard-style](https://img.shields.io/badge/code%20style-standard-brightgreen.svg)](http://standardjs.com) [![Backers on Open Collective](https://opencollective.com/brainjs/backers/badge.svg)](#backers) [![Sponsors on Open Collective](https://opencollective.com/brainjs/sponsors/badge.svg)](#sponsors) [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/brain-js/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Slack](https://slack.bri.im/badge.svg)](https://slack.bri.im) ![CI](https://github.com/BrainJS/brain.js/workflows/CI/badge.svg) [![codecov](https://codecov.io/gh/BrainJS/brain.js/branch/master/graph/badge.svg?token=3SJIBJ1679)](https://codecov.io/gh/BrainJS/brain.js) Twitter [![NPM](https://nodei.co/npm/brain.js.png?compact=true)](https://nodei.co/npm/brain.js/)

## About `brain.js` is a GPU accelerated library for [Neural Networks](http://en.wikipedia.org/wiki/Artificial_neural_network) written in JavaScript. :bulb: This is a continuation of the [**harthur/brain**](https://github.com/harthur/brain), which is not maintained anymore. [More info](https://github.com/harthur/brain/issues/72) ## Table of Contents - [Installation and Usage](#Installation-and-Usage) - [NPM](#NPM) - [CDN](#CDN) - [Download](#Download) - [Installation note](#Installation-note) - [Building from source](#Building-from-source) - [Examples](#examples) - [More Examples](#more-examples) - [Training](#training) - [Data format](#data-format) - [For training with NeuralNetwork](#for-training-with-neuralnetwork) - [For training with `RNNTimeStep`, `LSTMTimeStep` and `GRUTimeStep`](#for-training-with-rnntimestep-lstmtimestep-and-grutimestep) - [For training with `RNN`, `LSTM` and `GRU`](#for-training-with-rnn-lstm-and-gru) - [For training with `AE`](#for-training-with-ae) - [Training Options](#training-options) - [Async Training](#async-training) - [Cross Validation](#cross-validation) - [Train Stream](#streams) - [Methods](#methods) - [train](#traintrainingdata---trainingstatus) - [run](#runinput---prediction) - [forecast](#forecastinput-count---predictions) - [Failing](#failing) - [JSON](#json) - [Standalone Function](#standalone-function) - [Options](#options) - [activation](#activation) - [hiddenLayers](#hiddenlayers) - [Streams](#streams) - [Utilities](#utilities) - [`likely`](#likely) - [`toSVG`](#toSVG) - [Neural Network Types](#neural-network-types) - [Why different Neural Network Types?](#why-different-neural-network-types) ## Installation and Usage ### NPM If you can install `brain.js` with [npm](http://npmjs.org): ```bash npm install brain.js ``` ### CDN ```html ``` ### Download [Download the latest brain.js for browser](https://unpkg.com/brain.js) ### Installation note `Brain.js` depends on a native module [`headless-gl`](https://www.npmjs.com/package/headless-gl) for GPU support. In most cases installing `brain.js` from npm should just work. However, if you run into problems, this means prebuilt binaries are not able to download from GitHub repositories and you might need to build it yourself. #### Building from source Please make sure the following dependencies are installed and up to date and then run: ```bash npm rebuild ``` ##### System dependencies ###### Mac OS X - [A supported version of Python](https://devguide.python.org/versions) - [XCode](https://developer.apple.com/xcode/) ###### Ubuntu/Debian - [A supported version of Python](https://devguide.python.org/versions) - A GNU C++ environment (available via the `build-essential` package on `apt`) - [libxi-dev](http://www.x.org/wiki/) - Working and up-to-date OpenGL drivers - [GLEW](http://glew.sourceforge.net/) - [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/) ```bash sudo apt-get install -y build-essential libglew-dev libglu1-mesa-dev libxi-dev pkg-config ``` ###### Windows - [A supported version of Python](https://devguide.python.org/versions) __See:__ https://apps.microsoft.com/store/search/python - [Microsoft Visual Studio Build Tools 2022](https://visualstudio.microsoft.com/downloads) - run in cmd: `npm config set msvs_version 2022` __Note: This no longer works in modern versions of npm.__ - run in cmd: `npm config set python python3` __Note: This no longer works in modern versions of npm.__ \* If you are using `Build Tools 2017` then run `npm config set msvs_version 2017` __Note: This no longer works in modern versions of npm.__ ## Examples Here's an example showcasing how to approximate the XOR function using `brain.js`: more info on config [here](https://github.com/BrainJS/brain.js/blob/develop/src/neural-network.js#L31). :bulb: [A fun and practical introduction to Brain.js](https://scrimba.com/g/gneuralnetworks) ```javascript // provide optional config object (or undefined). Defaults shown. const config = { binaryThresh: 0.5, hiddenLayers: [3], // array of ints for the sizes of the hidden layers in the network activation: 'sigmoid', // supported activation types: ['sigmoid', 'relu', 'leaky-relu', 'tanh'], leakyReluAlpha: 0.01, // supported for activation type 'leaky-relu' }; // create a simple feed-forward neural network with backpropagation const net = new brain.NeuralNetwork(config); net.train([ { input: [0, 0], output: [0] }, { input: [0, 1], output: [1] }, { input: [1, 0], output: [1] }, { input: [1, 1], output: [0] }, ]); const output = net.run([1, 0]); // [0.987] ``` or more info on config [here](https://github.com/BrainJS/brain.js/blob/develop/src/recurrent/rnn.js#L726). ```javascript // provide optional config object, defaults shown. const config = { inputSize: 20, inputRange: 20, hiddenLayers: [20, 20], outputSize: 20, learningRate: 0.01, decayRate: 0.999, }; // create a simple recurrent neural network const net = new brain.recurrent.RNN(config); net.train([ { input: [0, 0], output: [0] }, { input: [0, 1], output: [1] }, { input: [1, 0], output: [1] }, { input: [1, 1], output: [0] }, ]); let output = net.run([0, 0]); // [0] output = net.run([0, 1]); // [1] output = net.run([1, 0]); // [1] output = net.run([1, 1]); // [0] ``` However, there is no reason to use a neural network to figure out XOR. (-: So, here is a more involved, realistic example: [Demo: training a neural network to recognize color contrast](https://brain.js.org/). ## More Examples [Brain.js Examples Repo](https://github.com/BrainJS/brain.js-examples) You can check out this fantastic screencast, which explains how to train a simple neural network using a real-world dataset: [How to create a neural network in the browser using Brain.js](https://scrimba.com/c/c36zkcb). ## Training Use `train()` to train the network with an array of training data. The network has to be trained with all the data in bulk in one call to `train()`. More training patterns will probably take longer to train, but will usually result in a network better at classifying new patterns. ### Note Training is computationally expensive, so you should try to train the network offline (or on a Worker) and use the `toFunction()` or `toJSON()` options to plug the pre-trained network into your website. ### Data format #### For training with `NeuralNetwork` Each training pattern should have an `input` and an `output`, both of which can be either an array of numbers from `0` to `1` or a hash of numbers from `0` to `1`. For the [color contrast demo](https://brain.js.org/) it looks something like this: ```javascript const net = new brain.NeuralNetwork(); net.train([ { input: { r: 0.03, g: 0.7, b: 0.5 }, output: { black: 1 } }, { input: { r: 0.16, g: 0.09, b: 0.2 }, output: { white: 1 } }, { input: { r: 0.5, g: 0.5, b: 1.0 }, output: { white: 1 } }, ]); const output = net.run({ r: 1, g: 0.4, b: 0 }); // { white: 0.99, black: 0.002 } ``` Here's another variation of the above example. (_Note_ that input objects do not need to be similar.) ```javascript net.train([ { input: { r: 0.03, g: 0.7 }, output: { black: 1 } }, { input: { r: 0.16, b: 0.2 }, output: { white: 1 } }, { input: { r: 0.5, g: 0.5, b: 1.0 }, output: { white: 1 } }, ]); const output = net.run({ r: 1, g: 0.4, b: 0 }); // { white: 0.81, black: 0.18 } ``` #### For training with `RNNTimeStep`, `LSTMTimeStep` and `GRUTimeStep` Each training pattern can either: - Be an array of numbers - Be an array of arrays of numbers Example using an array of numbers: ```javascript const net = new brain.recurrent.LSTMTimeStep(); net.train([[1, 2, 3]]); const output = net.run([1, 2]); // 3 ``` Example using an array of arrays of numbers: ```javascript const net = new brain.recurrent.LSTMTimeStep({ inputSize: 2, hiddenLayers: [10], outputSize: 2, }); net.train([ [1, 3], [2, 2], [3, 1], ]); const output = net.run([ [1, 3], [2, 2], ]); // [3, 1] ``` #### For training with `RNN`, `LSTM` and `GRU` Each training pattern can either: - Be an array of values - Be a string - Have an `input` and an `output` - Either of which can have an array of values or a string CAUTION: When using an array of values, you can use ANY value, however, the values are represented in the neural network by a single input. So the more _distinct values_ has _the larger your input layer_. If you have a hundreds, thousands, or millions of floating point values _THIS IS NOT THE RIGHT CLASS FOR THE JOB_. Also, when deviating from strings, this gets into beta Example using direct strings: Hello World Using Brainjs ```javascript const net = new brain.recurrent.LSTM(); net.train(['I am brainjs, Hello World!']); const output = net.run('I am brainjs'); alert(output); ``` ```javascript const net = new brain.recurrent.LSTM(); net.train([ 'doe, a deer, a female deer', 'ray, a drop of golden sun', 'me, a name I call myself', ]); const output = net.run('doe'); // ', a deer, a female deer' ``` Example using strings with inputs and outputs: ```javascript const net = new brain.recurrent.LSTM(); net.train([ { input: 'I feel great about the world!', output: 'happy' }, { input: 'The world is a terrible place!', output: 'sad' }, ]); const output = net.run('I feel great about the world!'); // 'happy' ``` #### For training with `AE` Each training pattern can either: - Be an array of numbers - Be an array of arrays of numbers Training an autoencoder to compress the values of a XOR calculation: ```javascript const net = new brain.AE( { hiddenLayers: [ 5, 2, 5 ] } ); net.train([ [ 0, 0, 0 ], [ 0, 1, 1 ], [ 1, 0, 1 ], [ 1, 1, 0 ] ]); ``` Encoding/decoding: ```javascript const input = [ 0, 1, 1 ]; const encoded = net.encode(input); const decoded = net.decode(encoded); ``` Denoise noisy data: ```javascript const noisyData = [ 0, 1, 0 ]; const data = net.denoise(noisyData); ``` Test for anomalies in data samples: ```javascript const shouldBeFalse = net.includesAnomalies([0, 1, 1]); const shouldBeTrue = net.includesAnomalies([0, 1, 0]); ``` ### Training Options `train()` takes a hash of options as its second argument: ```javascript net.train(data, { // Defaults values --> expected validation iterations: 20000, // the maximum times to iterate the training data --> number greater than 0 errorThresh: 0.005, // the acceptable error percentage from training data --> number between 0 and 1 log: false, // true to use console.log, when a function is supplied it is used --> Either true or a function logPeriod: 10, // iterations between logging out --> number greater than 0 learningRate: 0.3, // scales with delta to effect training rate --> number between 0 and 1 momentum: 0.1, // scales with next layer's change value --> number between 0 and 1 callback: null, // a periodic call back that can be triggered while training --> null or function callbackPeriod: 10, // the number of iterations through the training data between callback calls --> number greater than 0 timeout: number, // the max number of milliseconds to train for --> number greater than 0. Default --> Infinity }); ``` The network will stop training whenever one of the two criteria is met: the training error has gone below the threshold (default `0.005`), or the max number of iterations (default `20000`) has been reached. By default, training will not let you know how it's doing until the end, but set `log` to `true` to get periodic updates on the current training error of the network. The training error should decrease every time. The updates will be printed to the console. If you set `log` to a function, this function will be called with the updates instead of printing to the console. However, if you want to use the values of the updates in your own output, the `callback` can be set to a function to do so instead. The learning rate is a parameter that influences how quickly the network trains. It's a number from `0` to `1`. If the learning rate is close to `0`, it will take longer to train. If the learning rate is closer to `1`, it will train faster, but training results may be constrained to a local minimum and perform badly on new data.(_Overfitting_) The default learning rate is `0.3`. The momentum is similar to learning rate, expecting a value from `0` to `1` as well, but it is multiplied against the next level's change value. The default value is `0.1` Any of these training options can be passed into the constructor or passed into the `updateTrainingOptions(opts)` method and they will be saved on the network and used during the training time. If you save your network to json, these training options are saved and restored as well (except for callback and log, callback will be forgotten and log will be restored using console.log). A boolean property called `invalidTrainOptsShouldThrow` is set to `true` by default. While the option is `true`, if you enter a training option that is outside the normal range, an error will be thrown with a message about the abnormal option. When the option is set to `false`, no error will be sent, but a message will still be sent to `console.warn` with the related information. ### Async Training `trainAsync()` takes the same arguments as train (data and options). Instead of returning the results object from training, it returns a promise that when resolved will return the training results object. Does NOT work with: * `brain.recurrent.RNN` * `brain.recurrent.GRU` * `brain.recurrent.LSTM` * `brain.recurrent.RNNTimeStep` * `brain.recurrent.GRUTimeStep` * `brain.recurrent.LSTMTimeStep` ```javascript const net = new brain.NeuralNetwork(); net .trainAsync(data, options) .then((res) => { // do something with my trained network }) .catch(handleError); ``` With multiple networks you can train in parallel like this: ```javascript const net = new brain.NeuralNetwork(); const net2 = new brain.NeuralNetwork(); const p1 = net.trainAsync(data, options); const p2 = net2.trainAsync(data, options); Promise.all([p1, p2]) .then((values) => { const res = values[0]; const res2 = values[1]; console.log( `net trained in ${res.iterations} and net2 trained in ${res2.iterations}` ); // do something super cool with my 2 trained networks }) .catch(handleError); ``` ### Cross Validation [Cross Validation]() can provide a less fragile way of training on larger data sets. The brain.js api provides Cross Validation in this example: ```js const crossValidate = new brain.CrossValidate(() => new brain.NeuralNetwork(networkOptions)); crossValidate.train(data, trainingOptions, k); //note k (or KFolds) is optional const json = crossValidate.toJSON(); // all stats in json as well as neural networks const net = crossValidate.toNeuralNetwork(); // get top performing net out of `crossValidate` // optionally later const json = crossValidate.toJSON(); const net = crossValidate.fromJSON(json); ``` Use `CrossValidate` with these classes: - `brain.NeuralNetwork` - `brain.RNNTimeStep` - `brain.LSTMTimeStep` - `brain.GRUTimeStep` An example of using cross validate can be found in [cross-validate.ts](https://github.com/BrainJS/brain.js-examples/blob/main/src/cross-validate.ts) ## Methods ### `train(trainingData)` -> trainingStatus The output of `train()` is a hash of information about how the training went: ```javascript { error: 0.0039139985510105032, // training error iterations: 406 // training iterations } ``` ### `run(input)` -> prediction Supported on classes: - `brain.NeuralNetwork` - `brain.NeuralNetworkGPU` -> All the functionality of `brain.NeuralNetwork` but, ran on GPU (via gpu.js in WebGL2, WebGL1, or fallback to CPU) - `brain.recurrent.RNN` - `brain.recurrent.LSTM` - `brain.recurrent.GRU` - `brain.recurrent.RNNTimeStep` - `brain.recurrent.LSTMTimeStep` - `brain.recurrent.GRUTimeStep` Example: ```js // feed forward const net = new brain.NeuralNetwork(); net.fromJSON(json); net.run(input); // time step const net = new brain.LSTMTimeStep(); net.fromJSON(json); net.run(input); // recurrent const net = new brain.LSTM(); net.fromJSON(json); net.run(input); ``` ### `forecast(input, count)` -> predictions Available with the following classes. Outputs a array of predictions. Predictions being a continuation of the inputs. - `brain.recurrent.RNNTimeStep` - `brain.recurrent.LSTMTimeStep` - `brain.recurrent.GRUTimeStep` Example: ```js const net = new brain.LSTMTimeStep(); net.fromJSON(json); net.forecast(input, 3); ``` ### `toJSON() -> json` Serialize neural network to json ### `fromJSON(json)` Deserialize neural network from json ## Failing If the network failed to train, the error will be above the error threshold. This could happen if the training data is too noisy (most likely), the network does not have enough hidden layers or nodes to handle the complexity of the data, or it has not been trained for enough iterations. If the training error is still something huge like `0.4` after 20000 iterations, it's a good sign that the network can't make sense of the given data. ### RNN, LSTM, or GRU Output too short or too long The instance of the net's property `maxPredictionLength` (default 100) can be set to adjust the output of the net; Example: ```js const net = new brain.recurrent.LSTM(); // later in code, after training on a few novels, write me a new one! net.maxPredictionLength = 1000000000; // Be careful! net.run('Once upon a time'); ``` ## JSON Serialize or load in the state of a trained network with JSON: ```javascript const json = net.toJSON(); net.fromJSON(json); ``` ## Standalone Function You can also get a custom standalone function from a trained network that acts just like `run()`: ```javascript const run = net.toFunction(); const output = run({ r: 1, g: 0.4, b: 0 }); console.log(run.toString()); // copy and paste! no need to import brain.js ``` ## Options `NeuralNetwork()` takes a hash of options: ```javascript const net = new brain.NeuralNetwork({ activation: 'sigmoid', // activation function hiddenLayers: [4], learningRate: 0.6, // global learning rate, useful when training using streams }); ``` ### activation This parameter lets you specify which activation function your neural network should use. There are currently four supported activation functions, **sigmoid** being the default: - [sigmoid](https://www.wikiwand.com/en/Sigmoid_function) - [relu]() - [leaky-relu]() - related option - 'leakyReluAlpha' optional number, defaults to 0.01 - [tanh](https://theclevermachine.wordpress.com/tag/tanh-function/) Here's a table (thanks, Wikipedia!) summarizing a plethora of activation functions — [Activation Function](https://www.wikiwand.com/en/Activation_function) ### hiddenLayers You can use this to specify the number of hidden layers in the network and the size of each layer. For example, if you want two hidden layers - the first with 3 nodes and the second with 4 nodes, you'd give: ```js hiddenLayers: [3, 4]; ``` By default `brain.js` uses one hidden layer with size proportionate to the size of the input array. ## Streams Use https://www.npmjs.com/package/train-stream to stream data to a NeuralNetwork ## Utilities ### `likely` ```js const likely = require('brain/likely'); const key = likely(input, net); ``` Likely example see: [simple letter detection](https://github.com/BrainJS/brain.js-examples/blob/main/src/which-letter-simple.ts) ### `toSVG` ```js ``` Renders the network topology of a feedforward network ```js document.getElementById('result').innerHTML = brain.utilities.toSVG( network, options ); ``` toSVG example see: [network rendering](https://github.com/BrainJS/brain.js-examples/blob/main/src/rendering-svg/index.html) The user interface used: ![screenshot1](https://user-images.githubusercontent.com/43925925/48969024-e526ed80-f000-11e8-85bd-e10967cfaee2.png) ## Neural Network Types - [`brain.NeuralNetwork`](src/neural-network.ts) - [Feedforward Neural Network](https://en.wikipedia.org/wiki/Feedforward_neural_network) with backpropagation - [`brain.NeuralNetworkGPU`](src/neural-network-gpu.ts) - [Feedforward Neural Network](https://en.wikipedia.org/wiki/Feedforward_neural_network) with backpropagation, GPU version - [`brain.AE`](src/autoencoder.ts) - [Autoencoder or "AE"](https://en.wikipedia.org/wiki/Autoencoder) with backpropogation and GPU support - [`brain.recurrent.RNNTimeStep`](src/recurrent/rnn-time-step.ts) - [Time Step Recurrent Neural Network or "RNN"](https://en.wikipedia.org/wiki/Recurrent_neural_network) - [`brain.recurrent.LSTMTimeStep`](src/recurrent/lstm-time-step.ts) - [Time Step Long Short Term Memory Neural Network or "LSTM"](https://en.wikipedia.org/wiki/Long_short-term_memory) - [`brain.recurrent.GRUTimeStep`](src/recurrent/gru-time-step.ts) - [Time Step Gated Recurrent Unit or "GRU"](https://en.wikipedia.org/wiki/Gated_recurrent_unit) - [`brain.recurrent.RNN`](src/recurrent/rnn.ts) - [Recurrent Neural Network or "RNN"](https://en.wikipedia.org/wiki/Recurrent_neural_network) - [`brain.recurrent.LSTM`](src/recurrent/lstm.ts) - [Long Short Term Memory Neural Network or "LSTM"](https://en.wikipedia.org/wiki/Long_short-term_memory) - [`brain.recurrent.GRU`](src/recurrent/gru.ts) - [Gated Recurrent Unit or "GRU"](https://en.wikipedia.org/wiki/Gated_recurrent_unit) - [`brain.FeedForward`](src/feed-forward.ts) - [Highly Customizable Feedforward Neural Network](https://en.wikipedia.org/wiki/Feedforward_neural_network) with backpropagation - [`brain.Recurrent`](src/recurrent.ts) - [Highly Customizable Recurrent Neural Network](https://en.wikipedia.org/wiki/Recurrent_neural_network) with backpropagation ### Why different Neural Network Types Different neural nets do different things well. For example: - A Feedforward Neural Network can classify simple things very well, but it has no memory of previous actions and has infinite variation of results. - A Time Step Recurrent Neural Network _remembers_, and can predict future values. - A Recurrent Neural Network _remembers_, and has a finite set of results. ## Get Involved ### W3C machine learning standardization process If you are a developer or if you just care about how ML API should look like - please take a part and join W3C community and share your opinions or simply support opinions you like or agree with. Brain.js is a widely adopted open source machine learning library in the javascript world. There are several reasons for it, but most notable is **simplicity of usage while not sacrificing performance**. We would like to keep it also simple to learn, simple to use and performant when it comes to W3C standard. We think that current brain.js API is quite close to what we could expect to become a standard. And since supporting doesn't require much effort and still can make a huge difference feel free to join W3C community group and support us with brain.js like API. Get involved into W3C machine learning ongoing standardization process [here](https://www.w3.org/community/webmachinelearning/). You can also join our open discussion about standardization [here](https://github.com/BrainJS/brain.js/issues/337). ## Issues If you have an issue, either a bug or a feature you think would benefit your project let us know and we will do our best. Create issues [here](https://github.com/BrainJS/brain.js/issues) and follow the template. ### brain.js.org Source for `brain.js.org` is available at [Brain.js.org Repository](https://github.com/BrainJS/brain.js.org). Built using awesome `vue.js` & `bulma`. Contributions are always welcome. ## Contributors This project exists thanks to all the people who contribute. [[Contribute](/.github/CONTRIBUTING.md)]. ## Backers Thank you to all our backers! 🙏 [[Become a backer](https://opencollective.com/brainjs#backer)] ## Sponsors Support this project by becoming a sponsor. Your logo will show up here with a link to your website. [[Become a sponsor](https://opencollective.com/brainjs#sponsor)] ================================================ FILE: jest.config.json ================================================ { "collectCoverage": false, "collectCoverageFrom": ["src/**/*"], "coverageDirectory": "__coverage__", "coverageProvider": "v8", "globalSetup": "", "globals": { "ts-jest": { "tsconfig": "tsconfig.json" } }, "preset": "ts-jest/presets/js-with-ts", "setupFiles": [], "testEnvironment": "node" } ================================================ FILE: package.json ================================================ { "author": "Heather Arthur ", "browser": "dist/browser.js", "bugs": { "url": "https://github.com/brainjs/brain.js/issues" }, "description": "Neural networks in JavaScript", "dependencies": { "thaw.js": "^2.1.4" }, "peerDependencies": { "gpu.js": "^2.16.0" }, "devDependencies": { "rimraf": "^6.0.0", "@babel/preset-typescript": "^7.13.0", "@rollup/plugin-commonjs": "^15.1.0", "@rollup/plugin-json": "^4.1.0", "@rollup/plugin-node-resolve": "^9.0.0", "@rollup/plugin-typescript": "^8.2.0", "@types/eslint": "^7.2.4", "@types/eslint-plugin-prettier": "^3.1.0", "@types/jest": "^27.0.2", "@types/node": "^14.14.2", "@types/prettier": "^2.1.5", "@types/rollup-plugin-node-builtins": "^2.1.1", "@types/rollup-plugin-node-globals": "^1.4.0", "@typescript-eslint/eslint-plugin": "^4.5.0", "@typescript-eslint/parser": "^4.5.0", "acorn": "^8.0.4", "c8": "^7.3.4", "codecov": "^3.8.0", "del-cli": "^5.0.0", "eslint": "^7.12.0", "eslint-config-prettier": "^6.14.0", "eslint-config-standard": "^15.0.0", "eslint-config-standard-with-typescript": "^19.0.1", "eslint-plugin-import": "^2.22.1", "eslint-plugin-jest": "^24.1.0", "eslint-plugin-node": "^11.1.0", "eslint-plugin-prettier": "^3.1.4", "eslint-plugin-promise": "^4.2.1", "eslint-plugin-standard": "^4.0.2", "fast-xml-parser": "^3.17.4", "gpu-mock.js": "^1.3.1", "gpu.js": "^2.15.2", "husky": "^4.3.0", "jest": "^26.6.1", "npm-run-all": "^4.1.5", "prettier": "^2.1.2", "rollup": "^2.39.1", "rollup-plugin-terser": "^7.0.2", "ts-jest": "^26.4.2", "ts-node": "^9.1.1", "typescript": "^4.0.3" }, "engines": { "node": ">=8.6.x" }, "files": [ "dist/" ], "homepage": "https://github.com/brainjs/brain.js#readme", "husky": { "hooks": { "pre-commit": "npm run lint" } }, "keywords": [ "ai", "artificial-intelligence", "brain", "brainjs", "brain.js", "feed forward", "neural network", "classifier", "neural", "network", "neural-networks", "machine-learning", "synapse", "recurrent", "long short term memory", "gated recurrent unit", "time series", "time step", "prediction", "rnn", "lstm", "gru" ], "license": "MIT", "main": "dist/index.js", "name": "brain.js", "repository": { "type": "git", "url": "git+ssh://git@github.com/brainjs/brain.js.git" }, "scripts": { "build": "run-p build:**", "build:browser": "rollup -c rollup.config.browser.js", "build:node": "rollup -c rollup.config.js", "build:ts": "tsc --declaration --emitDeclarationOnly --declarationMap", "coverage": "jest --coverage --coverage-provider v8 && codecov", "dist": "npm run build", "lint": "run-p lint:**", "lint:eslint": "eslint --fix --ext .js,.ts src", "lint:typecheck": "tsc --noEmit", "test": "jest", "watch": "run-p watch:**", "watch:node": "rollup -c rollup.config.js -w", "watch:test": "jest --watch", "clean": "rimraf ./dist", "prepare": "npm run clean && npm run build" }, "types": "dist/", "unpkg": "dist/browser.js", "version": "2.0.0-beta.24" } ================================================ FILE: rollup.config.browser.js ================================================ import commonjs from '@rollup/plugin-commonjs'; import json from '@rollup/plugin-json'; import resolve from '@rollup/plugin-node-resolve'; import typescript from '@rollup/plugin-typescript'; import { fileURLToPath } from 'node:url'; const name = 'brain'; const extensions = ['.js', '.json', '.node', '.ts']; const file = 'dist/browser.js'; export default { input: './src/index.ts', // Specify here external modules which you don't want to include in your bundle (for instance: 'lodash', 'moment' etc.) // https://rollupjs.org/guide/en#external-e-external external: [ // brain js already uses gpu.js as peer dependencies so it shouldn't be like this fileURLToPath( new URL('./node_modules/gpu.js/src/index.js', import.meta.url) ), ], plugins: [ // Allows node_modules resolution resolve({ preferBuiltins: false, browser: true, extensions, }), // allow json importing json(), // Allow bundling cjs modules. Rollup doesn't understand cjs commonjs(), // Compile TypeScript/JavaScript files typescript(), ], output: [ { file, format: 'umd', sourcemap: true, globals: { 'gpu.js': `GPU`, }, name, }, ], }; ================================================ FILE: rollup.config.js ================================================ import commonjs from '@rollup/plugin-commonjs'; import json from '@rollup/plugin-json'; import resolve from '@rollup/plugin-node-resolve'; import typescript from '@rollup/plugin-typescript'; import * as pkg from './package.json'; const extensions = ['.js', '.json', '.node', '.ts']; export default { input: './src/index.ts', // Specify here external modules which you don't want to include in your bundle (for instance: 'lodash', 'moment' etc.) // https://rollupjs.org/guide/en#external-e-external external: [ 'gpu.js' ], plugins: [ // Allows node_modules resolution resolve({ preferBuiltins: true, browser: false, extensions, }), // allow json importing json(), // Allow bundling cjs modules. Rollup doesn't understand cjs commonjs(), // Compile TypeScript/JavaScript files typescript(), ], output: [ { file: pkg.main, format: 'cjs', sourcemap: true, }, ], }; ================================================ FILE: src/README.md ================================================ # Brain.js Source Files Welcome! Brain.js aims to be: * Javascript and Node based * an easy, well thought out api * simple enough to teach a child * performant enough for enterprise and hobbyists alike The industry wants you to think neural networks are complex, they are not. ================================================ FILE: src/activation/README.md ================================================ # [Activation](https://en.wikipedia.org/wiki/Activation_function) ## What is activation? Activation can be thought of as a sort of one way compressor for numerical values. Activation is useful for: * representing a wide array of numbers in a very small number * throwing away values the network does not care about * focusing on values the network does care about For every activation, there is usually a widely accepted means of measuring it, this term the activation's derivative function. For simplicity, it is referred to as our "measure" function. ## Programming structure ### Each activation has at least the following two exported functions * `activate` - the activation type's mathematical function * The term `activate` specifically correlates to the activation function type * `measure` - the derivative to measure the activation For programmatic, simplicity, and practicality, we namespace the structure as: ``` relu (activation type) |-activate |-measure ``` ================================================ FILE: src/activation/index.test.ts ================================================ import * as activation from '../activation'; import * as leakyRelu from '../activation/leaky-relu'; import * as relu from '../activation/relu'; import * as sigmoid from '../activation/sigmoid'; import * as tanh from '../activation/tanh'; describe('activation', () => { test('it has all expected activations', () => { expect(activation.leakyRelu).toBe(leakyRelu); expect(activation.relu).toBe(relu); expect(activation.sigmoid).toBe(sigmoid); expect(activation.tanh).toBe(tanh); }); }); ================================================ FILE: src/activation/index.ts ================================================ export * as relu from './relu'; export * as sigmoid from './sigmoid'; export * as tanh from './tanh'; export * as leakyRelu from './leaky-relu'; ================================================ FILE: src/activation/leaky-relu.test.ts ================================================ import * as leakyRelu from './leaky-relu'; describe('leakyRelu', () => { describe('.active()', () => { describe('when weight is greater than 0', () => { it('returns weight', () => { expect(leakyRelu.activate(1)).toBe(1); }); }); describe('when value is equal to 0', () => { it('returns value * 0.01', () => { expect(leakyRelu.activate(0)).toBe(0); }); }); describe('when value is less than 0', () => { it('returns value * 0.01', () => { expect(leakyRelu.activate(-1)).toBe(-0.01); }); }); }); describe('.measure()', () => { describe('when weight is greater than 0', () => { it('returns error', () => { const error = 0.1; expect(leakyRelu.measure(1, error)).toBe(error); }); }); describe('when weight is equal to 0', () => { it('returns error', () => { const error = 0.1; expect(leakyRelu.measure(1, error)).toBe(error); }); }); describe('when weight is less than 0', () => { it('returns error', () => { expect(leakyRelu.measure(-1, 1)).toBe(0.01); }); }); }); }); ================================================ FILE: src/activation/leaky-relu.ts ================================================ /** * Leaky Relu Activation, aka Leaky Rectified Linear Unit Activation * @description https://en.wikipedia.org/wiki/Rectifier_(neural_networks) */ export function activate(weight: number): number { return weight > 0 ? weight : 0.01 * weight; } /** * Leaky Relu derivative */ export function measure(weight: number, error: number): number { return weight > 0 ? error : 0.01 * error; } ================================================ FILE: src/activation/relu.test.ts ================================================ import * as relu from './relu'; describe('relu', () => { describe('.active()', () => { describe('when weight is greater than 0', () => { it('returns weight', () => { expect(relu.activate(99)).toBe(99); }); }); describe('when value is equal to 0', () => { it('returns 0', () => { expect(relu.activate(0)).toBe(0); }); }); describe('when value is less than 0', () => { it('returns 0', () => { expect(relu.activate(0)).toBe(0); }); }); }); describe('.measure()', () => { describe('when weight is greater than 0', () => { it('returns error', () => { const error = 0.1; expect(relu.measure(1, error)).toBe(error); }); }); describe('when weight is equal to 0', () => { it('returns error', () => { const error = 0.1; expect(relu.measure(1, error)).toBe(error); }); }); describe('when weight is less than 0', () => { it('returns 0', () => { expect(relu.measure(-1, 1)).toBe(0); }); }); }); }); ================================================ FILE: src/activation/relu.ts ================================================ /** * Relu Activation, aka Rectified Linear Unit Activation * @description https://en.wikipedia.org/wiki/Rectifier_(neural_networks) */ export function activate(weight: number): number { return Math.max(0, weight); } /** * Relu derivative */ export function measure(weight: number, delta: number): number { if (weight <= 0) { return 0; } return delta; } ================================================ FILE: src/activation/sigmoid.test.ts ================================================ import * as sigmoid from './sigmoid'; describe('sigmoid', () => { describe('.active()', () => { it('matches for value 1', () => { expect(sigmoid.activate(1).toFixed(5)).toBe('0.73106'); }); }); describe('.measure()', () => { it('matches for value .7', () => { expect(sigmoid.measure(0.7, 0.5).toFixed(5)).toBe('0.10500'); }); }); }); ================================================ FILE: src/activation/sigmoid.ts ================================================ /** * sigmoid activation */ export function activate(value: number): number { return 1 / (1 + Math.exp(-value)); } /** * sigmoid derivative */ export function measure(weight: number, error: number): number { return weight * (1 - weight) * error; } ================================================ FILE: src/activation/tanh.test.ts ================================================ import * as tanh from './tanh'; describe('tanh', () => { describe('.active()', () => { it('matches for value 1', () => { expect(tanh.activate(1).toFixed(5)).toBe(Math.tanh(1).toFixed(5)); }); }); describe('.measure()', () => { it('matches for value .7', () => { expect(tanh.measure(0.7, 0.5).toFixed(5)).toBe('0.25500'); }); }); }); ================================================ FILE: src/activation/tanh.ts ================================================ /** * Hyperbolic tan */ export function activate(weight: number): number { return Math.tanh(weight); } /** * @description grad for z = tanh(x) is (1 - z^2) */ export function measure(weight: number, error: number): number { return (1 - weight * weight) * error; } ================================================ FILE: src/autoencoder.test.ts ================================================ import AE from './autoencoder'; const trainingData = [ [0, 0, 0], [0, 1, 1], [1, 0, 1], [1, 1, 0], ]; const xornet = new AE({ decodedSize: 3, hiddenLayers: [5, 2, 5], }); const errorThresh = 0.011; const result = xornet.train(trainingData, { iterations: 100000, errorThresh, }); test('denoise a data sample', async () => { expect(result.error).toBeLessThanOrEqual(errorThresh); function xor(...args: number[]) { return Math.round(xornet.denoise(args)[2]); } const run1 = xor(0, 0, 0); const run2 = xor(0, 1, 1); const run3 = xor(1, 0, 1); const run4 = xor(1, 1, 0); expect(run1).toBe(0); expect(run2).toBe(1); expect(run3).toBe(1); expect(run4).toBe(0); }); test('encode and decode a data sample', async () => { expect(result.error).toBeLessThanOrEqual(errorThresh); const run1$input = [0, 0, 0]; const run1$encoded = xornet.encode(run1$input); const run1$decoded = xornet.decode(run1$encoded); const run2$input = [0, 1, 1]; const run2$encoded = xornet.encode(run2$input); const run2$decoded = xornet.decode(run2$encoded); for (let i = 0; i < 3; i++) expect(Math.round(run1$decoded[i])).toBe(run1$input[i]); for (let i = 0; i < 3; i++) expect(Math.round(run2$decoded[i])).toBe(run2$input[i]); }); test('test a data sample for anomalies', async () => { expect(result.error).toBeLessThanOrEqual(errorThresh); function includesAnomalies(...args: number[]) { expect(xornet.likelyIncludesAnomalies(args)).toBe(false); } includesAnomalies(0, 0, 0); includesAnomalies(0, 1, 1); includesAnomalies(1, 0, 1); includesAnomalies(1, 1, 0); }); ================================================ FILE: src/autoencoder.ts ================================================ import { KernelOutput, Texture, TextureArrayOutput } from 'gpu.js'; import { IJSONLayer, INeuralNetworkData, INeuralNetworkDatum, INeuralNetworkTrainOptions, } from './neural-network'; import { INeuralNetworkGPUOptions, NeuralNetworkGPU, } from './neural-network-gpu'; import { INeuralNetworkState } from './neural-network-types'; import { UntrainedNeuralNetworkError } from './errors/untrained-neural-network-error'; export interface IAEOptions { binaryThresh: number; decodedSize: number; hiddenLayers: number[]; } /** * An autoencoder learns to compress input data down to relevant features and reconstruct input data from its compressed representation. */ export class AE< DecodedData extends INeuralNetworkData, EncodedData extends INeuralNetworkData > { private decoder?: NeuralNetworkGPU; private readonly denoiser: NeuralNetworkGPU; constructor(options?: Partial) { // Create default options for the autoencoder. options ??= {}; // Create default options for the autoencoder's denoiser subnet. const denoiserOptions: Partial = {}; // Inherit the binary threshold of the parent autoencoder. denoiserOptions.binaryThresh = options.binaryThresh; // Inherit the hidden layers of the parent autoencoder. denoiserOptions.hiddenLayers = options.hiddenLayers; // Define the denoiser subnet's input and output sizes. if (options.decodedSize) denoiserOptions.inputSize = denoiserOptions.outputSize = options.decodedSize; // Create the denoiser subnet of the autoencoder. this.denoiser = new NeuralNetworkGPU(options); } /** * Denoise input data, removing any anomalies from the data. * @param {DecodedData} input * @returns {DecodedData} */ denoise(input: DecodedData): DecodedData { // Run the input through the generic denoiser. // This isn't the best denoiser implementation, but it's efficient. // Efficiency is important here because training should focus on // optimizing for feature extraction as quickly as possible rather than // denoising and anomaly detection; there are other specialized topologies // better suited for these tasks anyways, many of which can be implemented // by using an autoencoder. return this.denoiser.run(input); } /** * Decode `EncodedData` into an approximation of its original form. * * @param {EncodedData} input * @returns {DecodedData} */ decode(input: EncodedData): DecodedData { // If the decoder has not been trained yet, throw an error. if (!this.decoder) throw new UntrainedNeuralNetworkError(this); // Decode the encoded input. return this.decoder.run(input); } /** * Encode data to extract features, reduce dimensionality, etc. * * @param {DecodedData} input * @returns {EncodedData} */ encode(input: DecodedData): EncodedData { // If the decoder has not been trained yet, throw an error. if (!this.denoiser) throw new UntrainedNeuralNetworkError(this); // Process the input. this.denoiser.run(input); // Get the auto-encoded input. let encodedInput: TextureArrayOutput = this .encodedLayer as TextureArrayOutput; // If the encoded input is a `Texture`, convert it into an `Array`. if (encodedInput instanceof Texture) encodedInput = encodedInput.toArray(); else encodedInput = encodedInput.slice(0); // Return the encoded input. return encodedInput as EncodedData; } /** * Test whether or not a data sample likely contains anomalies. * If anomalies are likely present in the sample, returns `true`. * Otherwise, returns `false`. * * @param {DecodedData} input * @returns {boolean} */ likelyIncludesAnomalies(input: DecodedData, anomalyThreshold = 0.2): boolean { // Create the anomaly vector. const anomalies: number[] = []; // Attempt to denoise the input. const denoised = this.denoise(input); // Calculate the anomaly vector. for (let i = 0; i < (input.length ?? 0); i++) { anomalies[i] = Math.abs( (input as number[])[i] - (denoised as number[])[i] ); } // Calculate the sum of all anomalies within the vector. const sum = anomalies.reduce( (previousValue, value) => previousValue + value ); // Calculate the mean anomaly. const mean = sum / (input as number[]).length; // Return whether or not the mean anomaly rate is greater than the anomaly threshold. return mean > anomalyThreshold; } /** * Train the auto encoder. * * @param {DecodedData[]} data * @param {Partial} options * @returns {INeuralNetworkState} */ train( data: DecodedData[], options?: Partial ): INeuralNetworkState { const preprocessedData: Array, Partial >> = []; for (const datum of data) { preprocessedData.push({ input: datum, output: datum }); } const results = this.denoiser.train(preprocessedData, options); this.decoder = this.createDecoder(); return results; } /** * Create a new decoder from the trained denoiser. * * @returns {NeuralNetworkGPU} */ private createDecoder() { const json = this.denoiser.toJSON(); const layers: IJSONLayer[] = []; const sizes: number[] = []; for (let i = this.encodedLayerIndex; i < this.denoiser.sizes.length; i++) { layers.push(json.layers[i]); sizes.push(json.sizes[i]); } json.layers = layers; json.sizes = sizes; json.options.inputSize = json.sizes[0]; const decoder = new NeuralNetworkGPU().fromJSON(json); return (decoder as unknown) as NeuralNetworkGPU; } /** * Get the layer containing the encoded representation. */ private get encodedLayer(): KernelOutput { return this.denoiser.outputs[this.encodedLayerIndex]; } /** * Get the offset of the encoded layer. */ private get encodedLayerIndex(): number { return Math.round(this.denoiser.outputs.length * 0.5) - 1; } } export default AE; ================================================ FILE: src/cross-validate.test.ts ================================================ import CrossValidate from './cross-validate'; import { INeuralNetworkOptions, INeuralNetworkTrainOptions, NeuralNetwork, } from './neural-network'; import { LSTMTimeStep } from './recurrent/lstm-time-step'; describe('CrossValidate', () => { describe('.train()', () => { class FakeNN extends NeuralNetwork { constructor( options: Partial< INeuralNetworkOptions & INeuralNetworkTrainOptions > = {} ) { super(options); this.options.hiddenLayers = [1, 2, 3]; } train(data: Array<{ input: number[]; output: number[] }>) { this.prepTraining(data, this.trainOpts); return { iterations: 10, error: 0.05, }; } } it('throws exception when training set is too small', () => { const xorTrainingData = [{ input: [0, 1], output: [1] }]; const net = new CrossValidate(() => new FakeNN()); expect(() => { net.train(xorTrainingData); }).toThrow('Training set size is too small for 1 k folds of 4'); }); it('handles successful training', () => { class SpyFakeNN extends FakeNN { setActivation() { this.runInput = (inputs: Float32Array): Float32Array => { if (inputs[0] === 0 && inputs[1] === 1) return Float32Array.from([1]); if (inputs[0] === 0 && inputs[1] === 0) return Float32Array.from([0]); if (inputs[0] === 1 && inputs[1] === 1) return Float32Array.from([0]); if (inputs[0] === 1 && inputs[1] === 0) return Float32Array.from([1]); throw new Error('unknown input'); }; } } const xorTrainingData = [ { input: [0, 1], output: [1] }, { input: [0, 0], output: [0] }, { input: [1, 1], output: [0] }, { input: [1, 0], output: [1] }, { input: [0, 1], output: [1] }, { input: [0, 0], output: [0] }, { input: [1, 1], output: [0] }, { input: [1, 0], output: [1] }, ]; const net = new CrossValidate( () => new SpyFakeNN({ inputSize: 1, hiddenLayers: [10], outputSize: 1, }) ); net.shuffleArray = (input) => input; const result = net.train(xorTrainingData); if (!CrossValidate.isBinaryResults(result)) { fail('expected binary stats but did not find binary stats'); } expect(result.avgs.iterations).toBe(10); expect(result.avgs.error).toBe(0.05); expect(result.avgs.testTime >= 0).toBeTruthy(); expect(result.avgs.trainTime >= 0).toBeTruthy(); expect(result.stats.total).toBe(8); expect(result.stats.truePos).toBe(4); expect(result.stats.trueNeg).toBe(4); expect(result.stats.falsePos).toBe(0); expect(result.stats.falseNeg).toBe(0); expect(result.stats.precision).toBe(1); expect(result.stats.accuracy).toBe(1); expect(result.stats.testSize).toBe(2); expect(result.stats.trainSize).toBe(6); expect(result.sets.length).toBe(4); for (let i = 0; i < result.sets.length; i++) { const set = result.sets[0]; expect(set.accuracy).toBe(1); expect(set.error).toBe(0.05); expect(set.truePos >= 1 || set.trueNeg >= 1).toBeTruthy(); expect(set.falseNeg).toBe(0); expect(set.falsePos).toBe(0); expect(set.precision).toBe(1); expect(set.recall).toBe(1); expect(set.testTime >= 0).toBeTruthy(); expect(set.trainTime >= 0).toBeTruthy(); expect(set.total).toBe(2); expect(set.network).not.toBeFalsy(); expect(set.misclasses).toEqual([]); } }); it('handles unsuccessful training', () => { class SpyFakeNN extends FakeNN { setActivation() { this.runInput = (inputs: Float32Array): Float32Array => { if (inputs[0] === 0 && inputs[1] === 1) return Float32Array.from([0]); if (inputs[0] === 0 && inputs[1] === 0) return Float32Array.from([1]); if (inputs[0] === 1 && inputs[1] === 1) return Float32Array.from([1]); if (inputs[0] === 1 && inputs[1] === 0) return Float32Array.from([0]); throw new Error('unknown input'); }; } } const xorTrainingData = [ { input: [0, 1], output: [1] }, { input: [0, 0], output: [0] }, { input: [1, 1], output: [0] }, { input: [1, 0], output: [1] }, { input: [0, 1], output: [1] }, { input: [0, 0], output: [0] }, { input: [1, 1], output: [0] }, { input: [1, 0], output: [1] }, ]; const net = new CrossValidate(() => new SpyFakeNN()); net.shuffleArray = (input) => input; const result = net.train(xorTrainingData); if (!CrossValidate.isBinaryResults(result)) { fail('expected binary stats but did not find binary stats'); } expect(result.avgs.iterations).toBe(10); expect(result.avgs.error).toBe(0.05); expect(result.avgs.testTime >= 0).toBeTruthy(); expect(result.avgs.trainTime >= 0).toBeTruthy(); expect(result.stats.total).toBe(8); expect(result.stats.truePos).toBe(0); expect(result.stats.trueNeg).toBe(0); expect(result.stats.falsePos).toBe(4); expect(result.stats.falseNeg).toBe(4); expect(result.stats.precision).toBe(0); expect(result.stats.accuracy).toBe(0); expect(result.stats.testSize).toBe(2); expect(result.stats.trainSize).toBe(6); expect(result.sets.length).toBe(4); for (let i = 0; i < result.sets.length; i++) { const set = result.sets[0]; expect(set.accuracy).toBe(0); expect(set.error).toBe(0.05); expect(set.truePos).toBe(0); expect(set.trueNeg).toBe(0); expect(set.falseNeg >= 1 || set.falsePos >= 1).toBeTruthy(); expect(set.precision).toBe(0); expect(set.recall).toBe(0); expect(set.testTime >= 0).toBeTruthy(); expect(set.trainTime >= 0).toBeTruthy(); expect(set.total).toBe(2); expect(set.network).not.toBeFalsy(); interface T { misclasses: [ { input: number[]; output: number[]; actual: number; expected: number; } ]; } const misclasses = ((set as unknown) as T).misclasses; expect(misclasses.length > 0).toBeTruthy(); expect(misclasses[0].hasOwnProperty('input')).toBeTruthy(); expect(misclasses[0].input.length).toBeTruthy(); expect( xorTrainingData.filter((v) => v.input === misclasses[0].input) ).toBeTruthy(); expect( xorTrainingData.filter((v) => v.output === misclasses[0].output) ).toBeTruthy(); expect( misclasses[0].actual === 0 || misclasses[0].actual === 1 ).toBeTruthy(); expect( misclasses[0].expected === 0 || misclasses[0].expected === 1 ).toBeTruthy(); } }); }); describe('.toJSON()', () => { it('returns from this.json', () => { const cv = new CrossValidate(() => new NeuralNetwork()); const json = cv.json; expect(cv.toJSON()?.avgs?.error).toBe(0); expect(cv.toJSON()).toBe(json); }); }); describe('.fromJSON()', () => { class FakeNN extends NeuralNetwork {} it("creates a new instance of constructor from argument's sets.error", () => { const cv = new CrossValidate(() => new FakeNN(options)); const options = { inputSize: 1, hiddenLayers: [10], outputSize: 1 }; const details = { trainTime: 0, testTime: 0, total: 0, iterations: 0, misclasses: [0], learningRate: 0, hiddenLayers: [0], }; const bestNetwork = new FakeNN(options); bestNetwork.initialize(); const worstNetwork = new FakeNN(options); worstNetwork.initialize(); const midNetwork = new FakeNN(options); midNetwork.initialize(); const worstNetworkJSON = worstNetwork.toJSON(); const midNetworkJSON = midNetwork.toJSON(); const bestNetworkJSON = bestNetwork.toJSON(); const net = cv.fromJSON({ avgs: { trainTime: 1, testTime: 2, iterations: 3, error: 4, }, stats: { total: 5, testSize: 6, trainSize: 7, }, sets: [ { error: 10, network: worstNetworkJSON, ...details, }, { error: 5, network: midNetworkJSON, ...details, }, { error: 1, network: bestNetworkJSON, ...details, }, ], }); expect(net.toJSON()).toEqual(bestNetwork.toJSON()); }); }); describe('.toNeuralNetwork()', () => { class FakeNN extends NeuralNetwork {} it('creates a new instance of constructor from top .json sets.error', () => { const cv = new CrossValidate(() => new FakeNN()); const details = { trainTime: 0, testTime: 0, total: 0, iterations: 0, misclasses: [0], learningRate: 0, hiddenLayers: [0], }; const options = { inputSize: 10, hiddenLayers: [10], outputSize: 7, }; const bestNet = new FakeNN(options); bestNet.initialize(); const worstNet = new FakeNN(options); worstNet.initialize(); const midNet = new FakeNN(options); midNet.initialize(); cv.json = { sets: [ { error: 10, network: worstNet.toJSON(), ...details }, { error: 5, network: midNet.toJSON(), ...details }, { error: 1, network: bestNet.toJSON(), ...details }, ], avgs: { trainTime: 0, testTime: 0, iterations: 0, error: 0 }, stats: { total: 0, testSize: 0, trainSize: 0, }, }; const net = cv.toNeuralNetwork(); expect(net.toJSON()).toEqual(bestNet.toJSON()); }); }); describe('NeuralNetwork compatibility', () => { it('handles simple xor example', () => { const xorTrainingData = [ { input: [0, 1], output: [1] }, { input: [0, 0], output: [0] }, { input: [1, 1], output: [0] }, { input: [1, 0], output: [1] }, { input: [0, 1], output: [1] }, { input: [0, 0], output: [0] }, { input: [1, 1], output: [0] }, { input: [1, 0], output: [1] }, ]; const net = new CrossValidate( () => new NeuralNetwork() ); const result = net.train(xorTrainingData); expect(result.avgs.error >= 0).toBeTruthy(); expect(result.avgs.iterations >= 0).toBeTruthy(); expect(result.avgs.testTime >= 0).toBeTruthy(); expect(result.avgs.trainTime >= 0).toBeTruthy(); expect(result.stats.testSize >= 0).toBeTruthy(); expect(result.stats.trainSize >= 0).toBeTruthy(); expect(result.stats.total >= 0).toBeTruthy(); }); }); describe('RNNTimeStep compatibility', () => { it('can average error for array,array, counting forwards and backwards', () => { const trainingData = [ [0.1, 0.2, 0.3, 0.4, 0.5], [0.2, 0.3, 0.4, 0.5, 0.6], [0.3, 0.4, 0.5, 0.6, 0.7], [0.4, 0.5, 0.6, 0.7, 0.8], [0.5, 0.6, 0.7, 0.8, 0.9], [0.5, 0.4, 0.3, 0.2, 0.1], [0.6, 0.5, 0.4, 0.3, 0.2], [0.7, 0.6, 0.5, 0.4, 0.3], [0.8, 0.7, 0.6, 0.5, 0.4], [0.9, 0.8, 0.7, 0.6, 0.5], ]; const cv = new CrossValidate( () => new LSTMTimeStep({ inputSize: 1, hiddenLayers: [10], outputSize: 1, }) ); const result = cv.train(trainingData, { iterations: 10 }); expect(!isNaN(result.avgs.error)).toBeTruthy(); }); }); }); ================================================ FILE: src/cross-validate.ts ================================================ import { INeuralNetworkBinaryTestResult, INeuralNetworkState, INeuralNetworkTestResult, } from './neural-network-types'; export type InitClassifier< TrainOptsType, JsonType, DatumType > = () => IClassifier; export interface IClassifier { trainOpts: TrainOptsType; toJSON: () => JsonType; fromJSON: (json: JsonType) => this; train: ( data: DatumType[], options?: Partial ) => INeuralNetworkState; test: ( data: DatumType[] ) => INeuralNetworkTestResult | INeuralNetworkBinaryTestResult; initialize: () => void; } export type ICrossValidateJSON = | ICrossValidateStats | ICrossValidateBinaryStats; export interface ICrossValidateStatsAverages { trainTime: number; testTime: number; iterations: number; error: number; } export interface ICrossValidateStats { avgs: ICrossValidateStatsAverages; stats: ICrossValidateStatsResultStats; sets: Array>; } export interface ICrossValidateBinaryStats { avgs: ICrossValidateStatsAverages; stats: ICrossValidateStatsResultBinaryStats; sets: Array>; } export interface ICrossValidateStatsResultStats { total: number; testSize: number; trainSize: number; } export interface ICrossValidateStatsResultBinaryStats extends ICrossValidateStatsResultStats { total: number; truePos: number; trueNeg: number; falsePos: number; falseNeg: number; precision: number; recall: number; accuracy: number; } export interface ICrossValidationTestPartitionResults extends INeuralNetworkTestResult { trainTime: number; testTime: number; iterations: number; network: JsonType; total: number; } export type ICrossValidationTestPartitionBinaryResults< JsonType > = INeuralNetworkBinaryTestResult & ICrossValidationTestPartitionResults; export default class CrossValidate< InitClassifierType extends InitClassifier< ReturnType['trainOpts'], ReturnType['toJSON']>, Parameters['train']>[0][0] > > { initClassifier: InitClassifierType; json: ICrossValidateJSON< ReturnType['toJSON']> > = { avgs: { error: 0, iterations: 0, testTime: 0, trainTime: 0, }, stats: { total: 0, testSize: 0, trainSize: 0, }, sets: [], }; constructor(initClassifier: InitClassifierType) { this.initClassifier = initClassifier; } testPartition( trainOpts: Parameters['train']>[1], trainSet: Parameters['train']>[0], testSet: Parameters['train']>[0] ): | ICrossValidationTestPartitionResults< ReturnType['toJSON']> > | ICrossValidationTestPartitionBinaryResults< ReturnType['toJSON']> > { const classifier = this.initClassifier(); const beginTrain = Date.now(); const trainingStats = classifier.train(trainSet, trainOpts); const beginTest = Date.now(); const testStats: | INeuralNetworkTestResult | INeuralNetworkBinaryTestResult = classifier.test(testSet); const endTest = Date.now(); return { ...testStats, trainTime: beginTest - beginTrain, testTime: endTest - beginTest, iterations: trainingStats.iterations, error: trainingStats.error, total: testStats.total, network: (classifier as { toJSON: () => ReturnType['toJSON']>; }).toJSON(), }; } /** * Randomize array element order in-place. * Using Durstenfeld shuffle algorithm. * source: http://stackoverflow.com/a/12646864/1324039 */ shuffleArray(array: K[]): K[] { for (let i = array.length - 1; i > 0; i--) { const j = Math.floor(Math.random() * (i + 1)); const temp = array[i]; array[i] = array[j]; array[j] = temp; } return array; } static isBinaryStats = ( stats: ICrossValidateStatsResultStats | ICrossValidateStatsResultBinaryStats ): stats is ICrossValidateStatsResultBinaryStats => { return ( (stats as ICrossValidateStatsResultBinaryStats).accuracy !== undefined ); }; static isBinaryResults = ( stats: ICrossValidateStats | ICrossValidateBinaryStats ): stats is ICrossValidateBinaryStats => (stats as ICrossValidateBinaryStats).stats.accuracy !== undefined; static isBinaryPartitionResults = ( stats: | ICrossValidationTestPartitionResults | ICrossValidationTestPartitionBinaryResults ): stats is ICrossValidationTestPartitionBinaryResults => (stats as ICrossValidationTestPartitionBinaryResults).accuracy !== undefined; train( data: Array['train']>[0][0]>, trainOpts: Partial< Parameters['train']>[1] > = {}, k = 4 ): ICrossValidateStats['toJSON']> { if (data.length < k) { throw new Error( `Training set size is too small for ${data.length} k folds of ${k}` ); } this.shuffleArray(data); const size = data.length / k; const avgs: ICrossValidateStatsAverages = { trainTime: 0, testTime: 0, iterations: 0, error: 0, }; const stats: | ICrossValidateStatsResultStats | ICrossValidateStatsResultBinaryStats = { total: 0, testSize: 0, trainSize: 0, }; const binaryStats: ICrossValidateStatsResultBinaryStats = { total: 0, testSize: 0, trainSize: 0, truePos: 0, trueNeg: 0, falsePos: 0, falseNeg: 0, precision: 0, recall: 0, accuracy: 0, }; const results = []; let isBinary = null; for (let i = 0; i < k; i++) { const dclone = data.slice(0); const testSet = dclone.splice(i * size, size); const trainSet = dclone; const result = this.testPartition(trainOpts, trainSet, testSet); if (isBinary === null) { isBinary = result.hasOwnProperty('falseNeg') && result.hasOwnProperty('falsePos') && result.hasOwnProperty('trueNeg') && result.hasOwnProperty('truePos'); if (isBinary) { Object.assign(stats, binaryStats); } } avgs.iterations += result.iterations; avgs.testTime += result.testTime; avgs.trainTime += result.trainTime; avgs.error += result.error; stats.total += result.total; if ( CrossValidate.isBinaryStats(stats) && CrossValidate.isBinaryPartitionResults(result) ) { stats.accuracy += result.accuracy; stats.falseNeg += result.falseNeg; stats.falsePos += result.falsePos; stats.precision += result.precision; stats.recall += result.recall; stats.trueNeg += result.trueNeg; stats.truePos += result.truePos; } results.push(result); } avgs.error /= k; avgs.iterations /= k; avgs.testTime /= k; avgs.trainTime /= k; if (CrossValidate.isBinaryStats(stats)) { stats.precision = stats.truePos / (stats.truePos + stats.falsePos); stats.recall = stats.truePos / (stats.truePos + stats.falseNeg); stats.accuracy = (stats.trueNeg + stats.truePos) / stats.total; } stats.testSize = size; stats.trainSize = data.length - size; this.json = { avgs: avgs, stats: stats, sets: results, }; return this.json as ICrossValidateStats< ReturnType['toJSON'] >; } toNeuralNetwork(): ReturnType { return this.fromJSON(this.json); } toJSON(): ICrossValidateJSON< ReturnType['toJSON']> > | null { return this.json; } fromJSON( crossValidateJson: ICrossValidateJSON< ReturnType['toJSON']> > ): ReturnType { const winningJSON: | ICrossValidationTestPartitionResults< ReturnType['toJSON']> > | ICrossValidationTestPartitionBinaryResults< ReturnType['toJSON']> > = (crossValidateJson as ICrossValidateStats< ReturnType['toJSON']> >).sets.reduce((prev, cur) => (prev.error < cur.error ? prev : cur)); return (this.initClassifier() as ReturnType).fromJSON( winningJSON.network ); } } ================================================ FILE: src/errors/untrained-neural-network-error.ts ================================================ export class UntrainedNeuralNetworkError< T extends { constructor: { name: string } } > extends Error { constructor(neuralNetwork: T) { super( `Cannot run a ${neuralNetwork.constructor.name} before it is trained.` ); } } ================================================ FILE: src/estimator/mean-squared-error.ts ================================================ import { IKernelRunShortcut, IKernelFunctionThis } from 'gpu.js'; import { makeKernel } from '../utilities/kernel'; interface mse2dThis extends IKernelFunctionThis { constants: { height: number; width: number; length: number }; } /** * 2D Mean Squared Error */ export function mse2d( this: mse2dThis, errors: Array<[number, number]> ): number { let sum = 0; for (let y = 0; y < this.constants.height; y++) { for (let x = 0; x < this.constants.width; x++) { sum += errors[y][x] ** 2; } } return sum / this.constants.length; } export class MeanSquaredError { /** Calculate the mean squared error given an array of errors */ calculate: IKernelRunShortcut; /** Returns the sum of absolute values of previuous error and previous layer errors */ addAbsolute: IKernelRunShortcut; /** Adds two erros */ add: IKernelRunShortcut; /** Returns the ratio of sum of errors and length (ie the average) */ divide: IKernelRunShortcut; constructor({ width, height }: { width: number; height: number }) { this.calculate = makeKernel(mse2d, { output: [1], constants: { width, height, length: width * height, }, immutable: true, }); this.addAbsolute = makeKernel( function (prevError: number[], prevLayerErrors: number[][]) { return prevError[0] + Math.abs(prevLayerErrors[0][0]); }, { output: [1], immutable: true, } ); this.add = makeKernel( function (value1: number[], value2: number[]) { return value1[0] + value2[0]; }, { output: [1], immutable: true, } ); this.divide = makeKernel( function (length: number, mseSum: number[]) { const value = mseSum[0]; if (value > 0) { return value / length; } return 0; }, { output: [1], immutable: true, } ); } } ================================================ FILE: src/feed-forward.end-to-end.test.ts ================================================ import { GPU } from 'gpu.js'; import { NeuralNetwork } from './neural-network'; import { FeedForward } from './feed-forward'; import { input, output, target, Target, Sigmoid, arthurFeedForward, ILayer, ILayerSettings, feedForward as feedForwardLayer, } from './layer'; import { momentumRootMeanSquaredPropagation } from './praxis'; import { zeros2D } from './utilities/zeros-2d'; import { setup, teardown } from './utilities/kernel'; import { mockPraxis, xorTrainingData } from './test-utils'; import { IPraxis } from './praxis/base-praxis'; /* eslint-disable no-multi-assign */ describe('FeedForward Class: End to End', () => { beforeEach(() => { setup( new GPU({ mode: 'cpu', }) ); }); afterEach(() => { teardown(); }); describe('when configured like NeuralNetwork', () => { function setupTwinXORNetworks(useDecimals: boolean) { const standardNet = new NeuralNetwork(); const ffNet = new FeedForward({ inputLayer: () => input({ height: 2, id: 'input' }), hiddenLayers: [ (inputLayer) => arthurFeedForward({ height: 3 }, inputLayer), (inputLayer) => arthurFeedForward({ height: 1 }, inputLayer), ], outputLayer: (inputLayer) => target({ height: 1, id: 'output' }, inputLayer), }); ffNet.initialize(); standardNet.train([{ input: [1, 1], output: [1] }], { iterations: 1, }); // set both nets exactly the same, then train them once, and compare const ffNetLayers = ffNet.layers as ILayer[]; const biasLayers = ffNetLayers.filter((l) => l.id === 'biases'); const weightLayers = ffNetLayers.filter((l) => l.id === 'weights'); const sigmoidLayers = ffNetLayers.filter((l) => l instanceof Sigmoid); const targetLayer = ffNetLayers[ffNetLayers.length - 1]; // Use whole numbers to better test accuracy // set biases const standardNetBiases = standardNet.biases; const biasLayers0Weights = biasLayers[0].weights as number[][]; expect(standardNetBiases[1].length).toBe(3); standardNetBiases[1][0] = biasLayers0Weights[0][0] = useDecimals ? 0.5 : 5; standardNetBiases[1][1] = biasLayers0Weights[1][0] = useDecimals ? 0.7 : 7; standardNetBiases[1][2] = biasLayers0Weights[2][0] = useDecimals ? 0.2 : 2; const biasLayers1Weights = biasLayers[1].weights as number[][]; expect(standardNetBiases[2].length).toBe(1); standardNetBiases[2][0] = biasLayers1Weights[0][0] = useDecimals ? 0.12 : 12; // set weights const standardNetWeights = standardNet.weights; const weightLayers0Weights = weightLayers[0].weights as number[][]; expect(standardNetWeights[1].length).toBe(3); expect(standardNetWeights[1][0].length).toBe(2); standardNetWeights[1][0][0] = weightLayers0Weights[0][0] = useDecimals ? 0.5 : 5; standardNetWeights[1][0][1] = weightLayers0Weights[0][1] = useDecimals ? 0.1 : 10; expect(standardNetWeights[1][1].length).toBe(2); standardNetWeights[1][1][0] = weightLayers0Weights[1][0] = useDecimals ? 0.3 : 3; standardNetWeights[1][1][1] = weightLayers0Weights[1][1] = useDecimals ? 0.1 : 1; expect(standardNetWeights[1][2].length).toBe(2); standardNetWeights[1][2][0] = weightLayers0Weights[2][0] = useDecimals ? 0.8 : 8; standardNetWeights[1][2][1] = weightLayers0Weights[2][1] = useDecimals ? 0.4 : 4; const weightLayers1Weights = weightLayers[1].weights as number[][]; expect(standardNetWeights[2].length).toBe(1); expect(standardNetWeights[2][0].length).toBe(3); standardNetWeights[2][0][0] = weightLayers1Weights[0][0] = useDecimals ? 0.2 : 2; standardNetWeights[2][0][1] = weightLayers1Weights[0][1] = useDecimals ? 0.6 : 6; standardNetWeights[2][0][2] = weightLayers1Weights[0][2] = useDecimals ? 0.3 : 3; return { ffNet, standardNet, sigmoidLayers, targetLayer, }; } describe('prediction', () => { test('it matches NeuralNetworks.deltas & NeuralNetworks.errors for 2 inputs, 3 hidden neurons, and 1 output', () => { const { standardNet, ffNet, sigmoidLayers, targetLayer, } = setupTwinXORNetworks(true); // learning deviates, which we'll test elsewhere, for the time being, just don't learn standardNet.adjustWeights = () => {}; ffNet.adjustWeights = () => {}; // retrain with these new weights, only ffNet needs reinforce, otherwise, values are lost standardNet.train( [ { input: new Float32Array([0.9, 0.8]), output: new Float32Array([0.5]), }, ], { iterations: 1, } ); ffNet.train( [ { input: new Float32Array([0.9, 0.8]), output: new Float32Array([0.5]), }, ], { iterations: 1, } ); // test only the sigmoid layers and target layers, as that is the final equation location per layer // Also, NeuralNetwork uses a negative value, while FeedForward uses a positive one const sigmoidLayers0InputLayerDeltas = (sigmoidLayers[0] .inputLayer as ILayer).deltas as number[][]; const standardNetDeltas = standardNet.deltas; expect(-sigmoidLayers0InputLayerDeltas[0][0]).not.toEqual(0); expect(-sigmoidLayers0InputLayerDeltas[0][0]).toEqual( standardNetDeltas[1][0] ); expect(-sigmoidLayers0InputLayerDeltas[1][0]).not.toEqual(0); expect(-sigmoidLayers0InputLayerDeltas[1][0]).toBeCloseTo( standardNetDeltas[1][1] ); expect(-sigmoidLayers0InputLayerDeltas[2][0]).not.toEqual(0); expect(-sigmoidLayers0InputLayerDeltas[2][0]).toEqual( standardNetDeltas[1][2] ); const sigmoidLayers1InputLayerDeltas = (sigmoidLayers[1] .inputLayer as ILayer).deltas as number[][]; expect(-sigmoidLayers1InputLayerDeltas[0][0]).not.toEqual(0); expect(-sigmoidLayers1InputLayerDeltas[0][0]).toEqual( standardNetDeltas[2][0] ); const targetLayerInputLayerDeltas = (targetLayer.inputLayer as ILayer) .deltas as number[][]; const standardNetErrors = standardNet.errors; expect(-targetLayerInputLayerDeltas[0][0]).not.toEqual(0); expect(-targetLayerInputLayerDeltas[0][0]).toEqual( standardNetErrors[2][0] ); }); }); describe('comparison', () => { test('it matches NeuralNetwork.outputs for 2 inputs, 3 hidden neurons, and 1 output', () => { const { standardNet, ffNet, sigmoidLayers, targetLayer, } = setupTwinXORNetworks(true); // learning deviates, which we'll test elsewhere, for the time being, just don't learn standardNet.adjustWeights = function () {}; ffNet.adjustWeights = function () {}; // retrain with these new weights, only ffNet needs reinforce, otherwise, values are lost standardNet.train([{ input: [0.9, 0.8], output: [0.3] }], { iterations: 1, }); ffNet.train([{ input: [0.9, 0.8], output: [0.3] }], { iterations: 1, }); // test only the sigmoid layers, as that is the final equation location per layer const sigmoidLayers0Weights = sigmoidLayers[0].weights as number[][]; const standardNetOutputs = standardNet.outputs; expect(sigmoidLayers0Weights[0][0]).not.toEqual(0); expect(sigmoidLayers0Weights[0][0]).toEqual(standardNetOutputs[1][0]); expect(sigmoidLayers0Weights[1][0]).not.toEqual(0); expect(sigmoidLayers0Weights[1][0]).toEqual(standardNetOutputs[1][1]); expect(sigmoidLayers0Weights[2][0]).not.toEqual(0); expect(sigmoidLayers0Weights[2][0]).toEqual(standardNetOutputs[1][2]); const sigmoidLayers1Weights = sigmoidLayers[1].weights as number[][]; expect(sigmoidLayers1Weights[0][0]).not.toEqual(0); expect(sigmoidLayers1Weights[0][0]).toEqual(standardNetOutputs[2][0]); const targetLayerWeights = targetLayer.weights as number[][]; expect(targetLayerWeights[0][0]).not.toEqual(0); expect(targetLayerWeights[0][0]).toEqual(standardNetOutputs[2][0]); }); }); describe('learn', () => { test('is the same value for 2 inputs, 3 hidden neurons, and 1 output', () => { const { standardNet, ffNet, sigmoidLayers, targetLayer, } = setupTwinXORNetworks(true); const sigmoidLayers0WeightsBeforeTraining = sigmoidLayers[0] .weights as number[][]; expect(sigmoidLayers0WeightsBeforeTraining[0][0]).toEqual(0); expect(sigmoidLayers0WeightsBeforeTraining[1][0]).toEqual(0); expect(sigmoidLayers0WeightsBeforeTraining[2][0]).toEqual(0); expect(sigmoidLayers0WeightsBeforeTraining[0][0]).toEqual(0); // retrain with these new weights, only ffNet needs reinforce, otherwise, values are lost standardNet.train([{ input: [0.9, 0.8], output: [0.3] }], { iterations: 1, }); ffNet.train([{ input: [0.9, 0.8], output: [0.3] }], { iterations: 1, }); // test only the sigmoid layers, as that is the final equation location per layer const sigmoidLayers0WeightsAfterTraining = sigmoidLayers[0] .weights as number[][]; const standardNetOutputs = standardNet.outputs; expect(sigmoidLayers0WeightsAfterTraining[0][0]).not.toEqual(0); expect(sigmoidLayers0WeightsAfterTraining[0][0]).toEqual( standardNetOutputs[1][0] ); expect(sigmoidLayers0WeightsAfterTraining[1][0]).not.toEqual(0); expect(sigmoidLayers0WeightsAfterTraining[1][0]).toEqual( standardNetOutputs[1][1] ); expect(sigmoidLayers0WeightsAfterTraining[2][0]).not.toEqual(0); expect(sigmoidLayers0WeightsAfterTraining[2][0]).toEqual( standardNetOutputs[1][2] ); const sigmoidLayers1Weights = sigmoidLayers[1].weights as number[][]; expect(sigmoidLayers1Weights[0][0]).not.toEqual(0); expect(sigmoidLayers1Weights[0][0]).toEqual(standardNetOutputs[2][0]); const targetLayerWeights = targetLayer.weights as number[][]; expect(targetLayerWeights[0][0]).not.toEqual(0); expect(targetLayerWeights[0][0]).toEqual(standardNetOutputs[2][0]); }); }); }); describe('.runInput()', () => { test('outputs a number', () => { const net = new FeedForward({ inputLayer: () => input({ width: 1, height: 1 }), hiddenLayers: [ (inputLayer) => feedForwardLayer({ width: 1, height: 1 }, inputLayer), ], outputLayer: (inputLayer) => output({ width: 1, height: 1 }, inputLayer), }); net.initialize(); const result = net.runInput([[1]]) as number[][]; expect(typeof result[0][0] === 'number').toBeTruthy(); }); }); describe('.train()', () => { function testOutputsSmaller() { const net = new FeedForward({ inputLayer: () => input({ height: 2 }), hiddenLayers: [ (inputLayer) => feedForwardLayer({ height: 3 }, inputLayer), (inputLayer) => feedForwardLayer({ height: 1 }, inputLayer), ], outputLayer: (inputLayer) => target({ height: 1 }, inputLayer), }); const errors: number[] = []; net.train(xorTrainingData, { iterations: 10, callbackPeriod: 1, errorCheckInterval: 1, callback: (info) => errors.push(info.error), }); expect( errors.reduce((prev, cur) => prev && typeof cur === 'number', true) ).toBeTruthy(); expect(errors[0]).toBeGreaterThan(errors[errors.length - 1]); } function testCanLearnXOR() { // const errors: number[] = []; const net = new FeedForward({ initPraxis: (layer: ILayer): IPraxis => { switch (layer.id) { case 'biases': return momentumRootMeanSquaredPropagation(layer, { decayRate: 0.29, }); case 'weights': return momentumRootMeanSquaredPropagation(layer, { decayRate: 0.29, }); default: return mockPraxis(layer); } }, inputLayer: () => input({ height: 2 }), hiddenLayers: [ (inputLayer) => feedForwardLayer({ height: 3 }, inputLayer), (inputLayer) => feedForwardLayer({ height: 1 }, inputLayer), ], outputLayer: (inputLayer) => target({ height: 1 }, inputLayer), }); net.train(xorTrainingData, { callbackPeriod: 1, errorCheckInterval: 200, callback: (info) => { if (info.iterations % 200 === 0) { // errors.push(info.error); } }, }); // eslint-disable-next-line @typescript-eslint/no-unused-vars const result1 = net.run([0, 0]); // eslint-disable-next-line @typescript-eslint/no-unused-vars const result2 = net.run([0, 1]); // eslint-disable-next-line @typescript-eslint/no-unused-vars const result3 = net.run([1, 0]); // eslint-disable-next-line @typescript-eslint/no-unused-vars const result4 = net.run([1, 1]); // TODO: this should be easier than result[0][0] https://github.com/BrainJS/brain.js/issues/439 // expect(result1[0][0]).toBeLessThan(0.2); // expect(result2[0][0]).toBeGreaterThan(0.8); // expect(result3[0][0]).toBeGreaterThan(0.8); // expect(result4[0][0]).toBeLessThan(0.2); // expect(errors[errors.length - 1]).toBeLessThan(0.1); // expect(errors.length).toBeLessThan(net.trainOpts.iterations); } describe('on CPU', () => { test('outputs a number that is smaller than when it started', () => { testOutputsSmaller(); }); test('can learn xor', () => { testCanLearnXOR(); }); }); describe('on GPU', () => { if (!GPU.isGPUSupported) return; beforeEach(() => { setup(new GPU({ mode: 'gpu' })); }); afterEach(() => { teardown(); }); test('outputs a number that is smaller than when it started', () => { testOutputsSmaller(); }); test('can learn xor', () => { testCanLearnXOR(); }); }); }); describe('.trainAsync()', () => { it('can be used to train XOR', async () => { const net = new FeedForward({ inputLayer: () => input({ height: 2 }), hiddenLayers: [ (inputLayer) => feedForwardLayer({ height: 3 }, inputLayer), (inputLayer) => feedForwardLayer({ height: 1 }, inputLayer), ], outputLayer: (inputLayer) => target({ height: 1 }, inputLayer), }); const errors: number[] = []; const result = await net.trainAsync(xorTrainingData, { iterations: 10, callbackPeriod: 1, errorCheckInterval: 1, callback: (info) => errors.push(info.error), }); expect(result.error).toBeLessThan(1); expect(result.iterations).toBe(10); }); }); describe('._calculateDeltas()', () => { test('populates deltas from output to input', () => { class SuperOutput extends Target { constructor(settings: ILayerSettings, inputLayer: ILayer) { super(settings, inputLayer); this.deltas = zeros2D(this.width, this.height); this.inputLayer = inputLayer; } } const net = new FeedForward({ inputLayer: () => input({ width: 1, height: 1 }), hiddenLayers: [ (inputLayer) => feedForwardLayer({ width: 1, height: 1 }, inputLayer), ], outputLayer: (inputLayer) => new SuperOutput({ width: 1, height: 1 }, inputLayer), }); net.initialize(); const layers: ILayer[] = net.layers as ILayer[]; layers[0].weights = [[1]]; layers.forEach((layerLayer) => { (layerLayer.deltas as number[][]).forEach((row) => { row.forEach((delta) => { expect(delta).toBe(0); }); }); }); net.runInput([[1]]); net._calculateDeltas([[1]]); layers.forEach((l) => { (l.deltas as number[][]).forEach((row) => { row.forEach((delta) => { expect(delta === 0).toBeFalsy(); }); }); }); }); }); }); ================================================ FILE: src/feed-forward.ts ================================================ import { IKernelFunctionThis, KernelOutput, Texture } from 'gpu.js'; import { MeanSquaredError } from './estimator/mean-squared-error'; import { ILayer, ILayerJSON } from './layer'; import { Model } from './layer/types'; import { InputOutputValue, INumberArray, INumberHash, lookup } from './lookup'; import * as praxis from './praxis'; import { IPraxis, IPraxisSettings } from './praxis/base-praxis'; import { flattenLayers } from './utilities/flatten-layers'; import { makeKernel, release } from './utilities/kernel'; import { layerFromJSON } from './utilities/layer-from-json'; import { LookupTable } from './utilities/lookup-table'; import { Thaw } from 'thaw.js'; export interface IFeedForwardTrainingData< InputType extends InputOutputValue | KernelOutput = number[] | Float32Array, OutputType extends InputOutputValue | KernelOutput = number[] | Float32Array > { input: InputType; output: OutputType; } export interface IFeedForwardNormalizedTrainingData { input: Float32Array; output: Float32Array; } export interface IFeedForwardGPUTrainingData { input: KernelOutput; output: KernelOutput; } export interface ITrainingStatus { iterations: number; error: number; } export type Log = (status: string) => void; export type FeedForwardCallback = (status: ITrainingStatus) => void; export interface IFeedForwardTrainingOptions { iterations?: number; errorThresh?: number; log?: boolean | Log; logPeriod?: number; learningRate?: number; callback?: FeedForwardCallback; callbackPeriod?: number; errorCheckInterval?: number; timeout?: number; } export interface IFeedForwardOptions { learningRate?: number; binaryThresh?: number; hiddenLayers?: Array<(inputLayer: ILayer, layerIndex: number) => ILayer>; inputLayer?: () => ILayer; outputLayer?: (inputLayer: ILayer, index: number) => ILayer; praxisOpts?: Partial; initPraxis?: ( layerTemplate: ILayer, settings: Partial ) => IPraxis; praxis?: IPraxis; // JSON layers?: ILayer[]; inputLayerIndex?: number; outputLayerIndex?: number; sizes?: number[]; } export interface IFeedForwardPreppedTrainingData { status: ITrainingStatus; preparedData: IFeedForwardGPUTrainingData[]; endTime: number; } export const defaults: IFeedForwardOptions = { learningRate: 0.3, binaryThresh: 0.5, initPraxis: ( layerTemplate: ILayer, settings: Partial ): IPraxis => praxis.momentumRootMeanSquaredPropagation( layerTemplate, layerTemplate.settings.praxisOpts ?? settings ), }; export const trainDefaults: IFeedForwardTrainingOptions = { iterations: 20000, errorThresh: 0.005, log: false, logPeriod: 10, learningRate: 0.3, callbackPeriod: 10, errorCheckInterval: 100, timeout: Infinity, }; export interface IFeedForwardJSON { type: string; sizes: number[]; layers: ILayerJSON[]; inputLayerIndex: number; outputLayerIndex: number; } export class FeedForward< InputType extends InputOutputValue | KernelOutput = number[] | Float32Array, OutputType extends InputOutputValue | KernelOutput = number[] | Float32Array > { static _validateTrainingOptions( options: Partial ): void { const { iterations, errorThresh, log, logPeriod, learningRate, callback, callbackPeriod, timeout, } = options; interface IValidation { [optionName: string]: () => boolean; } const validations: IValidation = { iterations: () => typeof iterations === 'number' && iterations > 0, errorThresh: () => typeof errorThresh === 'number' && errorThresh > 0 && errorThresh < 1, log: () => typeof log === 'function' || typeof log === 'boolean', logPeriod: () => typeof logPeriod === 'number' && logPeriod > 0, learningRate: () => typeof learningRate === 'number' && learningRate > 0 && learningRate < 1, callback: () => typeof callback === 'function' || callback === null, callbackPeriod: () => typeof callbackPeriod === 'number' && callbackPeriod > 0, timeout: () => typeof timeout === 'number' && timeout > 0, }; Object.keys(trainDefaults).forEach((key: string): void => { if (validations.hasOwnProperty(key) && !validations[key]()) { const val = options[key as keyof IFeedForwardTrainingOptions]; throw new Error( `[${key}, ${( val ?? 'undefined' ).toString()}] is out of normal training range, your network will probably not train.` ); } }); } /** * if a method is passed in method is used * if false passed in nothing is logged */ _setLogMethod(log: Log | undefined | boolean): void { if (typeof log === 'function') { this.trainOpts.log = log; } else if (log) { // eslint-disable-next-line this.trainOpts.log = console.log; } else { this.trainOpts.log = false; } } _updateTrainingOptions(opts: Partial): void { this.trainOpts = { ...trainDefaults, ...this.trainOpts, ...opts }; FeedForward._validateTrainingOptions(this.trainOpts); this._setLogMethod(opts.log ?? this.trainOpts.log); const { callback, callbackPeriod, errorCheckInterval } = this.trainOpts; if (callback && callbackPeriod !== errorCheckInterval) { console.warn( `options.callbackPeriod with value of ${( callbackPeriod ?? 'undefined' ).toString()} does not match options.errorCheckInterval with value of ${( errorCheckInterval ?? 'undefined' ).toString()}, if logging error, it will repeat. These values may need to match` ); } } trainOpts: Partial = {}; options: IFeedForwardOptions; layers: ILayer[] | null = null; _inputLayer: ILayer | null = null; _hiddenLayers: ILayer[] | null = null; _outputLayer: ILayer | null = null; _model: ILayer[] | null = null; meanSquaredError: MeanSquaredError | null = null; inputLookup: INumberHash | null = null; inputLookupLength: number | null = null; outputLookup: INumberHash | null = null; outputLookupLength: number | null = null; constructor(options: IFeedForwardOptions = {}) { this.options = { ...defaults, ...options }; this._updateTrainingOptions({ ...trainDefaults, ...options, }); } _connectOptionsLayers(): ILayer[] { const { inputLayerIndex, outputLayerIndex, layers } = this.options; if (!layers) throw new Error('this.options.layers in unexpected state'); if (typeof inputLayerIndex !== 'number') throw new Error('inputLayerIndex not a number'); if (typeof outputLayerIndex !== 'number') throw new Error('inputLayerIndex not a number'); const inputLayer = layers[inputLayerIndex]; if (!inputLayer) { throw new Error('inputLayer not found in this.options.layers'); } const outputLayer = layers[outputLayerIndex]; if (!outputLayer) { throw new Error('outputLayer not found in this.options.layers'); } this._inputLayer = inputLayer; this._hiddenLayers = layers.slice( inputLayerIndex, outputLayerIndex - inputLayerIndex ); this._outputLayer = outputLayer; return layers; } _connectNewLayers(): ILayer[] { const { inputLayer, outputLayer } = this.options; if (!inputLayer) throw new Error('inputLayer not defined'); const layers: ILayer[] = []; this._inputLayer = inputLayer(); const hiddenLayers = this._connectHiddenLayers(this._inputLayer); if (!outputLayer) throw new Error('outputLayer not defined'); this._outputLayer = outputLayer( hiddenLayers[hiddenLayers.length - 1], hiddenLayers.length ); layers.push(this._inputLayer); layers.push(...hiddenLayers); layers.push(this._outputLayer); return flattenLayers(layers); } _connectHiddenLayers(previousLayer: ILayer): ILayer[] { this._hiddenLayers = []; const result: ILayer[] = []; const { hiddenLayers } = this.options; if (!hiddenLayers) throw new Error('hiddenLayers not defined'); for (let i = 0; i < hiddenLayers.length; i++) { const hiddenLayer = hiddenLayers[i](previousLayer, i); result.push(hiddenLayer); this._hiddenLayers.push(hiddenLayer); previousLayer = hiddenLayer; } return result; } initialize(): void { this.layers = this.options.layers ? this._connectOptionsLayers() : this._connectNewLayers(); this.initializeLayers(this.layers); this._model = this.layers.filter((l) => l instanceof Model); } initializeLayers(layers: ILayer[]): void { for (let i = 0; i < layers.length; i++) { const layer = layers[i]; // TODO: optimize for when training or just running layer.setupKernels(true); if ( layer instanceof Model && layer.praxis === null && typeof this.options.initPraxis === 'function' ) { layer.praxis = this.options.initPraxis( layer, layer.settings.praxisOpts ?? this.options.praxisOpts ?? {} ); layer.praxis.setupKernels(); } } const lastLayer = layers[layers.length - 1]; this.meanSquaredError = new MeanSquaredError({ width: lastLayer.width, height: lastLayer.height, }); } run(input: InputType): OutputType { let typeSafeInput: INumberArray | KernelOutput; if (Array.isArray(input) || (input as Float32Array).buffer) { typeSafeInput = input as INumberArray; } else { if (this.inputLookup) { typeSafeInput = lookup.toArray( this.inputLookup, input as INumberHash, this.inputLookupLength as number ); } else { throw new Error('input is incompatible with net'); } } let output = this.runInput(typeSafeInput as KernelOutput); if (output instanceof Texture) { output = output.toArray(); } if (this.outputLookup) { return lookup.toObject( this.outputLookup, output as number[] ) as OutputType; } return output as OutputType; } runInput(input: KernelOutput): KernelOutput { if (!this.layers) throw new Error('not initialized'); this.layers[0].predict(input); for (let i = 1; i < this.layers.length; i++) { this.layers[i].predict(); } return this.layers[this.layers.length - 1].weights as KernelOutput; } train( data: Array>, options: Partial = {} ): ITrainingStatus { const { preparedData, status, endTime } = this._prepTraining(data, options); let continueTicking = true; const calculateError = (): number => this._calculateTrainingError(preparedData); const trainPatterns = (): void => this._trainPatterns(preparedData); while (continueTicking) { continueTicking = this._trainingTick( status, endTime, calculateError, trainPatterns ); } return status; } async trainAsync( data: Array>, options: Partial = {} ): Promise { const { preparedData, status, endTime } = this._prepTraining(data, options); return await new Promise((resolve, reject) => { try { const calculateError = (): number => this._calculateTrainingError(preparedData); const trainPatterns = (): void => this._trainPatterns(preparedData); const thawedTrain: Thaw = new Thaw( new Array(this.trainOpts.iterations), { delay: true, each: () => this._trainingTick( status, endTime, calculateError, trainPatterns ) || thawedTrain.stop(), done: () => resolve(status), } ); thawedTrain.tick(); } catch (trainError) { reject(trainError); } }); } _trainingTick( status: ITrainingStatus, endTime: number, calculateError: () => number, trainPatterns: () => void ): boolean { const { trainOpts } = this; if ( status.iterations >= (trainOpts.iterations as number) || status.error <= (trainOpts.errorThresh as number) || Date.now() >= endTime ) { return false; } if ( typeof trainOpts.log === 'function' && status.iterations % (trainOpts.logPeriod as number) === 0 ) { status.error = calculateError(); trainOpts.log( `iterations: ${status.iterations}, training error: ${status.error}` ); } else if ( status.iterations % (trainOpts.errorCheckInterval as number) === 0 ) { status.error = calculateError(); } else { trainPatterns(); } if ( trainOpts.callback && status.iterations % (trainOpts.callbackPeriod as number) === 0 ) { trainOpts.callback(Object.assign(status)); } status.iterations++; return true; } _prepTraining( data: Array>, options: Partial ): IFeedForwardPreppedTrainingData { this._updateTrainingOptions(options); const formattedData = this.formatData(data); const endTime = this.trainOpts.timeout ? Date.now() + this.trainOpts.timeout : 0; const status = { error: 1, iterations: 0, }; this.verifyIsInitialized(); return { preparedData: this.transferData(formattedData), status, endTime, }; } verifyIsInitialized(): void { if (!this._model) { this.initialize(); } } _calculateTrainingError(preparedData: IFeedForwardGPUTrainingData[]): number { let sum: Float32Array | KernelOutput = new Float32Array([0]); const meanSquaredError = this.meanSquaredError as MeanSquaredError; for (let i = 0; i < preparedData.length; ++i) { const prevSum = sum; const error = this._trainPattern( preparedData[i].input, preparedData[i].output, true ) as number; sum = meanSquaredError.add(sum, error); release(error); release(prevSum); } const result = meanSquaredError.divide(preparedData.length, sum); release(sum); if (result instanceof Texture) { const resultArray: number[] = result.toArray() as number[]; release(result); return resultArray[0]; } return (result as number[])[0]; } /** * @param data * @private */ _trainPatterns(data: IFeedForwardGPUTrainingData[]): void { for (let i = 0; i < data.length; ++i) { this._trainPattern(data[i].input, data[i].output, false); } } _trainPattern( input: KernelOutput, target: KernelOutput, logErrorRate: boolean ): KernelOutput | null { // forward propagate this.runInput(input); // back propagate this._calculateDeltas(target); this.adjustWeights(); if (logErrorRate) { if (!this._outputLayer?.errors) { throw new Error('outputLayer.errors not defined'); } return (this.meanSquaredError as MeanSquaredError).calculate( this._outputLayer.errors ); } return null; } _calculateDeltas(target: KernelOutput): void { const layers = this.layers as ILayer[]; for (let i = layers.length - 1; i > -1; i--) { layers[i].compare(target); } } /** * */ adjustWeights(): void { const _model = this._model as ILayer[]; for (let i = 0; i < _model.length; i++) { _model[i].learn(this.trainOpts.learningRate as number); } } /** * * @param data * @returns {*} */ formatData( data: | Array> | IFeedForwardTrainingData ): IFeedForwardNormalizedTrainingData[] { if (!Array.isArray(data)) { // turn stream datum into array const tmp = []; tmp.push(data); data = tmp; } // turn sparse hash input into arrays with 0s as filler const inputDatumCheck = data[0].input; let formattedData: Array>; if ( Array.isArray(data) && !Array.isArray(inputDatumCheck) && !(inputDatumCheck instanceof Float32Array) ) { if (!this.inputLookup) { const lookupTable = new LookupTable(data, 'input'); this.inputLookup = lookupTable.table; this.inputLookupLength = lookupTable.length; } formattedData = data.map((datumParam): Partial< IFeedForwardNormalizedTrainingData > => { const array = lookup.toArray( this.inputLookup as INumberHash, datumParam.input as INumberHash, this.inputLookupLength as number ); return { input: array }; }, this); } else { formattedData = data as typeof formattedData; } const outputDatumCheck = data[0].output; if ( !Array.isArray(outputDatumCheck) && !(outputDatumCheck instanceof Float32Array) ) { if (!this.outputLookup) { const lookupTable = new LookupTable(data, 'output'); this.outputLookup = lookupTable.table; this.outputLookupLength = lookupTable.length; } formattedData = data.map( (datumParam, index): IFeedForwardNormalizedTrainingData => { const array = lookup.toArray( this.outputLookup as INumberHash, datumParam.output as INumberHash, this.inputLookupLength as number ); return { input: formattedData[index].input as Float32Array, output: array, }; }, this ); } return formattedData as IFeedForwardNormalizedTrainingData[]; } transferData( formattedData: IFeedForwardNormalizedTrainingData[] ): IFeedForwardGPUTrainingData[] { const transferredData = new Array(formattedData.length); const transferInput = makeKernel( function (value: number[]): number { return value[this.thread.x]; }, { output: [formattedData[0].input.length], immutable: true, } ); const transferOutput = makeKernel( function (this: IKernelFunctionThis, value: number[]): number { return value[this.thread.x]; }, { output: [formattedData[0].output.length], immutable: true, } ); for (let i = 0; i < formattedData.length; i++) { const formattedDatum = formattedData[i]; transferredData[i] = { input: transferInput(formattedDatum.input), output: transferOutput(formattedDatum.output), }; } return transferredData; } /** * * @param data * @returns { * { * error: number, * misclasses: Array * } * } */ test(): void { throw new Error(`${this.constructor.name}-test is not yet implemented`); } /** * */ toJSON(): IFeedForwardJSON { if (!this.layers) { this.initialize(); } if ( !this._model || !this.layers || !this._inputLayer || !this._hiddenLayers || !this._outputLayer ) { throw new Error('network is not initialized'); } const jsonLayers = []; for (let i = 0; i < this.layers.length; i++) { const layer = this.layers[i]; const jsonLayer = layer.toJSON(); if (layer.hasOwnProperty('inputLayer')) { jsonLayer.inputLayerIndex = this.layers.indexOf( layer.inputLayer as ILayer ); } else if ( layer.hasOwnProperty('inputLayer1') && layer.hasOwnProperty('inputLayer2') ) { jsonLayer.inputLayer1Index = this.layers.indexOf( layer.inputLayer1 as ILayer ); jsonLayer.inputLayer2Index = this.layers.indexOf( layer.inputLayer2 as ILayer ); } jsonLayers.push(jsonLayer); } return { type: this.constructor.name, sizes: this.options.sizes ?? [this._inputLayer.height] .concat(this._hiddenLayers.map((l) => l.height)) .concat([this._outputLayer.height]), outputLayerIndex: this.layers.indexOf(this._outputLayer), layers: jsonLayers as ILayerJSON[], inputLayerIndex: this.layers.indexOf(this._inputLayer), }; } static fromJSON( json: IFeedForwardJSON, getLayer?: ( layerJson: ILayerJSON, inputLayer1?: ILayer, inputLayer2?: ILayer ) => ILayer ): FeedForward { const jsonLayers = json.layers; const layers: ILayer[] = []; const inputLayer = getLayer ? layerFromJSON(jsonLayers[0]) ?? getLayer(jsonLayers[0]) : layerFromJSON(jsonLayers[0]); if (!inputLayer) throw new Error('unable to find layer'); layers.push(inputLayer); for (let i = 1; i < jsonLayers.length; i++) { const jsonLayer = jsonLayers[i]; if ( typeof jsonLayer.inputLayerIndex === 'undefined' && typeof jsonLayer.inputLayer1Index === 'undefined' && typeof jsonLayer.inputLayer2Index === 'undefined' ) { const layer = getLayer ? layerFromJSON(jsonLayer) ?? getLayer(jsonLayer) : layerFromJSON(jsonLayer); if (!layer) throw new Error('unable to find layer'); layers.push(layer); } else if (typeof jsonLayer.inputLayerIndex === 'number') { const inputLayer = layers[jsonLayer.inputLayerIndex]; if (!inputLayer) { throw new Error('inputLayer1 not found'); } const layer = getLayer ? layerFromJSON(jsonLayer, inputLayer) ?? getLayer(jsonLayer, inputLayer) : layerFromJSON(jsonLayer, inputLayer); if (!layer) throw new Error('unable to find layer'); layers.push(layer); } else { if (typeof jsonLayer.inputLayer1Index !== 'number') { throw new Error( 'Cannot create network from provided JSON. inputLayer1Index not defined.' ); } if (typeof jsonLayer.inputLayer2Index !== 'number') { throw new Error( 'Cannot create network from provided JSON. inputLayer2Index not defined.' ); } const inputLayer1 = layers[jsonLayer.inputLayer1Index]; const inputLayer2 = layers[jsonLayer.inputLayer2Index]; if (inputLayer1 === undefined) throw new Error( `Cannot create network from provided JSON. layer of index ${jsonLayer.inputLayer1Index} not found.` ); if (inputLayer2 === undefined) throw new Error( `Cannot create network from provided JSON. layer of index ${jsonLayer.inputLayer2Index} not found.` ); const layer = getLayer ? layerFromJSON(jsonLayer, inputLayer1, inputLayer2) ?? getLayer(jsonLayer, inputLayer1, inputLayer2) : layerFromJSON(jsonLayer, inputLayer1, inputLayer2); if (!layer) throw new Error('unable to find layer'); layers.push(layer); } } return new this({ ...json, layers }); } /** * * @returns {Function} */ toFunction(): void { throw new Error( `${this.constructor.name}-toFunction is not yet implemented` ); } } ================================================ FILE: src/feed-forward.unit.test.ts ================================================ import { GPU } from 'gpu.js'; import { setup, teardown } from './utilities/kernel'; import { FeedForward } from './feed-forward'; import { Add, BaseLayer, Convolution, convolution, feedForward, Input, input, Multiply, // Output, output, Pool, pool, Random, Relu, relu, Sigmoid, SoftMax, softMax, Target, // Zeros, layerTypes, ILayer, ILayerJSON, ILayerSettings, } from './layer'; import { mockLayer, mockPraxis } from './test-utils'; import SpyInstance = jest.SpyInstance; describe('FeedForward Class: Unit', () => { beforeEach(() => { setup( new GPU({ mode: 'cpu', }) ); }); afterEach(() => { teardown(); }); describe('.constructor()', () => { test('initially does not have any layers', () => { expect(new FeedForward().layers).toBeNull(); }); }); describe('layer composition', () => { const addValidate = Add.prototype.validate; beforeEach(() => { Add.prototype.validate = () => {}; }); afterEach(() => { Add.prototype.validate = addValidate; }); describe('flat', () => { test('can setup and traverse entire network as needed', () => { const net = new FeedForward({ inputLayer: () => input({ width: 1 }), hiddenLayers: [ (inputLayer: ILayer) => convolution( { filterCount: 8, filterWidth: 5, filterHeight: 5, padding: 2, stride: 1, }, inputLayer ), (inputLayer: ILayer) => relu(inputLayer), (inputLayer: ILayer) => pool( { filterHeight: 3, filterWidth: 3, filterCount: 1, padding: 2, stride: 2, }, inputLayer ), (inputLayer: ILayer) => convolution( { padding: 2, stride: 1, filterCount: 16, filterWidth: 5, filterHeight: 5, }, inputLayer ), (inputLayer: ILayer) => relu(inputLayer), (inputLayer: ILayer) => pool( { padding: 2, filterWidth: 3, filterHeight: 3, filterCount: 1, stride: 3, }, inputLayer ), (inputLayer: ILayer) => softMax(inputLayer), ], outputLayer: (inputLayer: ILayer) => output({ height: 10 }, inputLayer), }); net.initialize(); const layers = net.layers as ILayer[]; expect(layers.length).toBe(13); expect(layers.map((l: ILayer) => l.constructor)).toEqual([ Input, Convolution, Relu, Pool, Convolution, Relu, Pool, SoftMax, Random, Multiply, Random, Add, Target, ]); }); test('can setup and traverse entire network using layer composed of layers', () => { const net = new FeedForward({ inputLayer: () => input({ height: 1 }), hiddenLayers: [ (inputLayer) => feedForward({ height: 1 }, inputLayer), ], outputLayer: (inputLayer) => output({ height: 1 }, inputLayer), }); net.initialize(); const layers = net.layers as ILayer[]; expect(layers.length).toBe(11); expect(layers.map((l) => l.constructor)).toEqual([ Input, Random, Multiply, Random, Add, Sigmoid, Random, Multiply, Random, Add, Target, ]); }); }); describe('functional', () => { test('can setup and traverse entire network as needed', () => { const net = new FeedForward({ inputLayer: () => input({ width: 1 }), hiddenLayers: [ (inputParam) => softMax( pool( { filterWidth: 3, // TODO: setting height, width should behave same filterHeight: 3, filterCount: 3, padding: 2, stride: 3, }, relu( convolution( { padding: 2, stride: 1, filterCount: 16, filterWidth: 5, filterHeight: 5, }, pool( { filterWidth: 3, filterHeight: 3, filterCount: 16, padding: 2, stride: 2, }, relu( convolution( { filterCount: 8, filterWidth: 5, filterHeight: 5, padding: 2, stride: 1, }, inputParam ) ) ) ) ) ) ), ], outputLayer: (inputParam) => output({ height: 10 }, inputParam), }); net.initialize(); const layers = net.layers as ILayer[]; expect(layers.length).toBe(13); expect(layers.map((l) => l.constructor)).toEqual([ Input, Convolution, Relu, Pool, Convolution, Relu, Pool, SoftMax, Random, Multiply, Random, Add, Target, ]); }); }); }); describe('.initialize()', () => { test('initializes all layers', () => { class TestLayer extends BaseLayer { called?: boolean; setupKernels() { this.called = true; } } const net = new FeedForward({ inputLayer: () => new TestLayer(), hiddenLayers: [ () => new TestLayer(), () => new TestLayer(), () => new TestLayer(), ], outputLayer: () => new TestLayer(), }); net.initialize(); const layers = net.layers as ILayer[]; expect(layers.length).toBe(5); expect(layers.map((l) => l.constructor !== undefined)).toEqual([ true, true, true, true, true, ]); }); test('populates praxis on all layers when it is null', () => { class TestLayer extends layerTypes.Model { called?: boolean; setupKernels() { this.called = true; } } const net = new FeedForward({ inputLayer: () => new TestLayer(), hiddenLayers: [ () => new TestLayer(), () => new TestLayer(), () => new TestLayer(), ], outputLayer: () => new TestLayer(), }); net.initialize(); const layers = net.layers as ILayer[]; expect(layers.length).toBe(5); expect(layers.map((l) => (l as TestLayer).called)).toEqual([ true, true, true, true, true, ]); expect(layers.map((l) => Boolean(l.praxis))).toEqual([ true, true, true, true, true, ]); }); test('populates praxis when defined as setting on layer', () => { class TestLayer extends BaseLayer { called?: boolean; setupKernels() { this.called = true; } } const praxis = mockPraxis(mockLayer({})); const net = new FeedForward({ inputLayer: () => new TestLayer(), hiddenLayers: [ (l: ILayer) => new TestLayer({ initPraxis: () => praxis }), () => new TestLayer(), () => new TestLayer(), ], outputLayer: () => new TestLayer(), }); net.initialize(); const layers = net.layers as ILayer[]; expect(layers.length).toBe(5); expect(layers.map((l) => (l as TestLayer).called)).toEqual([ true, true, true, true, true, ]); expect(layers.map((l) => l.praxis === praxis)).toEqual([ false, true, false, false, false, ]); }); }); describe('.runInput()', () => { test('calls .predict() on all layers', () => { class TestLayer extends BaseLayer { // eslint-disable-next-line setupKernels() {} called?: boolean; predict() { this.called = true; } } const net = new FeedForward({ inputLayer: () => new TestLayer(), hiddenLayers: [ () => new TestLayer(), () => new TestLayer(), () => new TestLayer(), ], outputLayer: () => new TestLayer(), }); net.initialize(); net.runInput(); const layers = net.layers as ILayer[]; expect(layers.map((l) => (l as TestLayer).called)).toEqual([ true, true, true, true, true, ]); }); }); describe('._calculateDeltas()', () => { test('calls .compare() on all layers', () => { class TestLayer extends BaseLayer { // eslint-disable-next-line setupKernels() {} // eslint-disable-next-line predict() {} called?: boolean; compare() { this.called = true; } } const net = new FeedForward({ inputLayer: () => new TestLayer(), hiddenLayers: [ () => new TestLayer(), () => new TestLayer(), () => new TestLayer(), ], outputLayer: () => new TestLayer(), }); net.initialize(); net._calculateDeltas(); const layers = net.layers as ILayer[]; expect(layers.map((l) => (l as TestLayer).called)).toEqual([ true, true, true, true, true, ]); }); }); describe('.adjustWeights()', () => { test('calls .learn() on all layers', () => { class TestLayer extends layerTypes.Model { // eslint-disable-next-line setupKernels() {} // eslint-disable-next-line predict() {} // eslint-disable-next-line compare() {} called?: boolean; learn() { this.called = true; } } const net = new FeedForward({ inputLayer: () => new TestLayer(), hiddenLayers: [ () => new TestLayer(), () => new TestLayer(), () => new TestLayer(), ], outputLayer: () => new TestLayer(), }); net.initialize(); net.adjustWeights(); const layers = net.layers as ILayer[]; expect(layers.map((l) => (l as TestLayer).called)).toEqual([ true, true, true, true, true, ]); }); }); describe('.toJSON()', () => { test('can serialize to json', () => { class TestInputLayer extends BaseLayer { constructor(settings: ILayerSettings) { super(settings); this.weights = new Float32Array([0, 1, 3, 4, 5, 6, 7, 8, 9]); } } class TestLayer1 extends BaseLayer { inputLayer: ILayer; constructor(settings: ILayerSettings, inputLayer: ILayer) { super(settings); this.inputLayer = inputLayer; } // eslint-disable-next-line setupKernels() {} } class TestLayer2 extends BaseLayer { inputLayer: ILayer; constructor(settings: ILayerSettings, inputLayer: ILayer) { super(settings); this.inputLayer = inputLayer; } // eslint-disable-next-line setupKernels() {} } class TestOperatorLayer extends BaseLayer { inputLayer1: ILayer; inputLayer2: ILayer; constructor(inputLayer1: ILayer, inputLayer2: ILayer) { super(); this.inputLayer1 = inputLayer1; this.inputLayer2 = inputLayer2; } // eslint-disable-next-line setupKernels() {} } class TestOutputLayer extends BaseLayer { inputLayer: ILayer; constructor(settings: ILayerSettings, inputLayer: ILayer) { super(settings); this.inputLayer = inputLayer; } } const net = new FeedForward({ inputLayer: () => new TestInputLayer({ width: 10, height: 1 }), hiddenLayers: [ (inputParam) => new TestOperatorLayer( new TestLayer1({}, inputParam), new TestLayer2({}, inputParam) ), ], outputLayer: (inputParam) => new TestOutputLayer({ width: 10, height: 5 }, inputParam), }); net.initialize(); const json = net.toJSON(); expect(json.layers).toBeDefined(); expect(json.layers.every((l) => !l.hasOwnProperty('deltas'))).toBe(true); expect(json.layers.length).toBe(5); expect(json.layers[0]).toEqual({ type: 'TestInputLayer', praxisOpts: null, weights: [0, 1, 3, 4, 5, 6, 7, 8, 9], width: 10, height: 1, depth: 0, }); expect(json.layers[1]).toEqual({ type: 'TestLayer1', praxisOpts: null, weights: null, inputLayerIndex: 0, width: 1, height: 1, depth: 0, }); expect(json.layers[2]).toEqual({ type: 'TestLayer2', praxisOpts: null, weights: null, inputLayerIndex: 0, width: 1, height: 1, depth: 0, }); expect(json.layers[3]).toEqual({ type: 'TestOperatorLayer', praxisOpts: null, weights: null, inputLayer1Index: 1, inputLayer2Index: 2, width: 1, height: 1, depth: 0, }); expect(json.layers[4]).toEqual({ height: 5, inputLayerIndex: 3, type: 'TestOutputLayer', weights: null, praxisOpts: null, width: 10, depth: 0, }); }); }); describe('.fromJSON()', () => { test('can deserialize to object from json using inputLayerIndex', () => { class TestLayer extends BaseLayer implements ILayer { inputLayer?: ILayer; constructor(settings: ILayerSettings, inputLayer?: ILayer) { super(settings); this.inputLayer = inputLayer; } // eslint-disable-next-line setupKernels() {} } const net = FeedForward.fromJSON( { type: '', sizes: [], inputLayerIndex: 0, outputLayerIndex: 3, layers: [ { type: 'TestLayer', }, { type: 'TestLayer', inputLayerIndex: 0, }, { type: 'TestLayer', inputLayerIndex: 1, }, { type: 'TestLayer', inputLayerIndex: 2, }, ], }, (jsonLayer: ILayerJSON, inputLayer1?: ILayer, inputLayer2?: ILayer) => { switch (jsonLayer.type) { case 'TestLayer': return new TestLayer(jsonLayer, inputLayer1); default: throw new Error(`unknown layer ${jsonLayer.type}`); } } ); const layers = net.options.layers as ILayer[]; expect(layers.map((l) => l instanceof TestLayer)).toEqual([ true, true, true, true, ]); expect(layers.map((l) => l.inputLayer instanceof TestLayer)).toEqual([ false, true, true, true, ]); }); test('can deserialize to object from json using inputLayer1Index & inputLayer2Index', () => { class TestLayer extends BaseLayer { static get defaults() { return { foo: null }; } inputLayer?: ILayer; constructor(settings: ILayerSettings, inputLayer?: ILayer) { super(settings); this.inputLayer = inputLayer; } // eslint-disable-next-line setupKernels() {} } class TestOperatorLayer extends BaseLayer { static get defaults() { return { foo: null }; } inputLayer1?: ILayer; inputLayer2?: ILayer; constructor( settings: ILayerSettings, inputLayer1?: ILayer, inputLayer2?: ILayer ) { super(settings); this.inputLayer1 = inputLayer1; this.inputLayer2 = inputLayer2; } // eslint-disable-next-line setupKernels() {} } const net = FeedForward.fromJSON( { sizes: [], type: '', inputLayerIndex: 0, outputLayerIndex: 2, layers: [ { type: 'TestLayer', }, { type: 'TestLayer', inputLayerIndex: 0, }, { type: 'TestOperatorLayer', inputLayer1Index: 0, inputLayer2Index: 1, }, ], }, (jsonLayer: ILayerJSON, inputLayer1?: ILayer, inputLayer2?: ILayer) => { switch (jsonLayer.type) { case 'TestLayer': return new TestLayer(jsonLayer, inputLayer1); case 'TestOperatorLayer': return new TestOperatorLayer(jsonLayer, inputLayer1, inputLayer2); default: throw new Error(`unknown layer ${jsonLayer.type}`); } } ); const layers = net.options.layers as ILayer[]; expect(layers.length).toBe(3); expect(layers[0] instanceof TestLayer).toBeTruthy(); expect(layers[0] instanceof TestLayer).toBeTruthy(); expect(layers[1] instanceof TestLayer).toBeTruthy(); expect(layers[2] instanceof TestOperatorLayer).toBeTruthy(); expect(layers[2].inputLayer1).toEqual(layers[0]); expect(layers[2].inputLayer2).toEqual(layers[1]); }); }); describe('._trainPattern()', () => { let runInputSpy: jest.SpyInstance; let _calculateDeltasSpy: jest.SpyInstance; let adjustWeightsSpy: jest.SpyInstance; beforeEach(() => { runInputSpy = jest.spyOn(FeedForward.prototype, 'runInput'); _calculateDeltasSpy = jest.spyOn( FeedForward.prototype, '_calculateDeltas' ); adjustWeightsSpy = jest.spyOn(FeedForward.prototype, 'adjustWeights'); }); afterEach(() => { runInputSpy.mockRestore(); _calculateDeltasSpy.mockRestore(); adjustWeightsSpy.mockRestore(); }); test('calls training methods and mse2d and returns value', () => { const net = new FeedForward({ inputLayer: () => input({ height: 1 }), hiddenLayers: [(inputLayer) => feedForward({ height: 1 }, inputLayer)], outputLayer: (inputLayer) => output({ height: 1 }, inputLayer), }); net.initialize(); net._outputLayer = mockLayer({}); net._outputLayer.errors = [0]; net._trainPattern([1], [3], true); expect(runInputSpy).toHaveBeenCalled(); expect(_calculateDeltasSpy).toHaveBeenCalled(); expect(adjustWeightsSpy).toHaveBeenCalled(); }); }); describe('.trainOpts', () => { let net: FeedForward; let _calculateTrainingErrorSpy: SpyInstance; beforeEach(() => { const layer1 = mockLayer({ width: 1, height: 1 }); const layer2 = mockLayer({ width: 1, height: 1 }); layer2.errors = [1]; net = new FeedForward({ inputLayerIndex: 0, layers: [layer1, layer2], outputLayerIndex: 1, }); _calculateTrainingErrorSpy = jest.spyOn(net, '_calculateTrainingError'); }); afterEach(() => { _calculateTrainingErrorSpy.mockRestore(); }); test('.errorCheckInterval', () => { const trainOpts = { iterations: 2, errorCheckInterval: 1, errorThresh: 0.5, }; const mockData = [{ input: [1, 1], output: [1] }]; net.train(mockData, trainOpts); expect(_calculateTrainingErrorSpy).toHaveBeenCalled(); }); }); }); ================================================ FILE: src/index.ts ================================================ import * as activation from './activation'; import { AE } from './autoencoder'; import CrossValidate from './cross-validate'; import { FeedForward } from './feed-forward'; import * as layer from './layer'; import { layerTypes } from './layer'; import { likely } from './likely'; import { lookup } from './lookup'; import { NeuralNetwork } from './neural-network'; import { NeuralNetworkGPU } from './neural-network-gpu'; import * as praxis from './praxis'; import { Recurrent } from './recurrent'; import { GRU } from './recurrent/gru'; import { GRUTimeStep } from './recurrent/gru-time-step'; import { LSTM } from './recurrent/lstm'; import { LSTMTimeStep } from './recurrent/lstm-time-step'; import { RNN } from './recurrent/rnn'; import { RNNTimeStep } from './recurrent/rnn-time-step'; import { DataFormatter } from './utilities/data-formatter'; import { max } from './utilities/max'; import { mse } from './utilities/mse'; import { ones, ones2D } from './utilities/ones'; import * as random from './utilities/random'; import { randomWeight } from './utilities/random-weight'; import { randos } from './utilities/randos'; import { range } from './utilities/range'; import { toArray } from './utilities/to-array'; import { toSVG } from './utilities/to-svg'; import { zeros } from './utilities/zeros'; const recurrent = { RNNTimeStep, LSTMTimeStep, GRUTimeStep, RNN, LSTM, GRU, }; const utilities = { max, mse, ones, ones2D, random, randomWeight, randos, range, toArray, DataFormatter, zeros, toSVG, }; export { activation, AE, CrossValidate, likely, layer, layerTypes, lookup, praxis, FeedForward, NeuralNetwork, NeuralNetworkGPU, Recurrent, recurrent, utilities, }; ================================================ FILE: src/layer/README.md ================================================ # Layer ## Basics ### Memory A "basic layer" is composed of three types of Matrices which store what the neural network understand, its memory. * `weights` - how a layer forward propagates, or `predicts`. Usually weights initialize as random numbers and are * `errors` - how a network knows how far it was from an input or `target` during back propagation * `deltas` - how a network knows to adjust its `weights` during back propagation ### Action A layer has three different operations for it to "learn" * `predict` - usually referred to by non-mortals as "forward propagation", this is where `weights` are used * `compare` - the first of two steps in "back propagation", this compares what a network predicted to a `target` to calculate `deltas` and `errors` * `learn` - the second step in "back propagation", this step used to update the `weights` from what was measured from `deltas` and `errors` during `compare` ### Layer Composition A layer can be very simple, like `Random` or `Add`, but "compound layers" can also be described as "layers of layers". Layer Example: ```js import { FeedForward, layer } from 'brain.js'; const { input, output, add, random } = layer; function mySuperLayer(input) { return add(random(), input); } ``` Usage example: ```js const net = new FeedForward({ inputLayer: () => input(), hiddenLayers: [ input => mySuperLayer(input) ], outputLayer: input => output(input) }); ``` In this example both `add` and `random` are composed together, ie `layer composition`. This simple means of composing layers and in turn networks works with both simple (feed forward) or more complex (recurrent) networks. ================================================ FILE: src/layer/activation.test.ts ================================================ import { Activation } from './activation'; import { ILayerSettings } from './base-layer'; import { mockPraxis, mockLayer } from '../test-utils'; describe('Activation Abstract Layer', () => { describe('.constructor()', () => { describe('calls .validate()', () => { let mockValidate: jest.SpyInstance; beforeEach(() => { mockValidate = jest.spyOn(Activation.prototype, 'validate'); }); afterEach(() => { mockValidate.mockRestore(); }); test('.validate() call', () => { const mockInputLayer = mockLayer({ width: 1, height: 1, weights: [new Float32Array(1).fill(1)], deltas: [new Float32Array(1)], }); const praxis = mockPraxis(mockInputLayer); const l = new Activation(mockInputLayer, { praxis, }); expect(l.validate).toBeCalled(); }); }); test('inputLayer', () => { const mockInputLayer = mockLayer({ width: 1, height: 1, weights: [new Float32Array(1).fill(1)], deltas: [new Float32Array(1)], }); const l = new Activation(mockInputLayer); expect(l.inputLayer).toBe(mockInputLayer); }); test('dimensions', () => { const width = 3; const height = 4; const depth = 5; const testInputLayer = mockLayer({ width, height, depth }); const l = new Activation(testInputLayer); expect(l.width).toBe(width); expect(l.height).toBe(height); expect(l.depth).toBe(depth); }); test('2d weights & deltas', () => { const width = 3; const height = 4; const testInputLayer = mockLayer({ width, height }); const l = new Activation(testInputLayer); const weights = l.weights as Float32Array[]; expect(weights.length).toBe(height); expect(weights[0].length).toBe(width); expect(typeof weights[0][0]).toBe('number'); }); test('3d weights & deltas', () => { const width = 3; const height = 4; const depth = 5; const testInputLayer = mockLayer({ width, height, depth }); const l = new Activation(testInputLayer); const weights = l.weights as Float32Array[][]; expect(weights.length).toBe(depth); expect(weights[0].length).toBe(height); expect(weights[0][0].length).toBe(width); expect(typeof weights[0][0][0]).toBe('number'); }); test('initPraxis', () => { const mockPraxisInstance = mockPraxis(mockLayer({})); const mockInitPraxis = jest.fn(() => mockPraxisInstance); const settings: ILayerSettings = { initPraxis: mockInitPraxis, praxisOpts: {}, }; const mockInputLayer = mockLayer({ width: 1, height: 1, }); const l = new Activation(mockInputLayer, settings); expect(mockInitPraxis).toBeCalled(); expect(l.praxis).toBe(mockPraxisInstance); }); }); }); ================================================ FILE: src/layer/activation.ts ================================================ import { BaseLayer, ILayerSettings, ILayer } from './base-layer'; import { zeros2D } from '../utilities/zeros-2d'; import { zeros3D } from '../utilities/zeros-3d'; export type ActivationType = new ( inputLayer: ILayer, settings: Partial ) => ILayer; export class Activation extends BaseLayer { inputLayer: ILayer; get width(): number { return this.inputLayer.width; } get height(): number { return this.inputLayer.height; } get depth(): number { return this.inputLayer.depth; } constructor(inputLayer: ILayer, settings?: Partial) { super(settings); this.inputLayer = inputLayer; const { width, height, depth } = this; this.predictKernel = null; this.compareKernel = null; this.validate(); if (depth > 0) { this.weights = zeros3D(width, height, depth); this.deltas = zeros3D(width, height, depth); } else if (height > 0) { this.weights = zeros2D(width, height); this.deltas = zeros2D(width, height); } this.setupPraxis(); } } ================================================ FILE: src/layer/add.test.ts ================================================ import { GPU } from 'gpu.js'; import { gpuMock } from 'gpu-mock.js'; import { predict, Add, add } from './add'; import { setup, teardown, makeKernel, release } from '../utilities/kernel'; import { checkSameSize } from '../utilities/layer-size'; import { mockLayer, mockPraxis } from '../test-utils'; jest.mock('../utilities/layer-size'); jest.mock('../utilities/kernel', () => { return { makeKernel: jest.fn((fn) => () => [fn()]), setup: jest.fn(), release: jest.fn(), clear: jest.fn(), teardown: jest.fn(), clone: jest.fn((matrix: Float32Array[]) => matrix.map((row: Float32Array) => row.slice(0)) ), }; }); describe('Add Layer', () => { beforeEach(() => { setup( new GPU({ mode: 'cpu', }) ); }); afterEach(() => { teardown(); }); describe('.constructor', () => { let validateSpy: jest.SpyInstance; let setupPraxisSpy: jest.SpyInstance; beforeEach(() => { validateSpy = jest.spyOn(Add.prototype, 'validate'); setupPraxisSpy = jest.spyOn(Add.prototype, 'setupPraxis'); }); afterEach(() => { validateSpy.mockRestore(); setupPraxisSpy.mockRestore(); }); it('sets up the instance', () => { const mockInputLayer1 = mockLayer({ width: 1, height: 3 }); const mockInputLayer2 = mockLayer({ width: 1, height: 3 }); const praxis = mockPraxis(mockLayer({})); const settings = { praxis, }; const add = new Add(mockInputLayer1, mockInputLayer2, settings); expect(add.inputLayer1).toBe(mockInputLayer1); expect(add.inputLayer2).toBe(mockInputLayer2); expect(add.validate).toBeCalled(); expect(add.setupPraxis).toBeCalled(); expect(add.width).toBe(1); expect(add.height).toBe(3); }); }); describe('.predict (forward propagation)', () => { test('releases this.weights', () => { const praxis = mockPraxis(mockLayer({})); const add = new Add( mockLayer({ width: 1, height: 1, weights: [new Float32Array(1)] }), mockLayer({ width: 1, height: 1, weights: [new Float32Array(1)] }), { praxis, } ); add.predictKernel = makeKernel( function (weights1: number[][], weights2: number[][]) { return 1; }, { output: [1, 1] } ); const mockWeights = (add.weights = [new Float32Array(1)]); add.predict(); expect(release).toHaveBeenCalledWith(mockWeights); }); test('can add a simple matrix', () => { const inputs1 = [ [1, 2, 3], [4, 5, 6], [7, 8, 9], ]; const inputs2 = [ [1, 2, 3], [4, 5, 6], [7, 8, 9], ]; const results = gpuMock(predict, { output: [3, 3], })(inputs1, inputs2); expect(results).toEqual([ new Float32Array([2, 4, 6]), new Float32Array([8, 10, 12]), new Float32Array([14, 16, 18]), ]); }); }); describe('.validate', () => { it('calls LayerSize.checkSameSize()', () => { const mockLayer1 = { height: 1, width: 1, }; const mockLayer2 = { height: 1, width: 1, }; Add.prototype.validate.apply({ inputLayer1: mockLayer1, inputLayer2: mockLayer2, }); expect(checkSameSize).toHaveBeenCalledWith(mockLayer1, mockLayer2); }); }); describe('.setupKernels', () => { it('defines this.predictKernel', () => { const mockInstance = { width: 1, height: 1, predictKernel: null, }; Add.prototype.setupKernels.apply(mockInstance); expect(makeKernel).toHaveBeenCalledWith(predict, { output: [1, 1], immutable: true, }); expect(mockInstance.predictKernel).not.toBe(null); }); }); describe('.predict', () => { it('calls this.predictKernel with correct arguments', () => { const mockWeights1 = {}; const mockWeights2 = {}; const mockPredictKernel = jest.fn(); const mockInstance = { inputLayer1: { weights: mockWeights1 }, inputLayer2: { weights: mockWeights2 }, predictKernel: mockPredictKernel, }; Add.prototype.predict.apply(mockInstance); expect(mockPredictKernel).toHaveBeenCalled(); }); it('defined this.weights from this.predictKernel', () => { const mockWeights1 = {}; const mockWeights2 = {}; const mockResult = {}; const mockPredictKernel = () => mockResult; const mockInstance = { inputLayer1: { weights: mockWeights1 }, inputLayer2: { weights: mockWeights2 }, predictKernel: mockPredictKernel, weights: null, }; Add.prototype.predict.apply(mockInstance); expect(mockInstance.weights).toBe(mockResult); }); }); describe('.compare', () => { beforeEach(() => { jest.unmock('../../src/utilities/kernel'); }); it('sets this.inputLayer1.deltas & this.inputLayer2.deltas from this.deltas', () => { const mockDeltas = [new Float32Array([1])]; const mockInstance = { deltas: mockDeltas, inputLayer1: { deltas: null, }, inputLayer2: { deltas: null, }, }; Add.prototype.compare.apply(mockInstance); expect(mockInstance.inputLayer1.deltas).toEqual(mockDeltas); expect(mockInstance.inputLayer2.deltas).toEqual(mockDeltas); }); }); describe('add()', () => { let setupPraxisMock: jest.SpyInstance; beforeEach(() => { setupPraxisMock = jest.spyOn(Add.prototype, 'setupPraxis'); }); afterEach(() => { setupPraxisMock.mockRestore(); }); it('instantiates Add with inputLayer1, inputLayer2, and settings', () => { const mockInputLayer1 = mockLayer({ width: 1, height: 1, }); const mockInputLayer2 = mockLayer({ width: 1, height: 1, }); const praxis = mockPraxis(mockLayer({})); const settings = { praxis, }; const layer = add(mockInputLayer1, mockInputLayer2, settings); expect(layer.inputLayer1).toBe(mockInputLayer1); expect(layer.inputLayer2).toBe(mockInputLayer2); expect(layer.setupPraxis).toHaveBeenCalled(); }); }); }); ================================================ FILE: src/layer/add.ts ================================================ import { makeKernel, release, clone } from '../utilities/kernel'; import { checkSameSize } from '../utilities/layer-size'; import { Operator } from './operator'; import { IKernelFunctionThis, IKernelRunShortcut, Texture } from 'gpu.js'; import { ILayerSettings, ILayer } from './base-layer'; export function predict( this: IKernelFunctionThis, inputWeights1: number[][], inputWeights2: number[][] ): number { return ( inputWeights1[this.thread.y][this.thread.x] + inputWeights2[this.thread.y][this.thread.x] ); } export class Add extends Operator { get width(): number { return this.inputLayer1.width; } get height(): number { return this.inputLayer1.height; } get depth(): number { return this.inputLayer1.depth; } validate(): void { super.validate(); checkSameSize(this.inputLayer1, this.inputLayer2); } setupKernels(): void { this.predictKernel = makeKernel(predict, { output: [this.width, this.height], immutable: true, }); } predict(): void { release(this.weights); this.weights = (this.predictKernel as IKernelRunShortcut)( this.inputLayer1.weights, this.inputLayer2.weights ) as Texture; } compare(): void { // TODO: Do we need release and clone here? release(this.inputLayer1.deltas); release(this.inputLayer2.deltas); this.inputLayer1.deltas = clone(this.deltas); this.inputLayer2.deltas = clone(this.deltas); } } export function add( inputLayer1: ILayer, inputLayer2: ILayer, settings?: ILayerSettings ): Add { return new Add(inputLayer1, inputLayer2, settings); } ================================================ FILE: src/layer/arthur-feed-forward.ts ================================================ import { ArthurDeviationWeights, arthurDeviationWeights, IArthurDeviationWeightsSettings, } from '../praxis/arthur-deviation-weights'; import { arthurDeviationBiases, IArthurDeviationBiasesSettings, } from '../praxis/arthur-deviation-biases'; import { ILayer } from './base-layer'; import { add } from './add'; import { IRandomSettings, random } from './random'; import { multiply } from './multiply'; import { Sigmoid, sigmoid } from './sigmoid'; import { IPraxis } from '../praxis/base-praxis'; export interface IArthurFeedForwardPraxisSettings extends IArthurDeviationBiasesSettings, IArthurDeviationWeightsSettings {} export interface IArthurFeedForwardSettings extends IRandomSettings { initPraxis?: ( layerTemplate: ILayer, settings?: IArthurFeedForwardPraxisSettings | null ) => IPraxis; } export function arthurFeedForward( settings: IArthurFeedForwardPraxisSettings, inputLayer: ILayer ): Sigmoid { const { height } = settings; function initWeightsPraxis( layerTemplate: ILayer, settings?: IArthurDeviationWeightsSettings ): IPraxis { const praxis = arthurDeviationWeights(layerTemplate, settings); praxis.setupKernels(); return praxis; } function initBiasesPraxis( layerTemplate: ILayer, settings?: IArthurDeviationBiasesSettings ): IPraxis { const praxis = arthurDeviationBiases(layerTemplate, settings); praxis.setupKernels(); return praxis; } const weightsLayer = random({ id: 'weights', height, width: inputLayer.height, initPraxis: initWeightsPraxis, }); const biasesLayer = random({ id: 'biases', height, initPraxis: initBiasesPraxis, }); const multiplyLayer = multiply(weightsLayer, inputLayer); const addLayer = add(multiplyLayer, biasesLayer); const sigmoidLayer = sigmoid(addLayer); const weightsPraxis = weightsLayer.praxis as ArthurDeviationWeights; weightsPraxis.weightsLayer = weightsLayer; weightsPraxis.incomingLayer = inputLayer; weightsPraxis.deltaLayer = sigmoidLayer; return sigmoidLayer; } ================================================ FILE: src/layer/base-layer.test.ts ================================================ import { ILayer } from './base-layer'; import { IPraxis, IPraxisSettings } from '../praxis/base-praxis'; import { mockLayer, mockPraxis } from '../test-utils'; jest.mock('../utilities/kernel'); describe('BaseLayer Layer', () => { describe('dimensions', () => { describe('when given undefined for width, height, and depth', () => { test('automatically assigns 1 to width, height, and depth', () => { const base = mockLayer({ initPraxis: ( layerTemplate: ILayer, praxisSettings?: IPraxisSettings ): IPraxis => mockPraxis(layerTemplate, praxisSettings), praxisOpts: {}, }); expect(base.width).toBe(1); expect(base.height).toBe(1); expect(base.depth).toBe(null); }); }); }); describe('.praxisOpts', () => { test('are inherited to .praxis() call', () => { const initPraxis = jest.fn(); interface IPraxisExtendedSettings extends IPraxisSettings { test: number; } const praxisOpts: IPraxisExtendedSettings = { test: 100, }; const base = mockLayer({ initPraxis, praxisOpts, }); expect(initPraxis).toHaveBeenCalledWith(base, praxisOpts); }); }); }); ================================================ FILE: src/layer/base-layer.ts ================================================ import { IKernelRunShortcut, Input, Kernel, KernelOutput, Texture, TextureArrayOutput, } from 'gpu.js'; import { IPraxis, IPraxisSettings } from '../praxis/base-praxis'; import { clear } from '../utilities/kernel'; export interface ILayerJSON { width?: number; height?: number; depth?: number; weights?: number[] | number[][] | number[][][] | null; type: string; inputLayerIndex?: number; inputLayer1Index?: number; inputLayer2Index?: number; praxisOpts?: Partial | null; } export interface ILayer { width: number; height: number; depth: number; weights: KernelOutput | Input; deltas: KernelOutput; praxis: IPraxis | null; errors?: KernelOutput | null; setupKernels: (training?: boolean) => void; predictKernel: IKernelRunShortcut | null; compareKernel: IKernelRunShortcut | null; settings: Partial; reuseKernels: (layer: ILayer) => void; predict: (inputs?: KernelOutput) => void; compare: (targetValues?: KernelOutput) => void; learn: ((learningRate?: number) => void) | ((learningRate: number) => void); toJSON: () => Partial; inputLayer?: ILayer; inputLayer1?: ILayer; inputLayer2?: ILayer; index?: number; id?: string; } export interface ILayerSettings { width?: number | null; height?: number | null; depth?: number | null; weights?: KernelOutput | null; deltas?: KernelOutput | null; id?: string; praxis?: IPraxis | null; praxisOpts?: Partial | null; initPraxis?: | ((layerTemplate: ILayer, settings?: IPraxisSettings) => IPraxis) | null; cleanupDeltas?: boolean; } export const baseLayerDefaultSettings: ILayerSettings = { width: 1, height: 1, depth: null, weights: null, deltas: null, praxis: null, praxisOpts: null, cleanupDeltas: true, }; export type BaseLayerType = new (settings?: Partial) => ILayer; export class BaseLayer implements ILayer { praxis: IPraxis | null = null; predictKernel: IKernelRunShortcut | null = null; compareKernel: IKernelRunShortcut | null = null; settings: Partial; get width(): number { return this.settings.width ?? 0; } get height(): number { return this.settings.height ?? 0; } get depth(): number { return this.settings.depth ?? 0; } get weights(): KernelOutput | Input { return this.settings.weights as KernelOutput; } set weights(weights: KernelOutput | Input) { this.settings.weights = weights as KernelOutput; if (this.settings.cleanupDeltas && this.deltas) { clear(this.deltas); } } get deltas(): KernelOutput { return this.settings.deltas as KernelOutput; } set deltas(deltas: KernelOutput) { this.settings.deltas = deltas; } get id(): string { return this.settings.id ?? ''; } set id(title: string) { this.settings.id = title; } constructor(settings?: Partial) { if (settings) { this.settings = { ...baseLayerDefaultSettings, ...settings }; } else { this.settings = { ...baseLayerDefaultSettings }; } this.setupPraxis(); } setupPraxis(): void { const { initPraxis, praxis, praxisOpts } = this.settings; if (!this.praxis) { if (initPraxis) { if (praxisOpts) { this.praxis = initPraxis(this, praxisOpts); } else { this.praxis = initPraxis(this); } } else if (praxis) { this.praxis = praxis; } } } /* get weights() { return this._weights; } set weights(value) { if (value) { if (value.dimensions) { if (value.dimensions[0] !== this.width) { throw new Error(`${this.constructor.name}.weights being set with improper value width`); } if (value.dimensions[1] !== this.height) { throw new Error(`${this.constructor.name}.weights being set with improper value height`); } } else { if (value[0].length !== this.width) { throw new Error(`${this.constructor.name}.weights being set with improper value width`); } if (value.length !== this.height) { throw new Error(`${this.constructor.name}.weights being set with improper value height`); } } } this._weights = value; } get deltas() { return this._deltas; } set deltas(value) { if (value) { if (value.dimensions) { if (value.dimensions[0] !== this.width) { throw new Error(`${this.constructor.name}.deltas being set with improper value width`); } if (value.dimensions[1] !== this.height) { throw new Error(`${this.constructor.name}.deltas being set with improper value height`); } } else { if (value[0].length !== this.width) { throw new Error(`${this.constructor.name}.deltas being set with improper value width`); } if (value.length !== this.height) { throw new Error(`${this.constructor.name}.deltas being set with improper value height`); } } } this._deltas = value; } */ validate(): void { if (Number.isNaN(this.height)) { throw new Error(`${this.constructor.name} layer height is not a number`); } if (Number.isNaN(this.width)) { throw new Error(`${this.constructor.name} layer width is not a number`); } if (this.height < 1) { throw new Error(`${this.constructor.name} layer height is less than 1`); } if (this.width < 1) { throw new Error(`${this.constructor.name} layer width is less than 1`); } } setupKernels(isTraining?: boolean): void {} reuseKernels(layer: ILayer): void { if (layer.width !== this.width) { throw new Error( `${this.constructor.name} kernel width mismatch ${layer.width} is not ${this.width}` ); } if (layer.height !== this.height) { throw new Error( `${this.constructor.name} kernel width mismatch ${layer.height} is not ${this.height}` ); } if (layer.hasOwnProperty('predictKernel') && layer.predictKernel !== null) { if (!(layer.predictKernel as Kernel).immutable) { throw new Error( `${layer.constructor.name}.predictKernel is not reusable, set kernel.immutable = true` ); } this.predictKernel = layer.predictKernel; } if (layer.hasOwnProperty('compareKernel') && layer.compareKernel !== null) { if (!(layer.compareKernel as Kernel).immutable) { throw new Error( `${layer.constructor.name}.compareKernel is not reusable, set kernel.immutable = true` ); } this.compareKernel = layer.compareKernel; } this.praxis = layer.praxis; } predict(inputs?: KernelOutput): void {} compare(targetValues?: KernelOutput): void {} learn(learningRate?: number): void {} toArray(): TextureArrayOutput { return Array.isArray(this.weights) ? this.weights : (this.weights as Texture).toArray(); } toJSON(): Partial { return BaseLayer.toJSON(this); } static toJSON(layer: ILayer): Partial { const { weights } = layer; return { width: layer.width, height: layer.height, depth: layer.depth, weights: toUntypedArray( (weights && weights instanceof Texture ? weights.toArray() : weights) as | Float32Array | Float32Array[] | Float32Array[][] | number[] | number[][] | number[][][] | null ), type: layer.constructor.name, praxisOpts: layer.praxis ? layer.praxis.toJSON() : null, }; } } function toUntypedArray( weights: | Float32Array | Float32Array[] | Float32Array[][] | number[] | number[][] | number[][][] | null ): number[][][] | number[][] | number[] | null { if (weights === null) return null; if (Array.isArray(weights)) { if (typeof weights[0] === 'number') { return weights as number[]; } else if (Array.isArray(weights[0]) && typeof weights[0][0] === 'number') { return weights as number[][]; } else if ( Array.isArray(weights[0][0]) && typeof weights[0][0][0] === 'number' ) { return weights as number[][][]; } else if (weights[0] instanceof Float32Array) { const matrix = weights as Float32Array[]; return matrix.map((row: Float32Array) => { return Array.from(row); }); } else if (weights[0][0] instanceof Float32Array) { const cube = weights as Float32Array[][]; return cube.map((matrix: Float32Array[]): number[][] => { return matrix.map((row: Float32Array): number[] => { return Array.from(row); }); }); } } else if (weights) { return Array.from(weights); } throw new Error('unexpected value'); } ================================================ FILE: src/layer/convolution.test.ts ================================================ import { GPU } from 'gpu.js'; import { gpuMock } from 'gpu-mock.js'; import { predict, compareFilterDeltas, compareInputDeltas, compareBiases, } from './convolution'; import { setup, teardown } from '../utilities/kernel'; import { onePlusPlus3D } from '../test-utils'; describe('Convolution Layer', () => { beforeEach(() => { setup( new GPU({ mode: 'cpu', }) ); }); afterEach(() => { teardown(); }); describe('.predict (forward propagation)', () => { test('can convolve a simple matrix', () => { const inputs = [ [ [1, 2, 3], [4, 5, 6], [7, 8, 9], ], ]; const filters = [ [ [1, 2, 3], [4, 5, 6], [7, 8, 9], ], ]; const biases = [1, 2, 3]; const results = gpuMock(predict, { output: [3, 3], constants: { strideX: 1, strideY: 1, paddingY: 0, paddingX: 0, filterHeight: 3, filterWidth: 3, filterCount: 1, inputWidth: 3, inputHeight: 3, inputDepth: 1, }, })(filters, inputs, biases); expect(results).toEqual([ new Float32Array([286, 187, 91]), new Float32Array([155, 95, 43]), new Float32Array([51, 27, 10]), ]); }); }); describe('.compareFilterDeltas (back propagation)', () => { test('can convolve a simple matrix', () => { const filterWidth = 2; const filterHeight = 2; const inputWidth = 4; const inputHeight = 4; const inputDepth = 1; const width = 2; const height = 2; const depth = 1; const stride = 1; const padding = 0; const filterDeltas = onePlusPlus3D(filterWidth, filterHeight, inputDepth); const inputs = onePlusPlus3D(inputWidth, inputHeight, inputDepth); const deltas = onePlusPlus3D(width, height, depth); const results = gpuMock(compareFilterDeltas, { output: [filterWidth, filterHeight, 1], constants: { strideX: stride, strideY: stride, paddingY: padding, paddingX: padding, filterWidth, filterHeight, inputWidth, inputHeight, deltaZ: 0, deltaWidth: width, deltaHeight: height, }, })(filterDeltas, inputs, deltas); expect(results).toEqual([ [new Float32Array([45, 56]), new Float32Array([87, 98])], ]); }); }); describe('.compareInputDeltas (back propagation)', () => { test('can convolve a simple matrix', () => { const inputDeltas = [ [ [1, 2, 3], [4, 5, 6], [7, 8, 9], ], ]; const filters = [ [ [1, 2, 3], [4, 5, 6], [7, 8, 9], ], ]; const deltas = [ [ [1, 2, 3], [4, 5, 6], [7, 8, 9], ], ]; const results = gpuMock(compareInputDeltas, { output: [3, 3], constants: { strideX: 1, strideY: 1, paddingY: 0, paddingX: 0, filterHeight: 3, filterWidth: 3, filterCount: 1, deltaWidth: 3, deltaHeight: 3, deltaDepth: 1, deltaZ: 0, }, })(inputDeltas, filters, deltas); expect(results).toEqual([ new Float32Array([2, 6, 13]), new Float32Array([12, 31, 62]), new Float32Array([37, 92, 174]), ]); }); }); describe('.compareBiases (back propagation)', () => { const deltas = [ [ [0, 16], [8, 24], ], [ [1, 17], [9, 25], ], [ [2, 18], [10, 26], ], [ [3, 19], [11, 27], ], [ [4, 20], [12, 28], ], [ [5, 21], [13, 29], ], [ [6, 22], [14, 30], ], [ [7, 23], [15, 31], ], ]; test('accumulates values from deltas correctly from 0', () => { const biasDeltas = [ [[0]], [[0]], [[0]], [[0]], [[0]], [[0]], [[0]], [[0]], ]; const kernel = gpuMock(compareBiases, { output: [1, 1, 8], constants: { deltaWidth: 2, deltaHeight: 2, }, }); const result = kernel(biasDeltas, deltas); const expectedBiasDeltas = [ [new Float32Array([48])], [new Float32Array([52])], [new Float32Array([56])], [new Float32Array([60])], [new Float32Array([64])], [new Float32Array([68])], [new Float32Array([72])], [new Float32Array([76])], ]; expect(result).toEqual(expectedBiasDeltas); }); test('accumulates values from deltas correctly from greater than 0', () => { const biasDeltas = [ [[0]], [[1]], [[2]], [[3]], [[4]], [[5]], [[6]], [[7]], ]; const kernel = gpuMock(compareBiases, { output: [1, 1, 8], constants: { deltaWidth: 2, deltaHeight: 2, }, }); const result = kernel(biasDeltas, deltas); const expectedBiasDeltas = [ [new Float32Array([48])], [new Float32Array([53])], [new Float32Array([58])], [new Float32Array([63])], [new Float32Array([68])], [new Float32Array([73])], [new Float32Array([78])], [new Float32Array([83])], ]; expect(result).toEqual(expectedBiasDeltas); }); }); }); ================================================ FILE: src/layer/convolution.ts ================================================ import { makeKernel, release, clone } from '../utilities/kernel'; import { getStride, getPadding } from '../utilities/layer-setup'; import { Filter } from './filter'; import { randos, randos3D } from '../utilities/randos'; import { zeros3D } from '../utilities/zeros-3d'; import { values } from '../utilities/values'; import { IConstantsThis, IKernelFunctionThis, IKernelRunShortcut, KernelOutput, } from 'gpu.js'; import { ILayer, ILayerSettings } from './base-layer'; import { IPraxis } from '../praxis/base-praxis'; export interface IConvolutionConstantsBase extends IConstantsThis { paddingX: number; paddingY: number; strideX: number; strideY: number; filterWidth: number; filterHeight: number; } export interface IPredictConstants extends IConvolutionConstantsBase { inputWidth: number; inputHeight: number; inputDepth: number; } export function predict( this: IKernelFunctionThis, inputs: number[][][], filters: number[][][], biases: number[] ): number { const startFilterX = this.constants.paddingX - this.thread.x * this.constants.strideX; const startInputX = this.thread.x * this.constants.strideX - this.constants.paddingX; const endFilterX = Math.min( this.constants.filterWidth, startFilterX + this.constants.inputWidth ); const startFilterY = this.constants.paddingY - this.thread.y * this.constants.strideY; const startInputY = this.thread.y * this.constants.strideY - this.constants.paddingY; const endFilterY = Math.min( this.constants.filterHeight, startFilterY + this.constants.inputHeight ); let sum = 0; for (let z = 0; z < this.constants.inputDepth; z++) { for ( let filterY = Math.max(0, startFilterY), inputY = Math.max(0, startInputY); filterY < endFilterY; filterY++, inputY++ ) { for ( let filterX = Math.max(0, startFilterX), inputX = Math.max(0, startInputX); filterX < endFilterX; filterX++, inputX++ ) { sum += filters[z][filterY][filterX] * inputs[z][inputY][inputX]; } } } return sum + biases[this.thread.z]; } export interface ICompareFilterDeltasConstants extends IConvolutionConstantsBase { deltaWidth: number; deltaHeight: number; inputWidth: number; inputHeight: number; deltaZ: number; } export function compareFilterDeltas( this: IKernelFunctionThis, filterDeltas: number[][][], inputs: number[][][], deltas: number[][][] ): number { const startDeltaX = Math.max( 0, Math.ceil( (this.constants.paddingX - this.thread.x) / this.constants.strideX ) ); const startInputX = startDeltaX * this.constants.strideX + this.thread.x - this.constants.paddingX; const endDeltaX = Math.min( this.constants.deltaWidth, Math.floor( (this.constants.inputWidth - 1 - this.thread.x + this.constants.paddingX) / this.constants.strideX ) + 1 ); const startDeltaY = Math.max( 0, Math.ceil( (this.constants.paddingY - this.thread.y) / this.constants.strideY ) ); const startInputY = startDeltaY * this.constants.strideY + this.thread.y - this.constants.paddingY; const endDeltaY = Math.min( this.constants.deltaHeight, Math.floor( (this.constants.inputHeight - 1 - this.thread.y + this.constants.paddingY) / this.constants.strideY ) + 1 ); let sum = filterDeltas[this.thread.z][this.thread.y][this.thread.x]; for ( let deltaY = startDeltaY, inputY = startInputY; deltaY < endDeltaY; deltaY++, inputY += this.constants.strideY ) { for ( let deltaX = startDeltaX, inputX = startInputX; deltaX < endDeltaX; deltaX++, inputX += this.constants.strideX ) { sum += inputs[this.thread.z][inputY][inputX] * deltas[this.constants.deltaZ][deltaY][deltaX]; } } return sum; } export interface ICompareInputDeltasConstants extends IConvolutionConstantsBase { deltaHeight: number; deltaWidth: number; deltaZ: number; } export function compareInputDeltas( this: IKernelFunctionThis, inputDeltas: number[][][], filters: number[][][], deltas: number[][][] ): number { const x = this.thread.x + this.constants.paddingX; const startDeltaX = x < this.constants.filterWidth ? 0 : Math.floor( (x - this.constants.filterWidth + this.constants.strideX) / this.constants.strideX ); const startFilterX = x - startDeltaX * this.constants.strideX; const endDeltaX = Math.min( startDeltaX + Math.floor(startFilterX / this.constants.strideX) + 1, this.constants.deltaWidth ); const y = this.thread.y + this.constants.paddingY; const startDeltaY = y < this.constants.filterHeight ? 0 : Math.floor( (y - this.constants.filterHeight + this.constants.strideY) / this.constants.strideY ); const startFilterY = y - startDeltaY * this.constants.strideY; const endDeltaY = Math.min( startDeltaY + Math.floor(startFilterY / this.constants.strideY) + 1, this.constants.deltaHeight ); let sum = inputDeltas[this.thread.z][this.thread.y][this.thread.x]; let deltaY = startDeltaY; for ( let filterY = startFilterY; deltaY < endDeltaY; filterY -= this.constants.strideY, deltaY++ ) { let deltaX = startDeltaX; for ( let filterX = startFilterX; deltaX < endDeltaX; filterX -= this.constants.strideX, deltaX++ ) { sum += filters[this.thread.z][filterY][filterX] * deltas[this.constants.deltaZ][deltaY][deltaX]; } } return sum; } export interface ICompareBiasesConstants extends IConstantsThis { deltaHeight: number; deltaWidth: number; } export function compareBiases( this: IKernelFunctionThis, biasDeltas: number[][][], deltas: number[][][] ): number { let sum = 0; for (let y = 0; y < this.constants.deltaHeight; y++) { for (let x = 0; x < this.constants.deltaWidth; x++) { sum += deltas[this.thread.z][y][x]; } } return biasDeltas[this.thread.z][this.thread.y][this.thread.x] + sum; } export interface IConvolutionSettingsBase { stride?: number; strideX?: number; strideY?: number; padding?: number; paddingX?: number; paddingY?: number; filterCount?: number; filterWidth?: number; filterHeight?: number; } export interface IConvolutionSettings extends ILayerSettings, IConvolutionSettingsBase { bias?: number; biases?: KernelOutput; biasDeltas?: KernelOutput; filters?: KernelOutput; filterDeltas?: KernelOutput; } export const defaults: IConvolutionSettings = { stride: 0, padding: 0, bias: 0.1, filterCount: 1, filterWidth: 0, filterHeight: 0, }; export class Convolution extends Filter { settings: Partial; get strideX(): number { return this.settings.strideX as number; } get strideY(): number { return this.settings.strideY as number; } get paddingX(): number { return this.settings.paddingX as number; } get paddingY(): number { return this.settings.paddingX as number; } get width(): number { return Math.floor( (this.inputLayer.width + this.paddingX * 2 - this.filterWidth) / this.strideX + 1 ); } get height(): number { return Math.floor( (this.inputLayer.height + this.paddingY * 2 - this.filterHeight) / this.strideY + 1 ); } get bias(): number { return this.settings.bias as number; } get depth(): number { return this.filterCount; } get biases(): KernelOutput { return this.settings.biases; } set biases(biases: KernelOutput) { this.settings.biases = biases; } get biasDeltas(): KernelOutput { return this.settings.biasDeltas; } set biasDeltas(weights: KernelOutput) { this.settings.biasDeltas = weights; } get filters(): KernelOutput { return this.settings.filters; } set filters(filters: KernelOutput) { this.settings.filters = filters; } get filterDeltas(): KernelOutput { return this.settings.filterDeltas; } set filterDeltas(filterDeltas: KernelOutput) { this.settings.filterDeltas = filterDeltas; } constructor(settings: IConvolutionSettings, inputLayer: ILayer) { super(settings, inputLayer); this.settings = { ...defaults, ...settings, ...getPadding(settings, defaults), ...getStride(settings, defaults), }; this.weights = settings.weights ?? randos3D(this.width, this.height, this.depth); this.deltas = zeros3D(this.width, this.height, this.depth); this.biases = values(this.depth, this.bias); this.biasDeltas = settings.biasDeltas ?? randos(this.depth); this.filters = settings.filters ?? randos3D(this.filterWidth, this.filterHeight, this.filterCount); this.filterDeltas = zeros3D( this.filterWidth, this.filterHeight, this.filterCount ); this.validate(); } compareFilterDeltasKernel: IKernelRunShortcut | null = null; compareInputDeltasKernel: IKernelRunShortcut | null = null; compareBiasesKernel: IKernelRunShortcut | null = null; setupKernels(): void { this.predictKernel = makeKernel< Parameters, IPredictConstants >(predict, { constants: { inputWidth: this.inputLayer.width, inputHeight: this.inputLayer.height, inputDepth: this.inputLayer.depth, strideX: this.strideX, strideY: this.strideY, paddingX: this.paddingX, paddingY: this.paddingY, filterWidth: this.filterWidth, filterHeight: this.filterHeight, }, output: [this.width, this.height, this.depth], immutable: true, }); this.compareFilterDeltasKernel = makeKernel(compareFilterDeltas, { constants: { deltaWidth: this.width, deltaHeight: this.height, deltaZ: this.depth, inputWidth: this.inputLayer.width, inputHeight: this.inputLayer.height, inputDepth: this.inputLayer.depth, strideX: this.strideX, strideY: this.strideY, paddingX: this.paddingX, paddingY: this.paddingY, filterWidth: this.filterWidth, filterHeight: this.filterHeight, }, output: [this.width, this.height, this.depth], immutable: true, }); this.compareInputDeltasKernel = makeKernel(compareInputDeltas, { constants: { deltaWidth: this.width, deltaHeight: this.height, deltaZ: this.depth, strideX: this.strideX, strideY: this.strideY, paddingX: this.paddingX, paddingY: this.paddingY, filterWidth: this.filterWidth, filterHeight: this.filterHeight, filterCount: this.filterCount, }, output: [ this.inputLayer.width, this.inputLayer.height, this.inputLayer.depth, ], immutable: true, }); this.compareBiasesKernel = makeKernel(compareBiases, { output: [1, 1, this.depth], constants: { deltaWidth: this.width, deltaHeight: this.height, }, immutable: true, }); } predict(): void { this.weights = (this.predictKernel as IKernelRunShortcut)( this.inputLayer.weights, this.filters, this.biases ); } compare(): void { const { filterDeltas, biasDeltas } = this; this.filterDeltas = (this.compareFilterDeltasKernel as IKernelRunShortcut)( filterDeltas, this.inputLayer.weights, this.deltas ); release(filterDeltas); this.biasDeltas = (this.compareBiasesKernel as IKernelRunShortcut)( biasDeltas, this.deltas ); release(biasDeltas); release(this.deltas); this.deltas = (this.compareInputDeltasKernel as IKernelRunShortcut)( this.filters, this.inputLayer.deltas ); release(this.inputLayer.deltas); // TODO: do we need to clone here? this.inputLayer.deltas = clone(this.deltas); } learn(learningRate: number): void { // TODO: handle filters // TODO: do we need to release here? const { weights: oldWeights } = this; this.weights = (this.praxis as IPraxis).run(this, learningRate); release(oldWeights); } } export function convolution( settings: IConvolutionSettings, inputLayer: ILayer ): Convolution { return new Convolution(settings, inputLayer); } ================================================ FILE: src/layer/dropout.test.ts ================================================ import { GPU } from 'gpu.js'; import { gpuMock } from 'gpu-mock.js'; import { dropout, Dropout, trainingPredict, predict, compare, setDropout, IDropoutSettings, } from './dropout'; import { setup, teardown, makeKernel, makeKernelMap, } from '../utilities/kernel'; import { IWithCompareKernel, IWithPredictKernelMap, mockLayer, } from '../test-utils'; jest.mock('../utilities/kernel'); describe('Dropout Layer', () => { let validateMock: jest.SpyInstance; beforeEach(() => { setup( new GPU({ mode: 'cpu', }) ); validateMock = jest.spyOn(Dropout.prototype, 'validate'); }); afterEach(() => { teardown(); validateMock.mockRestore(); }); describe('dropout', () => { it('sends inputLayer and settings through and instantiates a Dropout', () => { const mockInputLayer = mockLayer({}); const settings: IDropoutSettings = { probability: 100, }; const layer = dropout(mockInputLayer, settings); expect(layer.constructor).toBe(Dropout); expect(layer.settings.probability).toBe(settings.probability); }); }); describe('.constructor', () => { it('sets inputLayer', () => { const mockInputLayer = mockLayer({}); const layer = new Dropout(mockInputLayer); expect(layer.inputLayer).toBe(mockInputLayer); }); it('sets height & width from inputLayer', () => { const mockInputLayer = mockLayer({ width: 1, height: 2, }); const layer = new Dropout(mockInputLayer); expect(layer.width).toBe(mockInputLayer.width); expect(layer.height).toBe(mockInputLayer.height); }); it('sets probability from settings', () => { const mockInputLayer = mockLayer({}); const settings: IDropoutSettings = { probability: 123, }; const layer = new Dropout(mockInputLayer, settings); expect(layer.settings.probability).toBe(settings.probability); }); it('calls this.validate', () => { const mockInputLayer = mockLayer({}); const settings: IDropoutSettings = { probability: 123, }; // eslint-disable-next-line no-new new Dropout(mockInputLayer, settings); expect(validateMock).toHaveBeenCalled(); }); it('sets dropouts to null', () => { const mockInputLayer = mockLayer({}); const layer = new Dropout(mockInputLayer); expect(layer.dropouts).toBe(null); }); }); describe('trainingPredict (forward propagation)', () => { test('can dropout a simple matrix', () => { const inputs = [ [1, 2, 3], [4, 5, 6], [7, 8, 9], ]; const results = gpuMock(trainingPredict, { output: [3, 3], constants: { probability: 0.5, }, })(inputs) as number[][]; let hasZero = false; let hasNumber = false; for (let y = 0; y < results.length; y++) { const row = results[y]; for (let x = 0; x < row.length; x++) { const value = row[x]; if (value === 0) { hasZero = true; } else if (!Number.isNaN(value)) { hasNumber = true; } } } expect(hasZero).toBeTruthy(); expect(hasNumber).toBeTruthy(); }); }); describe('.training (forward propagation)', () => { test('can dropout a simple matrix', () => { const inputs = [ [1, 2, 3], [4, 5, 6], [7, 8, 9], ]; const results = gpuMock(predict, { output: [3, 3], constants: { probability: 0.5, }, })(inputs); expect(results).toEqual([ new Float32Array([0.5, 1, 1.5]), new Float32Array([2, 2.5, 3]), new Float32Array([3.5, 4, 4.5]), ]); }); }); describe('.setupKernels', () => { describe('isTraining is true', () => { it('this.predictKernel should be set', () => { const inputLayer = mockLayer({ height: 2, width: 1 }); const layer = new Dropout(inputLayer); layer.setupKernels(true); expect(makeKernelMap).toHaveBeenCalledWith( { dropouts: setDropout }, trainingPredict, { output: [1, 2], immutable: true, } ); expect(layer.predictKernelMap).not.toBe(null); }); it('this.compareKernel should be set', () => { const inputLayer = mockLayer({ height: 2, width: 1 }); const layer = new Dropout(inputLayer); layer.setupKernels(true); expect(makeKernel).toHaveBeenCalledWith(compare, { output: [1, 2], immutable: true, }); expect(layer.compareKernel).not.toBe(null); }); }); describe('isTraining is false', () => { it('this.predictKernelMap should be set', () => { const inputLayer = mockLayer({ width: 1, height: 2 }); const layer = new Dropout(inputLayer); layer.setupKernels(false); expect(makeKernelMap).toHaveBeenCalledWith({}, predict, { output: [1, 2], immutable: true, }); expect(layer.predictKernelMap).not.toBe(null); }); }); }); describe('.predict', () => { it('calls this.predictKernelMap with this.inputLayer.weights', () => { const inputLayer = mockLayer({ weights: [[[0]]] }); const layer = new Dropout(inputLayer); const spy = (((layer as unknown) as IWithPredictKernelMap).predictKernelMap = jest.fn( () => { return [1]; } )); layer.predict(); expect(spy).toHaveBeenCalledWith(inputLayer.weights); }); it('sets this.weights from result', () => { const inputLayer = mockLayer({ weights: [[[0]]] }); const layer = new Dropout(inputLayer); const weights = [[[1]]]; ((layer as unknown) as IWithPredictKernelMap).predictKernelMap = jest.fn( () => { return { result: weights, }; } ); layer.predict(); expect(layer.weights).toBe(weights); }); it('sets this.dropouts from dropouts', () => { const inputLayer = mockLayer({ weights: [[[0]]] }); const layer = new Dropout(inputLayer); const dropouts = [[[1]]]; ((layer as unknown) as IWithPredictKernelMap).predictKernelMap = jest.fn( () => { return { dropouts: dropouts, }; } ); layer.predict(); expect(layer.dropouts).toBe(dropouts); }); }); describe('.compare', () => { it('calls this.compareKernel with this.dropouts and this.inputLayer.deltas', () => { const inputLayer = mockLayer({ width: 1, height: 2 }); const layer = new Dropout(inputLayer); const dropouts = (layer.dropouts = [1]); const inputLayerDeltas = (inputLayer.deltas = [42]); const compareKernel = (((layer as unknown) as IWithCompareKernel).compareKernel = jest.fn()); layer.compare(); expect(compareKernel).toBeCalledWith(dropouts, inputLayerDeltas); }); it('sets this.deltas', () => { const inputLayer = mockLayer({ width: 1, height: 2 }); const layer = new Dropout(inputLayer); const expectedResult = [42]; ((layer as unknown) as IWithCompareKernel).compareKernel = jest.fn(() => { return expectedResult; }); layer.compare(); expect(layer.deltas).toBe(expectedResult); }); }); describe('compare', () => { test('output', () => { const results = gpuMock(compare, { output: [3, 3], constants: { probability: 0.5, }, })( [ [1, 0, 1], [0, 1, 0], [1, 1, 1], ], [ [1, 2, 3], [4, 5, 6], [7, 8, 9], ] ); expect(results).toEqual([ new Float32Array([1, 0, 3]), new Float32Array([0, 5, 0]), new Float32Array([7, 8, 9]), ]); }); }); }); ================================================ FILE: src/layer/dropout.ts ================================================ import { Filter, IFilterSettings } from './filter'; import { makeKernel, makeKernelMap, release } from '../utilities/kernel'; import { IConstantsThis, IKernelFunctionThis, IKernelMapRunShortcut, IKernelRunShortcut, ISubKernelObject, KernelOutput, } from 'gpu.js'; import { ILayer, ILayerSettings, baseLayerDefaultSettings } from './base-layer'; export function setDropout(dropout: number): number { return dropout; } export interface IDropoutConstants extends IConstantsThis { probability: number; } export function trainingPredict( this: IKernelFunctionThis, inputs: number[][] ): number { if (setDropout(Math.random()) < this.constants.probability) { return 0; } return inputs[this.thread.y][this.thread.x]; } export function predict( this: IKernelFunctionThis, inputs: number[][] ): number { return inputs[this.thread.y][this.thread.x] * this.constants.probability; } export function compare( this: IKernelFunctionThis, dropouts: number[][], deltas: number[][] ): number { if (dropouts[this.thread.y][this.thread.x] === 0) { return 0; } return deltas[this.thread.y][this.thread.x]; } export interface IDropoutSettings extends ILayerSettings { probability: number; } export const dropoutDefaults: IDropoutSettings = { ...baseLayerDefaultSettings, probability: 0.5, }; export class Dropout extends Filter { dropouts: KernelOutput | null; predictKernelMap: IKernelMapRunShortcut | null = null; settings: Partial; constructor( inputLayer: ILayer, settings?: Partial & Partial ) { super(settings as Partial, inputLayer); this.settings = { ...dropoutDefaults, ...settings }; this.dropouts = null; this.validate(); } setupKernels(isTraining?: boolean): void { const output = [this.width, this.height]; if (isTraining) { this.predictKernelMap = makeKernelMap< Parameters, IDropoutConstants >({ dropouts: setDropout }, trainingPredict, { output, immutable: true, }); this.compareKernel = makeKernel(compare, { output, immutable: true }); } else { this.predictKernelMap = makeKernelMap< Parameters, IDropoutConstants >({}, predict, { output, immutable: true }); } } predict(): void { release(this.weights); if (this.dropouts) { release(this.dropouts); } const { result, dropouts } = (this .predictKernelMap as IKernelMapRunShortcut)( this.inputLayer.weights ); this.weights = result; this.dropouts = dropouts; } compare(): void { release(this.deltas); this.deltas = (this.compareKernel as IKernelRunShortcut)( this.dropouts as KernelOutput, this.inputLayer.deltas ); } } export function dropout( inputLayer: ILayer, settings?: Partial ): Dropout { return new Dropout(inputLayer, settings); } ================================================ FILE: src/layer/feed-forward.test.ts ================================================ import { feedForward } from './feed-forward'; import { mockLayer } from '../test-utils'; describe('FeedForward Layer', () => { test('properly sets width and height', () => { const input = mockLayer({ width: 1, height: 3 }); const settings = { height: 3 }; const layer = feedForward(settings, input); expect(layer.width).toBe(1); expect(layer.height).toBe(settings.height); }); }); ================================================ FILE: src/layer/feed-forward.ts ================================================ import { random } from './random'; import { add } from './add'; import { multiply } from './multiply'; import { sigmoid } from './sigmoid'; import { ILayer, ILayerSettings } from './base-layer'; export function feedForward(settings: ILayerSettings, input: ILayer): ILayer { const { height, praxisOpts = null } = settings; const weights = random({ id: 'weights', height, width: input.height, praxisOpts, }); const biases = random({ id: 'biases', height, praxisOpts }); return sigmoid( add(multiply(weights, input, { praxisOpts }), biases, { praxisOpts }), { praxisOpts } ); } ================================================ FILE: src/layer/filter.ts ================================================ import { KernelOutput } from 'gpu.js'; import { BaseLayer, ILayer, ILayerSettings } from './base-layer'; export interface IFilterSettings extends ILayerSettings { filterCount: number; filterWidth: number; filterHeight: number; filters?: KernelOutput; filterDeltas?: KernelOutput; } export type FilterType = new ( settings: Partial, inputLayer: ILayer ) => ILayer; export class Filter extends BaseLayer { get width(): number { return this.inputLayer.width; } get height(): number { return this.inputLayer.height; } get depth(): number { return this.inputLayer.depth; } get filterCount(): number { return this.settings.filterCount as number; } get filterWidth(): number { return this.settings.filterWidth as number; } get filterHeight(): number { return this.settings.filterHeight as number; } get filters(): KernelOutput { return this.settings.filters; } set filters(filters: KernelOutput) { this.settings.filters = filters; } get filterDeltas(): KernelOutput { return this.settings.filterDeltas; } set filterDeltas(filterDeltas: KernelOutput) { this.settings.filterDeltas = filterDeltas; } settings: Partial; inputLayer: ILayer; constructor(settings: Partial, inputLayer: ILayer) { super(); this.settings = settings; this.inputLayer = inputLayer; } } ================================================ FILE: src/layer/fully-connected.test.ts ================================================ import { GPU } from 'gpu.js'; import { gpuMock } from 'gpu-mock.js'; import { predict, predict3D, compareBiases, compareFilterDeltas, compareFilterDeltas3D, compareInputDeltas, compareInputDeltas3D, } from './fully-connected'; import { onePlusPlus2D, zero2D } from '../test-utils'; import { setup, teardown } from '../utilities/kernel'; describe('FullyConnected Layer', () => { beforeEach(() => { setup( new GPU({ mode: 'cpu', }) ); }); afterEach(() => { teardown(); }); describe('.predict (forward propagation)', () => { test('can predict a simple matrix', () => { const weights = [ [1, 2], [3, 4], ]; const filters = [ [1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16], ]; const biases = [0.2, 0.2, 0.2, 0.2]; const kernel = gpuMock(predict, { output: [4], constants: { inputDepth: 1, inputHeight: 2, inputWidth: 2, }, }); expect(kernel(weights, filters, biases)).toEqual( new Float32Array([30.2, 70.2, 110.2, 150.2]) ); }); test('can predict a matrix', () => { const results = gpuMock(predict, { output: [9], constants: { inputDepth: 1, inputHeight: 1, inputWidth: 9, }, })( [[0, 1, 2, 3, 4, 5, 6, 7, 8]], [ [0, 1, 2, 3, 4, 5, 6, 7, 8], [0, 1, 2, 3, 4, 5, 6, 7, 8], [0, 1, 2, 3, 4, 5, 6, 7, 8], [0, 1, 2, 3, 4, 5, 6, 7, 8], [0, 1, 2, 3, 4, 5, 6, 7, 8], [0, 1, 2, 3, 4, 5, 6, 7, 8], [0, 1, 2, 3, 4, 5, 6, 7, 8], [0, 1, 2, 3, 4, 5, 6, 7, 8], [0, 1, 2, 3, 4, 5, 6, 7, 8], ], [0, 1, 2, 3, 4, 5, 6, 7, 8] ); expect(results).toEqual( new Float32Array([204, 205, 206, 207, 208, 209, 210, 211, 212]) ); }); }); describe('.predict3D (forward propagation)', () => { test('can predict a simple matrix', () => { const weights = [ [ [1, 2], [3, 4], ], ]; const filters = [ [1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16], ]; const biases = [0.2, 0.2, 0.2, 0.2]; const kernel = gpuMock(predict3D, { output: [4, 1], constants: { inputDepth: 1, inputHeight: 2, inputWidth: 2, }, }); expect(kernel(weights, filters, biases)).toEqual([ new Float32Array([30.2, 70.2, 110.2, 150.2]), ]); }); test('can predict a matrix', () => { const results = gpuMock(predict3D, { output: [9, 1], constants: { inputDepth: 1, inputHeight: 1, inputWidth: 9, }, })( [[[0, 1, 2, 3, 4, 5, 6, 7, 8]]], [ [0, 1, 2, 3, 4, 5, 6, 7, 8], [0, 1, 2, 3, 4, 5, 6, 7, 8], [0, 1, 2, 3, 4, 5, 6, 7, 8], [0, 1, 2, 3, 4, 5, 6, 7, 8], [0, 1, 2, 3, 4, 5, 6, 7, 8], [0, 1, 2, 3, 4, 5, 6, 7, 8], [0, 1, 2, 3, 4, 5, 6, 7, 8], [0, 1, 2, 3, 4, 5, 6, 7, 8], [0, 1, 2, 3, 4, 5, 6, 7, 8], ], [0, 1, 2, 3, 4, 5, 6, 7, 8] ); expect(results).toEqual([ new Float32Array([204, 205, 206, 207, 208, 209, 210, 211, 212]), ]); }); }); describe('.compareBiases (back propagation)', () => { test('can compare a simple matrix', () => { const biases = [0, 0, 0, 0]; const deltas = [[1, 2, 3, 4]]; const kernel = gpuMock(compareBiases, { output: [4], constants: { connectionCount: 4, }, }); expect(kernel(biases, deltas)).toEqual(new Float32Array([1, 2, 3, 4])); }); test('can add a simple matrix', () => { const biases = [1, 2, 3, 4]; const deltas = [[1, 2, 3, 4]]; const kernel = gpuMock(compareBiases, { output: [4], constants: { connectionCount: 4, }, }); expect(kernel(biases, deltas)).toEqual(new Float32Array([2, 4, 6, 8])); }); }); describe('.compareFilterDeltas (back propagation)', () => { test('can compare a simple matrix', () => { const inputWeights = onePlusPlus2D(4, 4); const deltas = onePlusPlus2D(1, 16); const filterDeltas = zero2D(4, 4); const kernel = gpuMock(compareFilterDeltas, { output: [4, 4], constants: { deltaX: 0, deltaY: 0, deltaWidth: 4, deltaHeight: 4, }, }); expect(kernel(filterDeltas, inputWeights, deltas)).toEqual([ new Float32Array([1, 2, 3, 4]), new Float32Array([5, 6, 7, 8]), new Float32Array([9, 10, 11, 12]), new Float32Array([13, 14, 15, 16]), ]); }); test('can add a simple matrix', () => { const inputWeights = onePlusPlus2D(4, 4); const deltas = onePlusPlus2D(1, 16); const filterDeltas = onePlusPlus2D(4, 4); const kernel = gpuMock(compareFilterDeltas, { output: [4, 4], constants: { deltaX: 0, deltaY: 0, deltaWidth: 4, deltaHeight: 4, }, }); expect(kernel(filterDeltas, inputWeights, deltas)).toEqual([ new Float32Array([2, 4, 6, 8]), new Float32Array([10, 12, 14, 16]), new Float32Array([18, 20, 22, 24]), new Float32Array([26, 28, 30, 32]), ]); }); }); describe('.compareFilterDeltas3D (back propagation)', () => { test('can compare a simplge matrix', () => { const inputWeights = [ [ [1, 2], [3, 4], ], ]; const deltas = [[1, 2, 3, 4]]; const filterDeltas = [ [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], ]; const kernel = gpuMock(compareFilterDeltas3D, { output: [4, 4], constants: { inputWidth: 2, inputHeight: 2, }, }); expect(kernel(filterDeltas, inputWeights, deltas)).toEqual([ new Float32Array([1, 2, 3, 4]), new Float32Array([2, 4, 6, 8]), new Float32Array([3, 6, 9, 12]), new Float32Array([4, 8, 12, 16]), ]); }); test('can add a simplge matrix', () => { const inputWeights = [ [ [1, 2], [3, 4], ], ]; const deltas = [[1, 2, 3, 4]]; const filterDeltas = [ [1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16], ]; const kernel = gpuMock(compareFilterDeltas3D, { output: [4, 4], constants: { inputWidth: 2, inputHeight: 2, }, }); expect(kernel(filterDeltas, inputWeights, deltas)).toEqual([ new Float32Array([2, 4, 6, 8]), new Float32Array([7, 10, 13, 16]), new Float32Array([12, 16, 20, 24]), new Float32Array([17, 22, 27, 32]), ]); }); }); describe('.compareInputDeltas (back propagation)', () => { test('can compare a simple matrix', () => { const inputDeltas = [ [0, 0], [0, 0], ]; const deltas = [[1, 2, 3, 4]]; const filters = [ [1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16], ]; const kernel = gpuMock(compareInputDeltas, { output: [2, 2], constants: { filterCount: 4, }, }); expect(kernel(inputDeltas, deltas, filters)).toEqual([ new Float32Array([90, 100]), new Float32Array([110, 120]), ]); }); test('can add a simple matrix', () => { const inputDeltas = [ [1, 2], [3, 4], ]; const deltas = [[1, 2, 3, 4]]; const filters = [ [1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16], ]; const kernel = gpuMock(compareInputDeltas, { output: [2, 2], constants: { filterCount: 4, }, }); expect(kernel(inputDeltas, deltas, filters)).toEqual([ new Float32Array([91, 102]), new Float32Array([113, 124]), ]); }); }); describe('.compareInputDeltas3D (back propagation)', () => { test('can compare a simple matrix', () => { const inputDeltas = [ [ [0, 0], [0, 0], ], ]; const deltas = [[1, 2, 3, 4]]; const filters = [ [1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16], ]; const kernel = gpuMock(compareInputDeltas3D, { output: [2, 2, 1], constants: { filterCount: 4, }, }); expect(kernel(inputDeltas, deltas, filters)).toEqual([ [new Float32Array([90, 100]), new Float32Array([110, 120])], ]); }); test('can add a simple matrix', () => { const inputDeltas = [ [ [1, 2], [3, 4], ], ]; const deltas = [[1, 2, 3, 4]]; const filters = [ [1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16], ]; const kernel = gpuMock(compareInputDeltas3D, { output: [2, 2, 1], constants: { filterCount: 4, }, }); expect(kernel(inputDeltas, deltas, filters)).toEqual([ [new Float32Array([91, 102]), new Float32Array([113, 124])], ]); }); }); }); ================================================ FILE: src/layer/fully-connected.ts ================================================ import { IConstantsThis, IKernelFunctionThis, IKernelRunShortcut, KernelOutput, } from 'gpu.js'; import { Filter, IFilterSettings } from './filter'; import { makeKernel, release } from '../utilities/kernel'; import { values } from '../utilities/values'; import { randos2D, randos3D } from '../utilities/randos'; import { zeros } from '../utilities/zeros'; import { zeros2D } from '../utilities/zeros-2d'; import { zeros3D } from '../utilities/zeros-3d'; import { ILayer } from './base-layer'; export interface IPredictConstants extends IConstantsThis { inputWidth: number; inputHeight: number; } export function predict( this: IKernelFunctionThis, inputs: number[][], filters: number[][], biases: number[] ): number { let output = 0; let i = 0; for (let y = 0; y < this.constants.inputHeight; y++) { for (let x = 0; x < this.constants.inputWidth; x++) { output += inputs[y][x] * filters[this.thread.x][i]; i++; } } return output + biases[this.thread.x]; } export function predict3D( this: IKernelFunctionThis, inputs: number[][][], filters: number[][], biases: number[] ): number { let output = 0; let i = 0; for (let z = 0; z < this.constants.inputDepth; z++) { for (let y = 0; y < this.constants.inputHeight; y++) { for (let x = 0; x < this.constants.inputWidth; x++) { output += inputs[z][y][x] * filters[this.thread.x][i]; i++; } } } return output + biases[this.thread.x]; } export interface ICompareInputDeltasConstants extends IConstantsThis { filterCount: number; } export function compareInputDeltas( this: IKernelFunctionThis, inputDeltas: number[][], deltas: number[][], filters: number[][] ): number { let sum = 0; const filterX = this.thread.x + this.thread.y * this.output.x; for (let filterY = 0; filterY < this.constants.filterCount; filterY++) { sum += filters[filterY][filterX] * deltas[0][filterY]; } return sum + inputDeltas[this.thread.y][this.thread.x]; } export function compareInputDeltas3D( this: IKernelFunctionThis, inputDeltas: number[][][], deltas: number[][], filters: number[][] ): number { let sum = 0; const filterX = this.thread.x + this.thread.y * this.output.x; for (let filterY = 0; filterY < this.constants.filterCount; filterY++) { sum += filters[filterY][filterX] * deltas[0][filterY]; } return sum + inputDeltas[this.thread.z][this.thread.y][this.thread.x]; } export function compareBiases( this: IKernelFunctionThis, biases: number[], deltas: number[][] ): number { return biases[this.thread.x] + deltas[this.thread.y][this.thread.x]; } export interface ICompareFiltersDeltas extends IConstantsThis { deltaX: number; deltaY: number; inputWidth: number; inputHeight: number; } export function compareFilterDeltas( this: IKernelFunctionThis, filterDeltas: number[][], inputWeights: number[][], deltas: number[][] ): number { return ( filterDeltas[this.thread.y][this.thread.x] + inputWeights[this.thread.y][this.thread.x] * deltas[this.constants.deltaY][this.constants.deltaX] ); } export function compareFilterDeltas3D( this: IKernelFunctionThis, filterDeltas: number[][], inputWeights: number[][][], deltas: number[][] ): number { const inputZ = Math.floor( this.thread.x / (this.constants.inputWidth * this.constants.inputHeight) ); const inputY = Math.floor( (this.thread.x - inputZ * this.constants.inputWidth * this.constants.inputHeight) / this.constants.inputWidth ); const inputX = this.thread.x - this.constants.inputWidth * (inputY + this.constants.inputHeight * inputZ); return ( filterDeltas[this.thread.y][this.thread.x] + inputWeights[inputZ][inputY][inputX] * deltas[0][this.thread.y] ); } export interface IFullyConnectedDefaultSettings extends Partial { bias?: number; biases?: KernelOutput; biasDeltas?: KernelOutput; } export const defaults: IFullyConnectedDefaultSettings = { bias: 0.1, }; export class FullyConnected extends Filter { get bias(): number { return this.settings.bias as number; } get biases(): KernelOutput { return this.settings.biases; } set biases(biases: KernelOutput) { this.settings.biases = biases; } get biasDeltas(): KernelOutput { return this.settings.biases; } set biasDeltas(biasDeltas: KernelOutput) { this.settings.biasDeltas = biasDeltas; } settings: Partial; compareFilterDeltasKernel: IKernelRunShortcut | null = null; compareInputDeltasKernel: IKernelRunShortcut | null = null; compareBiasesKernel: IKernelRunShortcut | null = null; constructor( settings: Partial, inputLayer: ILayer ) { super(settings, inputLayer); this.settings = { ...settings }; this.validate(); const connectionCount = inputLayer.width * inputLayer.height * inputLayer.depth; this.biases = values(this.height, this.bias); this.biasDeltas = zeros(this.height); this.filters = randos2D(connectionCount, this.height); this.filterDeltas = zeros2D(connectionCount, this.height); if (this.depth > 0) { this.weights = randos3D(this.width, this.height, this.depth); this.deltas = zeros3D(this.width, this.height, this.depth); } else if (this.height > 0) { this.weights = randos2D(this.width, this.height); this.deltas = zeros2D(this.width, this.height); } } validate(): void { super.validate(); if (this.depth > 0) throw new Error('depth not supported'); } setupKernels(): void { const { inputLayer } = this; const connectionCount = inputLayer.width * inputLayer.height * inputLayer.depth; if (inputLayer.depth > 0) { this.predictKernel = makeKernel(predict3D, { output: [this.width, this.height], constants: { inputHeight: inputLayer.height, inputWidth: inputLayer.width, inputDepth: inputLayer.depth, }, }); this.compareFilterDeltasKernel = makeKernel(compareFilterDeltas3D, { output: [connectionCount, this.height], constants: { deltaX: 0, deltaY: 0, inputWidth: inputLayer.width, inputHeight: inputLayer.height, }, immutable: true, }); this.compareInputDeltasKernel = makeKernel(compareInputDeltas3D, { output: [inputLayer.width, inputLayer.height, inputLayer.depth], constants: { filterCount: this.height, }, immutable: true, }); } else { this.predictKernel = makeKernel(predict, { output: [this.width, this.height], constants: { inputHeight: inputLayer.height, inputWidth: inputLayer.width, }, }); this.compareFilterDeltasKernel = makeKernel(compareFilterDeltas, { output: [connectionCount, this.height], constants: { deltaX: 0, deltaY: 0, inputWidth: inputLayer.width, inputHeight: inputLayer.height, }, }); this.compareInputDeltasKernel = makeKernel(compareInputDeltas, { output: [inputLayer.width, inputLayer.height], constants: { filterCount: this.height, }, }); } this.compareBiasesKernel = makeKernel(compareBiases, { output: [this.width, this.height], }); } predict(): void { this.weights = (this.predictKernel as IKernelRunShortcut)( this.inputLayer.weights, this.filters, this.biases ); } compare(): void { const inputLayerDeltas = this.inputLayer.deltas; this.inputLayer.deltas = (this .compareInputDeltasKernel as IKernelRunShortcut)( inputLayerDeltas, this.deltas, this.filters ); release(inputLayerDeltas); const { biasDeltas, filterDeltas } = this; // TODO: handle biasDeltas learn this.biasDeltas = (this.compareBiasesKernel as IKernelRunShortcut)( this.biases, this.deltas ); // TODO: handle filterDeltas learn this.filterDeltas = (this.compareFilterDeltasKernel as IKernelRunShortcut)( filterDeltas, this.inputLayer.weights, this.deltas ); release(biasDeltas); release(filterDeltas); } } export function fullyConnected( settings: IFullyConnectedDefaultSettings, inputLayer: ILayer ): FullyConnected { return new FullyConnected(settings, inputLayer); } ================================================ FILE: src/layer/gru.ts ================================================ import { add } from './add'; import { negative } from './negative'; import { multiply } from './multiply'; import { multiplyElement } from './multiply-element'; import { ones } from './ones'; import { sigmoid } from './sigmoid'; import { random } from './random'; import { tanh } from './tanh'; import { zeros } from './zeros'; import { ILayer, ILayerSettings } from './base-layer'; import { RecurrentInput } from './recurrent-input'; export function gru( settings: ILayerSettings, recurrentInput: RecurrentInput, input: ILayer ): ILayer { const { height } = settings; const updateGateWeights = random({ height, width: input.height }); const updateGatePeepholes = random({ width: height, height }); const updateGateBias = zeros({ height }); const updateGate = sigmoid( add( add( multiply(updateGateWeights, input), multiply(updateGatePeepholes, recurrentInput) ), updateGateBias ) ); const resetGateWeights = random({ height, width: input.height }); const resetGatePeepholes = random({ width: height, height }); const resetGateBias = zeros({ height }); const resetGate = sigmoid( add( add( multiply(resetGateWeights, input), multiply(resetGatePeepholes, recurrentInput) ), resetGateBias ) ); const cellWeights = random({ height, width: input.height }); const cellPeepholes = random({ width: height, height }); const cellBias = zeros({ height }); const cell = tanh( add( add( multiply(cellWeights, input), multiply(cellPeepholes, multiplyElement(resetGate, recurrentInput)) ), cellBias ) ); // compute hidden state as gated, saturated cell activations // negate updateGate return add( multiplyElement( add( ones({ width: updateGate.width, height: updateGate.height }), negative(updateGate) ), cell ), multiplyElement(recurrentInput, updateGate) ); } ================================================ FILE: src/layer/index.ts ================================================ import { Activation, EntryPoint, Filter, Internal, InternalModel, Model, Modifier, Operator, Target, } from './types'; export { Add, add } from './add'; export { arthurFeedForward } from './arthur-feed-forward'; export { BaseLayer, ILayer, ILayerSettings, ILayerJSON, baseLayerDefaultSettings, } from './base-layer'; export { Convolution, convolution } from './convolution'; export { Dropout, dropout } from './dropout'; export { feedForward } from './feed-forward'; export { FullyConnected, fullyConnected } from './fully-connected'; export { gru } from './gru'; export { Input, input } from './input'; export { LeakyRelu, leakyRelu } from './leaky-relu'; export { lstmCell } from './lstm-cell'; export { Multiply, multiply } from './multiply'; export { MultiplyElement, multiplyElement } from './multiply-element'; export { Negative, negative } from './negative'; export { Ones, ones } from './ones'; export { output } from './output'; export { Pool, pool } from './pool'; export { Random, random } from './random'; export { RecurrentInput, IRecurrentInput } from './recurrent-input'; export { RecurrentZeros } from './recurrent-zeros'; export { rnnCell } from './rnn-cell'; export { Regression, regression } from './regression'; export { Relu, relu } from './relu'; export { Sigmoid, sigmoid } from './sigmoid'; export { SoftMax, softMax } from './soft-max'; export { SVM, svm } from './svm'; export { Tanh, tanh } from './tanh'; export { Target, target } from './target'; export { Transpose, transpose } from './transpose'; export { Zeros, zeros } from './zeros'; export const layerTypes = { Activation, Internal, InternalModel, EntryPoint, Filter, Model, Modifier, Operator, Target, }; ================================================ FILE: src/layer/input.test.ts ================================================ import { GPU } from 'gpu.js'; import { Input } from './input'; import { setup, teardown } from '../utilities/kernel'; describe('Input Layer', () => { beforeEach(() => { setup( new GPU({ mode: 'cpu', }) ); }); afterEach(() => { teardown(); }); describe('.predict (forward propagation)', () => { test('can handle 1D inputs', () => { const input = new Input({ height: 10 }); input.setupKernels(); expect(input.predict).toEqual(Input.prototype.predict1D); }); test('can handle 2D inputs', () => { const input = new Input({ width: 10, height: 10 }); input.setupKernels(); expect(input.predict).toEqual(Input.prototype.predict); }); }); }); ================================================ FILE: src/layer/input.ts ================================================ import { IKernelFunctionThis, IKernelRunShortcut, KernelOutput } from 'gpu.js'; import { EntryPoint } from './types'; import { ILayer, ILayerSettings } from './base-layer'; import { zeros2D } from '../utilities/zeros-2d'; import { makeKernel, release, kernelInput, clone } from '../utilities/kernel'; export const defaults: ILayerSettings = { weights: null, }; export class Input extends EntryPoint { reshapeInput: IKernelRunShortcut | null = null; constructor(settings: ILayerSettings) { super({ ...defaults, ...settings }); this.validate(); this.reshapeInput = null; this.deltas = zeros2D(this.width, this.height); } setupKernels(): void { if (this.width === 1) { this.predict = this.predict1D; this.reshapeInput = makeKernel( function (this: IKernelFunctionThis, value: number[]) { return value[this.thread.y]; }, { output: [1, this.height], immutable: true, } ); } } reuseKernels(layer: ILayer): void { // super.reuseKernels(layer); this.reshapeInput = (layer as Input).reshapeInput; } predict(inputs: KernelOutput): void { if ( (Array.isArray(inputs) || inputs instanceof Float32Array) && typeof inputs[0] === 'number' && inputs.length === this.height * this.width ) { release(this.weights); this.weights = kernelInput(inputs as number[], [this.width, this.height]); } else if ( Array.isArray(inputs) && inputs.length === this.height && (Array.isArray(inputs[0]) || inputs[0] instanceof Float32Array) && inputs[0].length === this.width ) { this.weights = clone(inputs); } else { throw new Error('Inputs are not of sized correctly'); } } predict1D(inputs: KernelOutput): void { if (this.weights) release(this.weights); if (this.reshapeInput) { this.weights = this.reshapeInput(inputs); } else { this.weights = inputs; } } compare(): void { // throw new Error(`${this.constructor.name}-compare is not yet implemented`) } learn(): void {} } export function input(settings: ILayerSettings): Input { return new Input(settings); } ================================================ FILE: src/layer/internal.ts ================================================ import { BaseLayer, ILayer, ILayerJSON, ILayerSettings } from './base-layer'; import { IKernelRunShortcut, Input, KernelOutput } from 'gpu.js'; import { IPraxis } from '../praxis/base-praxis'; export type InternalType = new (settings: Partial) => ILayer; export abstract class Internal implements ILayer { abstract settings: ILayerSettings; abstract predict(inputs?: KernelOutput): void; abstract compare(targetValues?: KernelOutput): void; abstract learn(learningRate?: number): void; abstract setupKernels(training?: boolean): void; predictKernel: IKernelRunShortcut | null = null; compareKernel: IKernelRunShortcut | null = null; praxis: IPraxis | null = null; get width(): number { return this.settings.width as number; } get height(): number { return this.settings.height as number; } get depth(): number { return this.settings.depth as number; } get weights(): KernelOutput | Input { return this.settings.weights as KernelOutput; } set weights(weights: KernelOutput | Input) { this.settings.weights = weights as KernelOutput; } get deltas(): KernelOutput { return this.settings.deltas as KernelOutput; } set deltas(deltas: KernelOutput) { this.settings.deltas = deltas; } toJSON(): Partial { return BaseLayer.toJSON(this); } abstract reuseKernels(layer: ILayer): void; } ================================================ FILE: src/layer/leaky-relu.test.ts ================================================ import { gpuMock } from 'gpu-mock.js'; import { GPU } from 'gpu.js'; import * as leakyReluActivation from '../activation/leaky-relu'; import { ILayerSettings } from './base-layer'; import { compare2D, compare3D, LeakyRelu, leakyRelu, predict2D, predict3D, } from './leaky-relu'; import { makeKernel, setup, teardown } from '../utilities/kernel'; import { ones2D } from '../utilities/ones'; import { randos2D } from '../utilities/randos'; import { IWithCompareKernel, IWithPredictKernel, mockLayer, mockPraxis, } from '../test-utils'; jest.mock('../../src/utilities/kernel', () => { return { setup: jest.fn(), teardown: jest.fn(), makeKernel: jest.fn(() => { return [[1]]; }), release: jest.fn(), clear: jest.fn(), }; }); describe('Leaky Relu Layer', () => { describe('predict2D() (forward propagation)', () => { test('can leaky relu a simple matrix', () => { const inputs = [ [0.1, -0.2, 0.3], [-0.4, 0.5, -0.6], [0.7, -0.8, 0.9], ]; const results = gpuMock(predict2D, { output: [3, 3], })(inputs); expect(results).toEqual([ new Float32Array([0.1, -0.002, 0.3]), new Float32Array([-0.004, 0.5, -0.006]), new Float32Array([0.7, -0.008, 0.9]), ]); }); }); describe('predict3D() (forward propagation)', () => { test('can leaky relu a simple matrix', () => { const inputs = [ [ [0.1, -0.2, 0.3], [-0.4, 0.5, -0.6], [0.7, -0.8, 0.9], ], [ [0.1, -0.2, 0.3], [-0.4, 0.5, -0.6], [0.7, -0.8, 0.9], ], ]; const results = gpuMock(predict3D, { output: [3, 3, 2], })(inputs); expect(results).toEqual([ [ new Float32Array([0.1, -0.002, 0.3]), new Float32Array([-0.004, 0.5, -0.006]), new Float32Array([0.7, -0.008, 0.9]), ], [ new Float32Array([0.1, -0.002, 0.3]), new Float32Array([-0.004, 0.5, -0.006]), new Float32Array([0.7, -0.008, 0.9]), ], ]); }); }); describe('compare2D() (back propagation)', () => { test('can leaky relu a simple matrix', () => { const inputs = [ [0.1, -0.2, 0.3], [-0.4, 0.5, -0.6], [0.7, -0.8, 0.9], ]; const deltas = [ [1, 1, 1], [1, 1, 1], [1, 1, 1], ]; const results = gpuMock(compare2D, { output: [3, 3], })(inputs, deltas); expect(results).toEqual([ new Float32Array([1, 0.01, 1]), new Float32Array([0.01, 1, 0.01]), new Float32Array([1, 0.01, 1]), ]); }); }); describe('compare3D() (back propagation)', () => { test('can leaky relu a simple matrix', () => { const inputs = [ [ [0.1, -0.2, 0.3], [-0.4, 0.5, -0.6], [0.7, -0.8, 0.9], ], [ [0.1, -0.2, 0.3], [-0.4, 0.5, -0.6], [0.7, -0.8, 0.9], ], ]; const deltas = [ [ [1, 1, 1], [1, 1, 1], [1, 1, 1], ], [ [1, 1, 1], [1, 1, 1], [1, 1, 1], ], ]; const results = gpuMock(compare3D, { output: [3, 3, 2], })(inputs, deltas); expect(results).toEqual([ [ new Float32Array([1, 0.01, 1]), new Float32Array([0.01, 1, 0.01]), new Float32Array([1, 0.01, 1]), ], [ new Float32Array([1, 0.01, 1]), new Float32Array([0.01, 1, 0.01]), new Float32Array([1, 0.01, 1]), ], ]); }); }); describe('.setupKernels()', () => { beforeEach(() => { setup( new GPU({ mode: 'cpu', }) ); }); afterEach(() => { teardown(); }); describe('2d', () => { it('sets up kernels correctly', () => { const width = 3; const height = 4; const mockInputLayer = mockLayer({ width, height }); const l = new LeakyRelu(mockInputLayer); expect(l.predictKernel).toBe(null); expect(l.compareKernel).toBe(null); l.setupKernels(); expect(l.predictKernel).not.toBe(null); expect(l.compareKernel).not.toBe(null); expect(makeKernel).toHaveBeenCalledWith(predict2D, { functions: [leakyReluActivation.activate], immutable: true, output: [3, 4], }); expect(makeKernel).toHaveBeenCalledWith(compare2D, { functions: [leakyReluActivation.measure], immutable: true, output: [3, 4], }); }); }); describe('3d', () => { it('sets up kernels correctly', () => { const width = 3; const height = 4; const depth = 5; const mockInputLayer = mockLayer({ width, height, depth }); const l = new LeakyRelu(mockInputLayer); expect(l.predictKernel).toBe(null); expect(l.compareKernel).toBe(null); l.setupKernels(); expect(l.predictKernel).not.toBe(null); expect(l.compareKernel).not.toBe(null); expect(makeKernel).toHaveBeenCalledWith(predict3D, { functions: [leakyReluActivation.activate], immutable: true, output: [3, 4, 5], }); expect(makeKernel).toHaveBeenCalledWith(compare3D, { functions: [leakyReluActivation.measure], immutable: true, output: [3, 4, 5], }); }); }); }); describe('.predict()', () => { it('calls this.predictKernel() with this.inputLayer.weights', () => { const mockWeights = ones2D(1, 1); const mockInputLayer = mockLayer({ weights: mockWeights, width: 1, height: 1, depth: 1, }); const l = new LeakyRelu(mockInputLayer); const spy = (((l as unknown) as IWithPredictKernel).predictKernel = jest.fn( (values) => values )); l.predict(); expect(spy).toBeCalledWith(mockWeights); expect(l.weights).toBe(mockWeights); }); }); describe('.compare()', () => { it('calls this.compareKernel() with this.inputLayer.weights & this.inputLayer.deltas', () => { const mockInputLayer = mockLayer({ width: 1, height: 1, depth: 1, }); const l = new LeakyRelu(mockInputLayer); const weights = (l.weights = randos2D(1, 1)); const deltas = (l.deltas = randos2D(1, 1)); const results = randos2D(1, 1); ((l as unknown) as IWithCompareKernel).compareKernel = jest.fn( (weights, deltas) => results ); l.compare(); expect(l.compareKernel).toBeCalledWith(weights, deltas); expect(l.deltas).toBe(results); }); }); describe('leakyRelu lambda', () => { test('creates a new instance of LeakyRelu', () => { const width = 3; const height = 4; const depth = 5; const mockInputLayer = mockLayer({ width, height, depth }); const praxis = mockPraxis(mockInputLayer); const praxisSettings = {}; const settings: ILayerSettings = { praxisOpts: praxisSettings, initPraxis: jest.fn((settings: typeof praxisSettings) => { return praxis; }), }; const l = leakyRelu(mockInputLayer, settings); expect(l.constructor).toBe(LeakyRelu); expect(l.width).toBe(width); expect(l.height).toBe(height); expect(l.depth).toBe(depth); expect(l.praxis).toBe(praxis); }); }); }); ================================================ FILE: src/layer/leaky-relu.ts ================================================ import { Activation } from './types'; import { makeKernel, release } from '../utilities/kernel'; import { activate, measure } from '../activation/leaky-relu'; import { IKernelFunctionThis, IKernelRunShortcut } from 'gpu.js'; import { ILayer, ILayerSettings } from './base-layer'; export function predict2D( this: IKernelFunctionThis, inputs: number[][] ): number { return activate(inputs[this.thread.y][this.thread.x]); } export function predict3D( this: IKernelFunctionThis, inputs: number[][][] ): number { return activate(inputs[this.thread.z][this.thread.y][this.thread.x]); } export function compare2D( this: IKernelFunctionThis, weights: number[][], deltas: number[][] ): number { return measure( weights[this.thread.y][this.thread.x], deltas[this.thread.y][this.thread.x] ); } export function compare3D( this: IKernelFunctionThis, weights: number[][][], deltas: number[][][] ): number { return measure( weights[this.thread.z][this.thread.y][this.thread.x], deltas[this.thread.z][this.thread.y][this.thread.x] ); } export class LeakyRelu extends Activation { setupKernels(): void { const { width, height, depth } = this.inputLayer; if (this.depth > 0) { this.predictKernel = makeKernel(predict3D, { output: [width, height, depth], functions: [activate], immutable: true, }); this.compareKernel = makeKernel(compare3D, { output: [width, height, depth], functions: [measure], immutable: true, }); } else { this.predictKernel = makeKernel(predict2D, { output: [width, height], functions: [activate], immutable: true, }); this.compareKernel = makeKernel(compare2D, { output: [width, height], functions: [measure], immutable: true, }); } } predict(): void { release(this.weights); this.weights = (this.predictKernel as IKernelRunShortcut)( this.inputLayer.weights ); } compare(): void { const { deltas } = this; this.deltas = (this.compareKernel as IKernelRunShortcut)( this.weights, deltas ); release(deltas); } } export function leakyRelu( inputLayer: ILayer, settings: ILayerSettings ): LeakyRelu { return new LeakyRelu(inputLayer, settings); } ================================================ FILE: src/layer/lstm-cell.test.ts ================================================ import { Add, lstmCell, Multiply, MultiplyElement, Random, RecurrentZeros, Sigmoid, Tanh, Zeros, } from '../../src/layer'; import { mockLayer, onePlusPlus2D, TestLayer } from '../test-utils'; import { flattenLayers } from '../utilities/flatten-layers'; import { ILayer } from './base-layer'; import { setup, teardown } from '../utilities/kernel'; import { GPU } from 'gpu.js'; describe('lstm Cell', () => { beforeEach(() => { setup( new GPU({ mode: 'cpu', }) ); }); afterEach(() => { teardown(); }); it('properly sets width and height', () => { const input = mockLayer({ width: 1, height: 3 }); const settings = { height: 3 }; const recurrentInput = new RecurrentZeros(); const layer = lstmCell(settings, input, recurrentInput); expect(layer.width).toEqual(1); expect(layer.height).toEqual(settings.height); }); it('throws if height is not a number', () => { const input = mockLayer({ width: 1, height: 3 }); const settings = { height: null }; const recurrentInput = new RecurrentZeros(); expect(() => { lstmCell(settings, input, recurrentInput); }).toThrow(); }); describe('when .setDimensions is available', () => { let setDimensionsSpy: jest.SpyInstance; beforeEach(() => { setDimensionsSpy = jest.spyOn(RecurrentZeros.prototype, 'setDimensions'); }); afterEach(() => { setDimensionsSpy.mockRestore(); }); it('is called', () => { const input = mockLayer({ width: 1, height: 3 }); const settings = { height: 33 }; const recurrentInput = new RecurrentZeros(); lstmCell(settings, input, recurrentInput); expect(setDimensionsSpy).toHaveBeenCalledWith(1, 33); }); }); it('properly sets up layers', () => { const input = mockLayer({ width: 1, height: 3 }); const settings = { height: 3 }; const recurrentInput = new RecurrentZeros(); const layer = lstmCell(settings, input, recurrentInput); const layers = flattenLayers([layer]); expect(layers.length).toBe(39); // output gate expect(layers[0]).toBeInstanceOf(Random); expect(layers[0].id).toBe('outputGateWeights'); expect((layers[0] as Random).settings.std).toBe(0.08); expect(layers[1]).toBeInstanceOf(TestLayer); expect(layers[1].id).toBe('MockLayer'); expect(layers[2]).toBeInstanceOf(Multiply); expect(layers[3]).toBeInstanceOf(Random); expect(layers[3].id).toBe('outputGatePeepholes'); expect((layers[3] as Random).settings.std).toBe(0.08); expect(layers[4]).toBeInstanceOf(RecurrentZeros); expect(layers[5]).toBeInstanceOf(Multiply); expect(layers[6]).toBeInstanceOf(Add); expect(layers[7]).toBeInstanceOf(Zeros); expect(layers[7].id).toBe('outputGateBias'); expect(layers[8]).toBeInstanceOf(Add); expect(layers[9]).toBeInstanceOf(Sigmoid); expect(layers[9].id).toBe('outputGate'); // forget gate expect(layers[10]).toBeInstanceOf(Random); expect(layers[10].id).toBe('forgetGateWeights'); expect((layers[10] as Random).settings.std).toBe(0.08); expect(layers[11]).toBeInstanceOf(Multiply); expect(layers[12]).toBeInstanceOf(Random); expect(layers[12].id).toBe('forgetGatePeepholes'); expect((layers[12] as Random).settings.std).toBe(0.08); expect(layers[13]).toBeInstanceOf(Multiply); expect(layers[14]).toBeInstanceOf(Add); expect(layers[14].id).toBe(''); expect(layers[15]).toBeInstanceOf(Zeros); expect(layers[15].id).toBe('forgetGateBias'); expect(layers[16]).toBeInstanceOf(Add); expect(layers[17]).toBeInstanceOf(Sigmoid); expect(layers[17].id).toBe('forgetGate'); // input gate expect(layers[18]).toBeInstanceOf(MultiplyElement); expect(layers[18].id).toBe('retainCell'); expect(layers[19]).toBeInstanceOf(Random); expect(layers[19].id).toBe('inputGateWeights'); expect((layers[19] as Random).settings.std).toBe(0.08); expect(layers[20]).toBeInstanceOf(Multiply); expect(layers[21]).toBeInstanceOf(Random); expect(layers[21].id).toBe('inputGatePeepholes'); expect((layers[21] as Random).settings.std).toBe(0.08); expect(layers[22]).toBeInstanceOf(Multiply); expect(layers[23]).toBeInstanceOf(Add); expect(layers[24]).toBeInstanceOf(Zeros); expect(layers[24].id).toBe('inputGateBias'); expect(layers[25]).toBeInstanceOf(Add); expect(layers[26]).toBeInstanceOf(Sigmoid); expect(layers[26].id).toBe('inputGate'); // memory expect(layers[27]).toBeInstanceOf(Random); expect(layers[27].id).toBe('memoryWeights'); expect((layers[27] as Random).settings.std).toBe(0.08); expect(layers[28]).toBeInstanceOf(Multiply); expect(layers[29]).toBeInstanceOf(Random); expect(layers[29].id).toBe('memoryPeepholes'); expect((layers[29] as Random).settings.std).toBe(0.08); expect(layers[30]).toBeInstanceOf(Multiply); expect(layers[31]).toBeInstanceOf(Add); expect(layers[32]).toBeInstanceOf(Zeros); expect(layers[32].id).toBe('memoryBias'); expect(layers[33]).toBeInstanceOf(Add); expect(layers[34]).toBeInstanceOf(Tanh); expect(layers[34].id).toBe('memory'); // writeCell expect(layers[35]).toBeInstanceOf(MultiplyElement); expect(layers[35].id).toBe('writeCell'); // cell expect(layers[36]).toBeInstanceOf(Add); expect(layers[36].id).toBe('cell'); expect(layers[37]).toBeInstanceOf(Tanh); // activations expect(layers[38]).toBeInstanceOf(MultiplyElement); expect(layers[38].id).toBe('activations'); }); describe('result', () => { it('arrives at correct output', () => { const input = mockLayer({ width: 1, height: 3 }); input.weights = onePlusPlus2D(input.width, input.height); const settings = { height: 3 }; const recurrentInput = new RecurrentZeros(); const layer = lstmCell(settings, input, recurrentInput); const layers = flattenLayers([layer]); recurrentInput.weights = onePlusPlus2D( recurrentInput.width, recurrentInput.height ); const memoryLayers = layers.filter( (layer: ILayer) => layer instanceof Random ); memoryLayers.forEach((layer: ILayer) => { layer.weights = onePlusPlus2D(layer.width, layer.height); }); layers.forEach((layer) => layer.setupKernels()); layers.forEach((layer) => layer.predict()); expect(layers[layers.length - 1].weights).toEqual([ Float32Array.from([0.9640275835990906]), Float32Array.from([0.9950547814369202]), Float32Array.from([0.9993293285369873]), ]); }); }); }); ================================================ FILE: src/layer/lstm-cell.ts ================================================ import { add } from './add'; import { multiply } from './multiply'; import { multiplyElement } from './multiply-element'; import { random } from './random'; import { sigmoid } from './sigmoid'; import { tanh } from './tanh'; import { zeros } from './zeros'; import { ILayer, ILayerSettings } from './base-layer'; import { IRecurrentInput } from './recurrent-input'; export function lstmCell( settings: ILayerSettings, input: ILayer, recurrentInput: IRecurrentInput ): ILayer { const { height } = settings; if (typeof height !== 'number') { throw new Error('no settings.height given'); } if (recurrentInput.setDimensions) { recurrentInput.setDimensions(1, height); } const inputGateWeights = random({ width: input.height, height, std: 0.08, id: 'inputGateWeights', }); const inputGatePeepholes = random({ width: height, height, std: 0.08, id: 'inputGatePeepholes', }); const inputGateBias = zeros({ width: 1, height, id: 'inputGateBias' }); const inputGate = sigmoid( add( add( multiply(inputGateWeights, input), multiply(inputGatePeepholes, recurrentInput) ), inputGateBias ), { id: 'inputGate' } ); const forgetGateWeights = random({ width: input.height, height, std: 0.08, id: 'forgetGateWeights', }); const forgetGatePeepholes = random({ width: height, height, std: 0.08, id: 'forgetGatePeepholes', }); const forgetGateBias = zeros({ width: 1, height, id: 'forgetGateBias' }); const forgetGate = sigmoid( add( add( multiply(forgetGateWeights, input), multiply(forgetGatePeepholes, recurrentInput) ), forgetGateBias ), { id: 'forgetGate' } ); const outputGateWeights = random({ width: input.height, height, std: 0.08, id: 'outputGateWeights', }); const outputGatePeepholes = random({ width: height, height, std: 0.08, id: 'outputGatePeepholes', }); const outputGateBias = zeros({ width: 1, height, id: 'outputGateBias' }); const outputGate = sigmoid( add( add( multiply(outputGateWeights, input), multiply(outputGatePeepholes, recurrentInput) ), outputGateBias ), { id: 'outputGate' } ); const memoryWeights = random({ width: input.height, height, std: 0.08, id: 'memoryWeights', }); const memoryPeepholes = random({ width: height, height, std: 0.08, id: 'memoryPeepholes', }); const memoryBias = zeros({ width: 1, height, id: 'memoryBias' }); const memory = tanh( add( add( multiply(memoryWeights, input), multiply(memoryPeepholes, recurrentInput) ), memoryBias ), { id: 'memory' } ); // compute new cell activation const retainCell = multiplyElement(forgetGate, recurrentInput, { id: 'retainCell', }); // what do we keep from cell const writeCell = multiplyElement(inputGate, memory, { id: 'writeCell' }); // what do we write to cell const cell = add(retainCell, writeCell, { id: 'cell' }); // new cell contents // compute hidden state as gated, saturated cell activations return multiplyElement(outputGate, tanh(cell), { id: 'activations' }); } ================================================ FILE: src/layer/modifier.ts ================================================ import { BaseLayer, ILayer, ILayerSettings } from './base-layer'; export type ModifierType = new ( inputLayer: ILayer, settings?: Partial ) => ILayer; export class Modifier extends BaseLayer { inputLayer: ILayer; constructor(inputLayer: ILayer, settings?: Partial) { super({ ...settings, width: inputLayer.width, height: inputLayer.height, depth: inputLayer.depth, }); this.inputLayer = inputLayer; } validate(): void { super.validate(); if (this.width !== this.inputLayer.width) { throw new Error( `width of ${this.width} does not match inputLayer.width of ${this.inputLayer.width}` ); } if (this.height !== this.inputLayer.height) { throw new Error( `height of ${this.height} does not match inputLayer.height of ${this.inputLayer.height}` ); } if (this.depth !== (this.inputLayer.depth ?? 0)) { throw new Error( `depth of ${this.depth} does not match inputLayer.depth of ${this.inputLayer.depth}` ); } } } ================================================ FILE: src/layer/multiply-element.test.ts ================================================ import { GPU } from 'gpu.js'; import { gpuMock } from 'gpu-mock.js'; import { MultiplyElement, multiplyElement, predict, compare, } from './multiply-element'; import { setup, teardown } from '../utilities/kernel'; import { mockLayer, mockTexture } from '../test-utils'; import { ILayer } from './base-layer'; describe('MultiplyElement Layer', () => { beforeEach(() => { setup( new GPU({ mode: 'cpu', }) ); }); afterEach(() => { teardown(); }); describe('.constructor', () => { let mockInputLayer1: ILayer; let mockInputLayer2: ILayer; let layer: MultiplyElement; beforeEach(() => { mockInputLayer1 = mockLayer({ width: 3, height: 2 }); mockInputLayer2 = mockLayer({ width: 3, height: 2 }); layer = new MultiplyElement(mockInputLayer1, mockInputLayer2); }); test('sets inputLayer1 and inputLayer2', () => { expect(layer.inputLayer1).toBe(mockInputLayer1); expect(layer.inputLayer2).toBe(mockInputLayer2); }); test('gets its dimensions from the first inputLayer', () => { expect(layer.width).toBe(mockInputLayer1.width); expect(layer.height).toBe(mockInputLayer1.height); }); test('throws if widths are mismatched', () => { mockInputLayer1 = mockLayer({ width: 3, height: 2 }); mockInputLayer2 = mockLayer({ width: 1, height: 2 }); expect(() => { layer = new MultiplyElement(mockInputLayer1, mockInputLayer2); }).toThrow(); }); test('throws if heights are mismatched', () => { mockInputLayer1 = mockLayer({ width: 3, height: 2 }); mockInputLayer2 = mockLayer({ width: 3, height: 1 }); expect(() => { layer = new MultiplyElement(mockInputLayer1, mockInputLayer2); }).toThrow(); }); test('.weights are set to same as inputLayer as zeros', () => { expect(layer.weights).toEqual([ new Float32Array([0, 0, 0]), new Float32Array([0, 0, 0]), ]); }); test('.deltas are set to same as inputLayer as zeros', () => { expect(layer.deltas).toEqual([ new Float32Array([0, 0, 0]), new Float32Array([0, 0, 0]), ]); }); }); describe('.predict (forward propagation)', () => { let mockInputLayer1: ILayer; let mockInputLayer2: ILayer; let layer: MultiplyElement; beforeEach(() => { mockInputLayer1 = mockLayer({ width: 3, height: 2, weights: [ [1, 2, 3], [4, 5, 6], ], }); mockInputLayer2 = mockLayer({ width: 3, height: 2, weights: [ [7, 8, 9], [10, 11, 12], ], }); layer = new MultiplyElement(mockInputLayer1, mockInputLayer2); layer.setupKernels(); }); it('releases .weights', () => { const deleteMock = jest.fn(); const texture = mockTexture(); layer.weights = texture; texture.delete = deleteMock; layer.predict(); expect(deleteMock).toBeCalled(); }); test('can forward propagate from input layers', () => { layer.predict(); expect(layer.weights).toEqual([ new Float32Array([7, 16, 27]), new Float32Array([40, 55, 72]), ]); }); it('clears deltas', () => { const texture = mockTexture(); const clearMock = jest.fn(); texture.clear = clearMock; layer.deltas = texture; layer.predict(); expect(clearMock).toBeCalled(); }); }); describe('.compare (back propagation)', () => { let mockInputLayer1: ILayer; let mockInputLayer2: ILayer; let layer: MultiplyElement; beforeEach(() => { mockInputLayer1 = mockLayer({ width: 3, height: 2, weights: [ [1, 2, 3], [4, 5, 6], ], deltas: null, }); mockInputLayer2 = mockLayer({ width: 3, height: 2, weights: [ [7, 8, 9], [10, 11, 12], ], deltas: null, }); layer = new MultiplyElement(mockInputLayer1, mockInputLayer2); layer.setupKernels(); layer.deltas = [ [13, 14, 15], [16, 17, 18], ]; }); test('can back propagate to input layers', () => { layer.compare(); expect(mockInputLayer1.deltas).toEqual([ new Float32Array([91, 112, 135]), new Float32Array([160, 187, 216]), ]); expect(mockInputLayer2.deltas).toEqual([ new Float32Array([13, 28, 45]), new Float32Array([64, 85, 108]), ]); }); test('releases inputLayer textures', () => { const deleteTexture1Mock = jest.fn(); const deleteTexture2Mock = jest.fn(); const deltas1 = mockTexture(); const deltas2 = mockTexture(); deltas1.delete = deleteTexture1Mock; deltas2.delete = deleteTexture2Mock; mockInputLayer1.deltas = deltas1; mockInputLayer2.deltas = deltas2; layer.compare(); expect(deleteTexture1Mock).toHaveBeenCalled(); expect(deleteTexture2Mock).toHaveBeenCalled(); }); }); describe('predict (forward propagation)', () => { test('can multiply a simple matrix', () => { const inputs1 = [ [1, 2, 3], [4, 5, 6], ]; const inputs2 = [ [7, 8, 9], [10, 11, 12], ]; const results = gpuMock(predict, { output: [3, 2], })(inputs1, inputs2); expect(results).toEqual([ new Float32Array([7, 16, 27]), new Float32Array([40, 55, 72]), ]); }); }); // yea it is basically a clone of `predict`, but for naming conventions, we'll keep them separated describe('compare (back propagation)', () => { test('can multiply a simple matrix', () => { const weights = [ [1, 2, 3], [4, 5, 6], ]; const deltas = [ [7, 8, 9], [10, 11, 12], ]; const results = gpuMock(compare, { output: [3, 2], })(weights, deltas); expect(results).toEqual([ new Float32Array([7, 16, 27]), new Float32Array([40, 55, 72]), ]); }); }); describe('multiplyElement function', () => { it('calls new MultiplyElement with inputLayer1 and inputLayer2', () => { const mockInputLayer1 = mockLayer({}); const mockInputLayer2 = mockLayer({}); const layer = multiplyElement(mockInputLayer1, mockInputLayer2); expect(layer.inputLayer1).toBe(mockInputLayer1); expect(layer.inputLayer2).toBe(mockInputLayer2); }); it('calls new MultiplyElement with settings', () => { const mockInputLayer1 = mockLayer({}); const mockInputLayer2 = mockLayer({}); const layer = multiplyElement(mockInputLayer1, mockInputLayer2, { id: 'id', }); expect(layer.id).toBe('id'); }); }); }); ================================================ FILE: src/layer/multiply-element.ts ================================================ import { makeKernel, release } from '../utilities/kernel'; import { Operator } from './operator'; import { checkSameSize } from '../utilities/layer-size'; import { ILayer, ILayerSettings } from './base-layer'; import { IKernelFunctionThis, IKernelRunShortcut } from 'gpu.js'; export function predict( this: IKernelFunctionThis, inputLayerWeights1: number[][], inputLayerWeights2: number[][] ): number { return ( inputLayerWeights1[this.thread.y][this.thread.x] * inputLayerWeights2[this.thread.y][this.thread.x] ); } export function compare( this: IKernelFunctionThis, weights: number[][], deltas: number[][] ): number { return ( weights[this.thread.y][this.thread.x] * deltas[this.thread.y][this.thread.x] ); } export class MultiplyElement extends Operator { get width(): number { return this.inputLayer1.width; } get height(): number { return this.inputLayer1.height; } get depth(): number { return this.inputLayer1.depth; } validate(): void { super.validate(); checkSameSize(this.inputLayer1, this.inputLayer2); } setupKernels(): void { this.predictKernel = makeKernel(predict, { output: [this.width, this.height], immutable: true, }); this.compareKernel = makeKernel(compare, { output: [this.width, this.height], immutable: true, }); } predict(): void { release(this.weights); this.weights = (this.predictKernel as IKernelRunShortcut)( this.inputLayer1.weights, this.inputLayer2.weights ); } compare(): void { release(this.inputLayer1.deltas); release(this.inputLayer2.deltas); this.inputLayer1.deltas = (this.compareKernel as IKernelRunShortcut)( this.inputLayer2.weights, this.deltas ); this.inputLayer2.deltas = (this.compareKernel as IKernelRunShortcut)( this.inputLayer1.weights, this.deltas ); } } export function multiplyElement( inputLayer1: ILayer, inputLayer2: ILayer, settings?: ILayerSettings ): MultiplyElement { return new MultiplyElement(inputLayer1, inputLayer2, settings); } ================================================ FILE: src/layer/multiply.test.ts ================================================ import { GPU } from 'gpu.js'; import { gpuMock } from 'gpu-mock.js'; import { Input } from './input'; import { Multiply, predict, compareFromX, compareFromY, IMultiplyConstants, } from './multiply'; import { Random } from './random'; import { setup, teardown } from '../utilities/kernel'; import { mockLayer, mockPraxis } from '../test-utils'; describe('Multiply Layer', () => { beforeEach(() => { setup( new GPU({ mode: 'cpu', }) ); }); afterEach(() => { teardown(); }); describe('.predict (forward propagation)', () => { test('can multiply a simple matrix', () => { const inputs1 = [ [1, 2, 3], [4, 5, 6], ]; const inputs2 = [ [7, 8], [9, 10], [11, 12], ]; const predictKernel = gpuMock< Parameters, IMultiplyConstants >(predict, { output: [2, 2], constants: { size: inputs2.length, }, }); const results = predictKernel(inputs1, inputs2); expect(results).toEqual([ new Float32Array([58, 64]), new Float32Array([139, 154]), ]); }); }); describe('.compareFromX (back propagation)', () => { test('can multiply a simple matrix', () => { const m1 = [ [3, 3], [3, 3], ]; const m2 = [ [3, 3], [3, 3], ]; const deltas = [ [3, 3], [3, 3], ]; const result = gpuMock(compareFromX, { output: [2, 2], constants: { size: 2, }, })(deltas, m1, m2); expect(result).toEqual([ new Float32Array([21, 21]), new Float32Array([21, 21]), ]); }); test('can compare a simple matrix', () => { const deltas = [[1], [2], [3]]; const inputDeltas = [ [1, 2], [3, 4], [5, 6], ]; const inputWeights = [[1], [2]]; const result = gpuMock(compareFromX, { output: [2, 3], constants: { size: 1, }, })(deltas, inputDeltas, inputWeights); expect(result).toEqual([ new Float32Array([2, 4]), new Float32Array([5, 8]), new Float32Array([8, 12]), ]); }); }); describe('.compareFromY (back propagation)', () => { test('can multiply a simple matrix 2x2 * 2x2 = 2x2', () => { const m1 = [ [3, 3], [3, 3], ]; const m2 = [ [3, 3], [3, 3], ]; const deltas = [ [3, 3], [3, 3], ]; const result = gpuMock(compareFromY, { output: [2, 2], constants: { size: 2, }, })(deltas, m1, m2); expect(result).toEqual([ new Float32Array([21, 21]), new Float32Array([21, 21]), ]); }); test('can compare a simple matrix 3x1 * 2x1 = 3x2', () => { const deltas = [[1], [2], [3]]; const inputDeltas = [[1], [2]]; const inputWeights = [ [1, 2], [3, 4], [5, 6], ]; const result = gpuMock(compareFromY, { output: [1, 2], constants: { size: 3, }, })(deltas, inputDeltas, inputWeights); expect(result).toEqual([new Float32Array([23]), new Float32Array([30])]); }); test('can compare a simple matrix 3x1 * 1x3 = 3x1', () => { const deltas = [[1, 2, 3]]; const inputDeltas = [[1], [2], [3]]; const inputWeights = [[1, 2, 3]]; const result = gpuMock(compareFromY, { output: [1, 3], constants: { size: 1, }, })(deltas, inputDeltas, inputWeights); expect(result).toEqual([ new Float32Array([2]), new Float32Array([4]), new Float32Array([6]), ]); }); }); describe('.validate', () => { test('throws error when dimension are incompatible', () => { expect(() => { Multiply.prototype.validate.call({ inputLayer1: { width: 1, height: 1 }, inputLayer2: { width: 1, height: 2 }, height: 1, width: 1, }); }).toThrow(); }); test('validates when dimension are compatible', () => { Multiply.prototype.validate.call({ inputLayer1: { width: 1, height: 1 }, inputLayer2: { width: 1, height: 1 }, height: 1, width: 1, }); }); }); describe('instance', () => { describe('.predict method', () => { test('validates, multiplies, and sets .weights', () => { const inputLayer1 = mockLayer({ width: 3, height: 2, weights: [ [1, 2, 3], [4, 5, 6], ], }); const inputLayer2 = mockLayer({ width: 2, height: 3, weights: [ [7, 8], [9, 10], [11, 12], ], }); const multiplyLayer = new Multiply(inputLayer1, inputLayer2); multiplyLayer.validate(); multiplyLayer.setupKernels(); multiplyLayer.predict(); expect(multiplyLayer.weights).toEqual([ new Float32Array([58, 64]), new Float32Array([139, 154]), ]); }); }); describe('when used with Input layer', () => { test('is compatible', () => { const praxis = mockPraxis(mockLayer({})); const random = new Random({ height: 3, width: 2, praxis }); const input = new Input({ height: 2, praxis }); const multiply = new Multiply(random, input); random.validate(); random.setupKernels(); input.validate(); input.setupKernels(); multiply.validate(); multiply.setupKernels(); input.predict([0, 1]); random.predict(); multiply.predict(); expect(multiply.width).toEqual(1); expect(multiply.height).toEqual(3); }); }); }); }); ================================================ FILE: src/layer/multiply.ts ================================================ import { makeKernel, release } from '../utilities/kernel'; import { Operator } from './operator'; import { IConstantsThis, IKernelFunctionThis, IKernelRunShortcut, Texture, } from 'gpu.js'; import { ILayer, ILayerJSON, ILayerSettings } from './base-layer'; export interface IMultiplyConstants extends IConstantsThis { size: number; } export function predict( this: IKernelFunctionThis, weights1: number[][], weights2: number[][] ): number { let sum = 0; for (let i = 0; i < this.constants.size; i++) { sum += weights1[this.thread.y][i] * weights2[i][this.thread.x]; } return sum; } export function compareFromX( this: IKernelFunctionThis, deltas: number[][], inputDeltas: number[][], inputWeights: number[][] ): number { let sum = inputDeltas[this.thread.y][this.thread.x]; for (let i = 0; i < this.constants.size; i++) { sum += deltas[this.thread.y][i] * inputWeights[this.thread.x][i]; } return sum; } export function compareFromY( this: IKernelFunctionThis, deltas: number[][], inputDeltas: number[][], inputWeights: number[][] ): number { let sum = inputDeltas[this.thread.y][this.thread.x]; for (let i = 0; i < this.constants.size; i++) { sum += deltas[i][this.thread.x] * inputWeights[i][this.thread.y]; } return sum; } export class Multiply extends Operator { compareKernel1: IKernelRunShortcut | null = null; compareKernel2: IKernelRunShortcut | null = null; get width(): number { return this.inputLayer2.width; } set width(width: number) { throw new Error('Cannot set width on Multiply'); } get height(): number { return this.inputLayer1.height; } set height(height: number) { throw new Error('Cannot set height on Multiply'); } get depth(): number { return this.inputLayer1.depth; } set depth(depth: number) { throw new Error('Cannot set depth on Multiply'); } validate(): void { super.validate(); if (this.inputLayer1.width !== this.inputLayer2.height) { throw new Error( `Layer width mismatch of ${this.inputLayer1.width} and ${this.inputLayer2.height}` ); } } setupKernels(): void { this.predictKernel = makeKernel(predict, { output: [this.width, this.height], constants: { size: this.inputLayer2.height, }, immutable: true, }); this.compareKernel1 = makeKernel(compareFromX, { output: [this.inputLayer1.width, this.inputLayer1.height], constants: { size: this.inputLayer2.width, }, immutable: true, }); this.compareKernel2 = makeKernel(compareFromY, { output: [this.inputLayer2.width, this.inputLayer2.height], constants: { size: this.inputLayer1.height, }, immutable: true, }); } reuseKernels(layer: ILayer): void { super.reuseKernels(layer); this.compareKernel1 = (layer as Multiply).compareKernel1; this.compareKernel2 = (layer as Multiply).compareKernel2; } predict(): void { release(this.weights); if (!this.predictKernel) throw new Error('this.predictKernel is not set'); this.weights = this.predictKernel( this.inputLayer1.weights, this.inputLayer2.weights ) as Texture; } compare(): void { if (!this.compareKernel1) throw new Error('this.compareKernel1 not set'); if (!this.compareKernel2) throw new Error('this.compareKernel2 not set'); const inputLayer1Deltas = this.inputLayer1.deltas; const inputLayer2Deltas = this.inputLayer2.deltas; const newDeltas1 = this.compareKernel1( this.deltas, this.inputLayer1.deltas, this.inputLayer2.weights ); const newDeltas2 = this.compareKernel2( this.deltas, this.inputLayer2.deltas, this.inputLayer1.weights ); this.inputLayer2.deltas = newDeltas2 as Texture; this.inputLayer1.deltas = newDeltas1 as Texture; release(inputLayer1Deltas); release(inputLayer2Deltas); } setupPraxis(): void {} toJSON(): Partial { return { ...super.toJSON(), width: this.width, height: this.height, }; } } export function multiply( inputLayer1: ILayer, inputLayer2: ILayer, settings?: ILayerSettings ): Multiply { return new Multiply(inputLayer1, inputLayer2, settings); } ================================================ FILE: src/layer/negative.test.ts ================================================ import { gpuMock } from 'gpu-mock.js'; import { input } from './input'; import { Negative, predict } from './negative'; import { makeKernel } from '../utilities/kernel'; import { mockLayer } from '../test-utils'; jest.mock('../utilities/kernel', () => { return { makeKernel: jest.fn((fn, settings) => gpuMock(fn, settings)), }; }); describe('Negative Layer', () => { describe('.constructor()', () => { let validateSpy: jest.SpyInstance; beforeEach(() => { validateSpy = jest.spyOn(Negative.prototype, 'validate'); }); afterEach(() => { validateSpy.mockRestore(); }); it('calls .validate()', () => { // eslint-disable-next-line no-new new Negative(mockLayer({ width: 1, height: 1, depth: 0 })); expect(validateSpy).toHaveBeenCalled(); }); }); describe('.setupKernels()', () => { it('sets this.predictKernel', () => { const layer = new Negative(input({ width: 1, height: 2 })); layer.setupKernels(); expect(makeKernel).toHaveBeenCalledWith(predict, { output: [1, 2], }); }); }); describe('.predict()', () => { let layer: Negative; beforeEach(() => { layer = new Negative(mockLayer({ width: 1, height: 1, depth: 0 })); layer.setupKernels(); }); it('sets this.weights from this.predictKernel()', () => { layer.inputLayer.weights = [[42]]; layer.predict(); expect(layer.weights).toEqual([Float32Array.from([-42])]); }); }); }); ================================================ FILE: src/layer/negative.ts ================================================ import { makeKernel } from '../utilities/kernel'; import { Modifier } from './types'; import { IKernelFunctionThis, IKernelRunShortcut } from 'gpu.js'; import { ILayer, ILayerSettings } from './base-layer'; export function predict( this: IKernelFunctionThis, weights: number[][] ): number { return -weights[this.thread.y][this.thread.x]; } export class Negative extends Modifier { constructor(inputLayer: ILayer, settings?: ILayerSettings) { super(inputLayer, settings); this.validate(); } setupKernels(): void { this.predictKernel = makeKernel(predict, { output: [this.width, this.height], }); } predict(): void { this.weights = (this.predictKernel as IKernelRunShortcut)( this.inputLayer.weights ); } } export function negative( inputLayer: ILayer, settings?: ILayerSettings ): Negative { return new Negative(inputLayer, settings); } ================================================ FILE: src/layer/ones.ts ================================================ import { ILayerSettings } from './base-layer'; import { ones2D } from '../utilities/ones'; import { zeros2D } from '../utilities/zeros-2d'; import { Model } from './types'; export class Ones extends Model { constructor(settings: ILayerSettings) { super(settings); this.validate(); this.weights = ones2D(this.width, this.height); this.deltas = zeros2D(this.width, this.height); } } export function ones(settings: ILayerSettings): Ones { return new Ones(settings); } ================================================ FILE: src/layer/operator.ts ================================================ import { BaseLayer, ILayerSettings, ILayer } from './base-layer'; import { zeros2D } from '../utilities/zeros-2d'; export type OperatorType = new ( inputLayer1: ILayer, inputLayer2: ILayer, settings?: Partial ) => ILayer; export abstract class Operator extends BaseLayer { inputLayer1: ILayer; inputLayer2: ILayer; constructor( inputLayer1: ILayer, inputLayer2: ILayer, settings?: Partial ) { super(settings); this.inputLayer1 = inputLayer1; this.inputLayer2 = inputLayer2; this.validate(); this.weights = zeros2D(this.width, this.height); this.deltas = zeros2D(this.width, this.height); this.setupPraxis(); } } ================================================ FILE: src/layer/output.ts ================================================ import { add } from './add'; import { multiply } from './multiply'; import { random } from './random'; import { target } from './target'; import { ILayer, ILayerSettings } from './base-layer'; export function output(settings: ILayerSettings, inputLayer: ILayer): ILayer { const { height } = settings; const outputGate = random({ height, width: inputLayer.height, id: 'outputGate', std: 0.08, }); const output = random({ height, id: 'output', std: 0.08 }); const outputGateConnector = multiply(outputGate, inputLayer, { id: 'outputGateConnected', }); return target( { id: 'target', ...settings }, add(outputGateConnector, output) ); } ================================================ FILE: src/layer/pool.test.ts ================================================ import { GPU } from 'gpu.js'; import { gpuMock } from 'gpu-mock.js'; import { Pool, predict, compare, compare3D, ICompareConstants, IPredictConstants, } from './pool'; import { setup, teardown } from '../utilities/kernel'; import { mockLayer } from '../test-utils'; describe('Pool Layer', () => { beforeEach(() => { setup( new GPU({ mode: 'cpu', }) ); }); afterEach(() => { teardown(); }); describe('constructor', () => { test('correctly sets dimensions', () => { const layer = new Pool( { filterWidth: 2, filterHeight: 2, filterCount: 8, stride: 2, }, mockLayer({ width: 24, height: 24, }) ); expect(layer.width).toEqual(12); expect(layer.height).toEqual(12); expect(layer.depth).toEqual(8); }); }); describe('.predict (forward propagation)', () => { test('can pool a simple matrix', () => { const inputs = [ [ [1, 2, 3], [4, 5, 6], [7, 8, 9], ], ]; const constants: IPredictConstants = { filterWidth: 3, filterHeight: 3, filterCount: 1, inputWidth: 3, inputHeight: 3, inputDepth: 1, paddingX: 0, paddingY: 0, strideX: 1, strideY: 1, }; const results = gpuMock(predict, { output: [1, 1, 0], constants, })(inputs); expect(results).toEqual([new Float32Array([9])]); }); }); describe('.compare (back propagation)', () => { test('can pool a simple matrix', () => { const deltas = [ [1, 2], [3, 4], ]; const switchX = [ [0, 2], [1, 1], ]; const switchY = [ [1, 1], [2, 2], ]; const constants: ICompareConstants = { inputWidth: 2, inputHeight: 2, outputWidth: 2, outputHeight: 2, paddingX: 0, paddingY: 0, strideX: 1, strideY: 1, filterWidth: 2, filterHeight: 2, }; const results = gpuMock(compare, { output: [3, 3], constants, })(deltas, switchX, switchY); expect(results).toEqual([ new Float32Array([0, 0, 0]), new Float32Array([1, 0, 2]), new Float32Array([0, 7, 0]), ]); }); test('can pool a simple matrix 2', () => { // Tests backprop of 2x2 matrix with 1 padding, resulting in 3x3 const deltas = [ [1, 2, 3], [4, 5, 6], [7, 8, 9], ]; const switchX = [ [0, 1, 1], [0, 0, 1], [0, 1, 1], ]; const switchY = [ [0, 0, 0], [0, 1, 0], [1, 1, 1], ]; const constants: ICompareConstants = { inputWidth: 2, inputHeight: 2, outputWidth: 3, outputHeight: 3, paddingX: 1, paddingY: 1, strideX: 1, strideY: 1, filterWidth: 2, filterHeight: 2, }; const results = gpuMock(compare, { output: [2, 2], constants, })(deltas, switchX, switchY); expect(results).toEqual([ new Float32Array([5, 11]), new Float32Array([12, 17]), ]); }); }); describe('.compare3D (back propagation)', () => { test('can pool a simple matrix', () => { const deltas = [ [ [1, 2], [3, 4], ], ]; const switchX = [ [ [0, 2], [1, 2], ], ]; const switchY = [ [ [0, 1], [2, 2], ], ]; const constants: ICompareConstants = { inputWidth: 2, inputHeight: 2, outputWidth: 2, outputHeight: 2, paddingX: 0, paddingY: 0, strideX: 1, strideY: 1, filterWidth: 2, filterHeight: 2, }; const results = gpuMock(compare3D, { output: [3, 3, 1], constants, })(deltas, switchY, switchX); expect(results).toEqual([ [ new Float32Array([1, 0, 0]), new Float32Array([0, 0, 2]), new Float32Array([0, 3, 4]), ], ]); }); test('can pool a simple matrix 2', () => { const deltas = [ [ [1, 2], [3, 4], ], ]; const switchX = [ [ [0, 1], [0, 1], ], ]; const switchY = [ [ [0, 0], [1, 1], ], ]; const constants: ICompareConstants = { inputWidth: 2, inputHeight: 2, outputWidth: 2, outputHeight: 2, paddingX: 1, paddingY: 1, strideX: 1, strideY: 1, filterWidth: 2, filterHeight: 2, }; const results = gpuMock(compare3D, { output: [2, 2, 1], constants, })(deltas, switchY, switchX); expect(results).toEqual([ [new Float32Array([1, 2]), new Float32Array([3, 4])], ]); }); }); }); describe('Pool Layer', () => { describe('.predict (forward propagation)', () => { it('can predict pool a simple matrix', () => { const inputs = [ [ [1, 2, 3], [4, 5, 6], [7, 8, 9], ], ]; const constants: IPredictConstants = { filterWidth: 3, filterHeight: 3, filterCount: 1, inputWidth: 3, inputHeight: 3, inputDepth: 1, paddingX: 0, paddingY: 0, strideX: 1, strideY: 1, }; const results = gpuMock(predict, { output: [1, 1, 0], constants, })(inputs); expect(results).toEqual([Float32Array.from([9])]); }); }); describe('.compare (back propagation)', () => { it('can compare pool a simple matrix', () => { const deltas = [[9]]; const switchX = [[0]]; const switchY = [[0]]; const constants: ICompareConstants = { inputWidth: 3, inputHeight: 3, outputWidth: 1, outputHeight: 1, paddingX: 0, paddingY: 0, strideX: 1, strideY: 1, filterWidth: 3, filterHeight: 3, }; const results = gpuMock(compare, { output: [3, 3, 0], constants, })(deltas, switchX, switchY); expect(results).toEqual([ Float32Array.from([9, 0, 0]), Float32Array.from([0, 0, 0]), Float32Array.from([0, 0, 0]), ]); }); }); }); ================================================ FILE: src/layer/pool.ts ================================================ import { Filter } from './filter'; import { makeKernel, makeKernelMap, release } from '../utilities/kernel'; import { getPadding, getStride } from '../utilities/layer-setup'; import { zeros3D } from '../utilities/zeros-3d'; import { randos3D } from '../utilities/randos'; import { IKernelFunctionThis, IKernelMapRunShortcut, IKernelRunShortcut, ISubKernelObject, KernelOutput, } from 'gpu.js'; import { IConvolutionSettingsBase, IConvolutionConstantsBase, } from './convolution'; import { ILayer, ILayerSettings } from './base-layer'; export function setSwitchY(value: number): number { return value; } export function setSwitchX(value: number): number { return value; } export interface IPredictConstants extends IConvolutionConstantsBase { inputWidth: number; inputHeight: number; } export function predict( this: IKernelFunctionThis, inputs: number[][][] ): number { // Ends are exclusive, that is if end=4, the last item is 3 const unclippedStartInputX = this.thread.x * this.constants.strideX - this.constants.paddingX; const unclippedStartInputY = this.thread.y * this.constants.strideY - this.constants.paddingY; const unclippedEndInputX = unclippedStartInputX + this.constants.filterWidth; const unclippedEndInputY = unclippedStartInputY + this.constants.filterHeight; const startInputX = Math.max(unclippedStartInputX, 0); const startInputY = Math.max(unclippedStartInputY, 0); const endInputX = Math.min(unclippedEndInputX, this.constants.inputWidth); const endInputY = Math.min(unclippedEndInputY, this.constants.inputHeight); let largestValue = inputs[this.thread.z][startInputY][startInputX]; let largestX = startInputX; let largestY = startInputY; for (let y = startInputY; y < endInputY; y++) { for (let x = startInputX; x < endInputX; x++) { const input = inputs[this.thread.z][y][x]; if (input > largestValue) { largestValue = input; largestY = y; largestX = x; } } } setSwitchY(largestY); setSwitchX(largestX); return largestValue; } export interface ICompareConstants extends IConvolutionConstantsBase { inputWidth: number; inputHeight: number; outputWidth: number; outputHeight: number; } export function compare( this: IKernelFunctionThis, deltas: number[][], switchX: number[][], switchY: number[][] ): number { const xCenter = this.thread.x + 0.5; const yCenter = this.thread.y + 0.5; const invStrideX = 1 / this.constants.strideX; const invStrideY = 1 / this.constants.strideY; const startSourceX = Math.max( 0, Math.ceil( (xCenter - this.constants.filterWidth + this.constants.paddingX) * invStrideX ) ); const startSourceY = Math.max( 0, Math.ceil( (yCenter - this.constants.filterHeight + this.constants.paddingY) * invStrideY ) ); const endSourceX = Math.min( Math.ceil((xCenter + this.constants.paddingX) * invStrideX), this.constants.outputWidth ); const endSourceY = Math.min( Math.ceil((yCenter + this.constants.paddingY) * invStrideY), this.constants.outputHeight ); let result = 0; for (let backY = startSourceY; backY < endSourceY; backY++) { for (let backX = startSourceX; backX < endSourceX; backX++) { const switchXValue = switchX[backY][backX]; const switchYValue = switchY[backY][backX]; if ( Math.abs(switchXValue - this.thread.x) < 0.1 && Math.abs(switchYValue - this.thread.y) < 0.1 ) { result += deltas[backY][backX]; } } } return result; } export function compare3D( this: IKernelFunctionThis, deltas: number[][][], switchY: number[][][], switchX: number[][][] ): number { const xCenter = this.thread.x + 0.5; const yCenter = this.thread.y + 0.5; const invStrideX = 1 / this.constants.strideX; const invStrideY = 1 / this.constants.strideY; const startSourceX = Math.max( 0, Math.ceil( (xCenter - this.constants.filterWidth + this.constants.paddingX) * invStrideX ) ); const startSourceY = Math.max( 0, Math.ceil( (yCenter - this.constants.filterHeight + this.constants.paddingY) * invStrideY ) ); const endSourceX = Math.min( Math.ceil((xCenter + this.constants.paddingX) * invStrideX), this.constants.inputWidth ); const endSourceY = Math.min( Math.ceil((yCenter + this.constants.paddingY) * invStrideY), this.constants.inputHeight ); let result = 0; for (let backY = startSourceY; backY < endSourceY; backY++) { for (let backX = startSourceX; backX < endSourceX; backX++) { const switchXValue = switchX[this.thread.z][backY][backX]; const switchYValue = switchY[this.thread.z][backY][backX]; if ( Math.abs(switchXValue - this.thread.x) < 0.1 && Math.abs(switchYValue - this.thread.y) < 0.1 ) { result += deltas[this.thread.z][backY][backX]; } } } return result; } export interface IPoolSettings extends ILayerSettings, IConvolutionSettingsBase { switchX?: KernelOutput; switchY?: KernelOutput; } export const defaults: IPoolSettings = { padding: 0, stride: 0, filterWidth: 0, filterHeight: 0, filterCount: 0, }; export class Pool extends Filter { settings: Partial; get strideX(): number { return this.settings.strideX as number; } get strideY(): number { return this.settings.strideY as number; } get paddingX(): number { return this.settings.paddingX as number; } get paddingY(): number { return this.settings.paddingY as number; } get width(): number { // Using floor prefers to pad less (or use negative padding) on the right // using ceil prefers to pad more return Math.ceil( (this.inputLayer.width + this.paddingX * 2 - this.filterWidth) / this.strideX + 1 ); } get height(): number { // Using floor prefers to pad less (or use negative padding) on the bottom // using ceil prefers to pad more return Math.floor( (this.inputLayer.height + this.paddingY * 2 - this.filterHeight) / this.strideY + 1 ); } get depth(): number { return this.settings.filterCount as number; } get filterCount(): number { // TODO: handle 1 depth? return this.settings.filterCount as number; } get switchX(): KernelOutput { return this.settings.switchX; } set switchX(switchX: KernelOutput) { this.settings.switchX = switchX; } get switchY(): KernelOutput { return this.settings.switchY; } set switchY(switchY: KernelOutput) { this.settings.switchY = switchY; } predictKernelMap: IKernelMapRunShortcut | null = null; constructor(settings: IPoolSettings, inputLayer: ILayer) { super(settings, inputLayer); this.settings = { ...settings, ...getStride(settings, defaults), ...getPadding(settings, defaults), }; this.weights = randos3D(this.width, this.height, this.depth); this.deltas = zeros3D(this.width, this.height, this.depth); this.validate(); } setupKernels(): void { this.predictKernelMap = makeKernelMap( { switchX: setSwitchX, switchY: setSwitchY, }, predict, { output: [this.width, this.height, this.depth], constants: { inputWidth: this.inputLayer.width, inputHeight: this.inputLayer.height, paddingX: this.paddingX, paddingY: this.paddingY, filterHeight: this.filterHeight, filterWidth: this.filterWidth, strideX: this.strideX, strideY: this.strideY, }, } ); this.compareKernel = makeKernel(compare, { output: [ this.inputLayer.width, this.inputLayer.height, this.inputLayer.depth, ], constants: { inputWidth: this.inputLayer.width, inputHeight: this.inputLayer.height, outputWidth: this.width, outputHeight: this.height, filterWidth: this.filterWidth, filterHeight: this.filterHeight, paddingX: this.paddingX, paddingY: this.paddingY, strideX: this.strideX, strideY: this.strideY, }, }); } predict(): void { const { result: weights, switchX, switchY } = (this .predictKernelMap as IKernelMapRunShortcut)( this.inputLayer.weights ); this.switchX = switchX; this.switchY = switchY; this.weights = weights; } compare(): void { // debugger; // const depth = this.inputLayer.deltas.length; // const height = this.inputLayer.deltas[0].length; // const width = this.inputLayer.deltas[0][0].length; // const type = typeof this.inputLayer.deltas[0][0][0]; const inputLayerDeltas = this.inputLayer.deltas; this.inputLayer.deltas = (this.compareKernel as IKernelRunShortcut)( this.deltas, this.switchX, this.switchY ); release(inputLayerDeltas); // debugger; // if (depth !== this.inputLayer.deltas.length) debugger; // if (height !== this.inputLayer.deltas[0].length) debugger; // if (width !== this.inputLayer.deltas[0][0].length) debugger; // if (type !== typeof this.inputLayer.deltas[0][0][0]) debugger; } } export function pool(settings: IPoolSettings, inputLayer: ILayer): Pool { return new Pool(settings, inputLayer); } ================================================ FILE: src/layer/random.test.ts ================================================ import { random, Random } from './random'; import { randos2D } from '../utilities/randos'; import { release } from '../utilities/kernel'; import { mockLayer, mockPraxis } from '../test-utils'; jest.mock('../utilities/kernel'); describe('Random Layer', () => { describe('constructor', () => { describe('width and height only', () => { it('is instantiates sizes correctly', () => { const settings = { width: 5, height: 7 }; const layer = new Random(settings); expect(layer.width).toBe(settings.width); expect(layer.height).toBe(settings.height); expect(layer.depth).toBe(0); expect((layer.weights as number[][]).length).toBe(7); expect((layer.weights as number[][])[0].length).toBe(5); }); }); describe('layer.weights', () => { describe('when given from settings', () => { it('uses them', () => { const width = 2; const height = 3; const weights = randos2D(width, height); const settings = { weights, width, height }; const layer = new Random(settings); expect(layer.weights).toBe(weights); }); }); }); describe('layer.deltas', () => { describe('when given from settings', () => { it('uses them', () => { const width = 2; const height = 3; const deltas = randos2D(width, height); const settings = { deltas, width, height }; const layer = new Random(settings); expect(layer.deltas).toBe(deltas); }); }); }); }); describe('.learn', () => { it('releases both this.weights and this.deltas', () => { const width = 3; const height = 4; const depth = 5; const mockInputLayer = mockLayer({ width, height, depth }); const mockPraxisInstance = mockPraxis(mockInputLayer); const settings = { width: 1, height: 1, praxis: mockPraxisInstance, }; const layer = new Random(settings); const mockWeights = (layer.settings.weights = new Float32Array()); const mockDeltas = (layer.settings.deltas = new Float32Array()); layer.learn(0); expect(release).toHaveBeenCalledWith(mockWeights); expect(release).toHaveBeenCalledWith(mockDeltas); }); }); describe('random lambda', () => { it('passes settings on to Random constructor, and returns it', () => { const settings = { width: 5, height: 7 }; const layer = random(settings); expect(layer.settings.width).toBe(settings.width); expect(layer.settings.height).toBe(settings.height); }); }); }); ================================================ FILE: src/layer/random.ts ================================================ import { randos2D } from '../utilities/randos'; import { zeros2D } from '../utilities/zeros-2d'; import { baseLayerDefaultSettings, ILayer, ILayerSettings } from './base-layer'; import { Model } from './types'; export interface IRandomSettings extends ILayerSettings { std?: number | null; } export const defaults: IRandomSettings = { ...baseLayerDefaultSettings, std: null, }; export class Random extends Model implements ILayer { settings: IRandomSettings; constructor(settings: Partial) { super(); this.settings = { ...defaults, ...settings }; this.setupPraxis(); this.validate(); if (!this.weights) { this.weights = randos2D(this.width, this.height, settings.std); } if (!this.deltas) { this.deltas = zeros2D(this.width, this.height); } } predict(): void {} compare(): void {} } export function random(settings: IRandomSettings): Random { return new Random(settings); } ================================================ FILE: src/layer/recurrent-connection.ts ================================================ import { KernelOutput } from 'gpu.js'; import { Internal } from './internal'; import { release } from '../utilities/kernel'; import { ILayer, ILayerSettings } from './base-layer'; export class RecurrentConnection extends Internal { settings: ILayerSettings = {}; layer: ILayer | null = null; setLayer(layer: ILayer): void { this.layer = layer; } get width(): number { if (!this.layer) throw new Error('layer not set'); return this.layer.width; } set width(value: number) { throw new Error(`${this.constructor.name}-width is not yet implemented`); } get height(): number { if (!this.layer) throw new Error('layer not set'); return this.layer.height; } set height(value: number) { throw new Error(`${this.constructor.name}-height is not yet implemented`); } get deltas(): KernelOutput { if (!this.layer) throw new Error('layer not set'); return this.layer.deltas; } set deltas(deltas: KernelOutput) { if (!this.layer) throw new Error('layer not set'); release(this.layer.deltas); this.layer.deltas = deltas; } get weights(): KernelOutput { if (!this.layer) throw new Error('layer not set'); return this.layer.weights as KernelOutput; } set weights(weights: KernelOutput) { if (!this.layer) throw new Error('layer not set'); release(this.layer.weights); this.layer.weights = weights; } predict(): void { // throw new Error(`${this.constructor.name}-predict is not yet implemented`) } compare(): void { // throw new Error(`${this.constructor.name}-compare is not yet implemented`) } learn(): void { throw new Error('no longer using'); } setupKernels(): void { // throw new Error( // `${this.constructor.name}-setupKernels is not yet implemented` // ) } reuseKernels(): void { // throw new Error( // `${this.constructor.name}-reuseKernels is not yet implemented` // ) } } ================================================ FILE: src/layer/recurrent-input.ts ================================================ import { KernelOutput } from 'gpu.js'; import { IPraxis } from '../praxis/base-praxis'; import { release } from '../utilities/kernel'; import { BaseLayer, ILayer } from './base-layer'; import { Internal } from './internal'; export interface IRecurrentInput extends ILayer { setDimensions?: (width: number, height: number) => void; } export class RecurrentInput extends Internal implements IRecurrentInput { recurrentInput: ILayer; praxis: IPraxis | null = null; predictKernel = null; compareKernel = null; settings = {}; constructor(recurrentInput: ILayer) { super(); this.recurrentInput = recurrentInput; this.validate(); } get width(): number { return this.recurrentInput.width; } get height(): number { return this.recurrentInput.height; } get depth(): number { return this.recurrentInput.depth; } get deltas(): KernelOutput { return this.recurrentInput.deltas; } set deltas(deltas: KernelOutput) { const recurrentInputDeltas = this.recurrentInput.deltas; this.recurrentInput.deltas = deltas; release(recurrentInputDeltas); } get weights(): KernelOutput { return this.recurrentInput.weights as KernelOutput; } set weights(weights: KernelOutput) { const recurrentInputWeights = this.recurrentInput.weights; this.recurrentInput.weights = weights; release(recurrentInputWeights); } validate(): void { BaseLayer.prototype.validate.call(this); if (this.width !== this.recurrentInput.width) { throw new Error( `${this.constructor.name} layer width ${this.width} and ${this.recurrentInput.constructor.name} width (${this.recurrentInput.width}) are not same` ); } if (this.height !== this.recurrentInput.height) { throw new Error( `${this.constructor.name} layer height ${this.height} and ${this.recurrentInput.constructor.name} width (${this.recurrentInput.height}) are not same` ); } } setDimensions(width: number, height: number): void { this.recurrentInput.width = width; this.recurrentInput.height = height; } predict(): void { // throw new Error(`${this.constructor.name}-predict is not yet implemented`) } compare(): void { // throw new Error(`${this.constructor.name}-compare is not yet implemented`) } learn(): void { // throw new Error(`${this.constructor.name}-learn is not yet implemented`) } setupKernels(): void { // throw new Error( // `${this.constructor.name}-setupKernels is not yet implemented` // ) } reuseKernels(): void { // throw new Error( // `${this.constructor.name}-reuseKernels is not yet implemented` // ) } } ================================================ FILE: src/layer/recurrent-zeros.ts ================================================ import { IPraxis } from '../praxis/base-praxis'; import { release } from '../utilities/kernel'; import { zeros2D } from '../utilities/zeros-2d'; import { ILayerSettings } from './base-layer'; import { Internal } from './internal'; import { IRecurrentInput } from './recurrent-input'; export class RecurrentZeros extends Internal implements IRecurrentInput { praxis: IPraxis | null = null; settings: Partial = {}; predictKernel = null; compareKernel = null; constructor(settings?: Partial) { super(); if (settings) { this.settings = { ...settings }; } } setDimensions(width: number, height: number): void { this.praxis = null; this.settings = { ...this.settings, width, height, weights: zeros2D(width, height), deltas: zeros2D(width, height), }; } setupKernels(): void { // throw new Error( // `${this.constructor.name}-setupKernels is not yet implemented` // ) } reuseKernels(): void { // throw new Error( // `${this.constructor.name}-reuseKernels is not yet implemented` // ) } predict(): void { // throw new Error(`${this.constructor.name}-predict is not yet implemented`) } compare(): void { // throw new Error(`${this.constructor.name}-compare is not yet implemented`) } learn(learningRate: number): void { const { weights: oldWeights } = this; this.weights = (this.praxis as IPraxis).run(this, learningRate); // this.deltas = deltas; release(oldWeights); } // validate(): void { // throw new Error(`${this.constructor.name}-validate is not yet implemented`); // } // reset(): void { // throw new Error(`${this.constructor.name}-reset is not yet implemented`); // } } export function recurrentZeros(): RecurrentZeros { return new RecurrentZeros(); } ================================================ FILE: src/layer/regression.ts ================================================ import { IKernelFunctionThis, KernelOutput } from 'gpu.js'; import { BaseLayer, ILayer, ILayerSettings } from './base-layer'; import { clone, release } from '../utilities/kernel'; export class Regression extends BaseLayer { inputLayer: ILayer; constructor(settings: ILayerSettings, inputLayer: ILayer) { super(settings); this.inputLayer = inputLayer; this.validate(); } predict(): void { release(this.weights); this.weights = clone(this.inputLayer.weights as KernelOutput); } learn(): void { // throw new Error(`${this.constructor.name}-learn is not yet implemented`) } } // TODO: Connect up export function learn( this: IKernelFunctionThis, inputs: number[], targets: number[] ): number { return inputs[this.thread.x] - targets[this.thread.x]; } // TODO: handle `loss += 0.5*dy*dy;` total and sum in learn export function regression( settings: ILayerSettings, inputLayer: ILayer ): Regression { return new Regression(settings, inputLayer); } ================================================ FILE: src/layer/relu.test.ts ================================================ import { GPU } from 'gpu.js'; import { gpuMock } from 'gpu-mock.js'; import { Relu, relu, predict2D, predict3D, compare2D, compare3D } from './relu'; import * as reluActivation from '../activation/relu'; import { IWithCompareKernel, IWithPredictKernel, mockLayer, mockPraxis, } from '../test-utils'; import { makeKernel, setup, teardown } from '../utilities/kernel'; import { randos2D } from '../utilities/randos'; import { ILayerSettings } from './base-layer'; jest.mock('../utilities/kernel', () => { return { setup: jest.fn(), teardown: jest.fn(), makeKernel: jest.fn(() => { return [[1]]; }), release: jest.fn(), clear: jest.fn(), }; }); describe('Relu Layer', () => { describe('predict2D() (forward propagation)', () => { test('can relu a simple matrix', () => { const inputs = [ [0.1, -0.2, 0.3], [-0.4, 0.5, -0.6], [0.7, -0.8, 0.9], ]; const results = gpuMock(predict2D, { output: [3, 3] })(inputs); expect(results).toEqual([ new Float32Array([0.1, 0, 0.3]), new Float32Array([0, 0.5, 0]), new Float32Array([0.7, 0, 0.9]), ]); }); }); describe('predict3D() (forward propagation)', () => { test('can relu a simple matrix', () => { const inputs = [ [ [0.1, -0.2, 0.3], [-0.4, 0.5, -0.6], [0.7, -0.8, 0.9], ], [ [0.1, -0.2, 0.3], [-0.4, 0.5, -0.6], [0.7, -0.8, 0.9], ], ]; const results = gpuMock(predict3D, { output: [3, 3, 2] })(inputs); expect(results).toEqual([ [ new Float32Array([0.1, 0, 0.3]), new Float32Array([0, 0.5, 0]), new Float32Array([0.7, 0, 0.9]), ], [ new Float32Array([0.1, 0, 0.3]), new Float32Array([0, 0.5, 0]), new Float32Array([0.7, 0, 0.9]), ], ]); }); }); describe('compare2D (back propagation)', () => { test('can relu a simple matrix', () => { const inputs = [ [0.1, -0.2, 0.3], [-0.4, 0.5, -0.6], [0.7, -0.8, 0.9], ]; const deltas = [ [1, 1, 1], [1, 1, 1], [1, 1, 1], ]; const results = gpuMock(compare2D, { output: [3, 3] })(inputs, deltas); expect(results).toEqual([ new Float32Array([1, 0, 1]), new Float32Array([0, 1, 0]), new Float32Array([1, 0, 1]), ]); }); }); describe('compare3D (back propagation)', () => { test('can relu a simple matrix', () => { const inputs = [ [ [0.1, -0.2, 0.3], [-0.4, 0.5, -0.6], [0.7, -0.8, 0.9], ], [ [0.1, -0.2, 0.3], [-0.4, 0.5, -0.6], [0.7, -0.8, 0.9], ], ]; const deltas = [ [ [1, 1, 1], [1, 1, 1], [1, 1, 1], ], [ [1, 1, 1], [1, 1, 1], [1, 1, 1], ], ]; const results = gpuMock(compare3D, { output: [3, 3, 2] })(inputs, deltas); expect(results).toEqual([ [ new Float32Array([1, 0, 1]), new Float32Array([0, 1, 0]), new Float32Array([1, 0, 1]), ], [ new Float32Array([1, 0, 1]), new Float32Array([0, 1, 0]), new Float32Array([1, 0, 1]), ], ]); }); }); describe('.setupKernels()', () => { beforeEach(() => { setup( new GPU({ mode: 'cpu', }) ); }); afterEach(() => { teardown(); }); describe('2d', () => { it('sets up kernels correctly', () => { const width = 3; const height = 4; const mockInputLayer = mockLayer({ width, height }); const l = new Relu(mockInputLayer); expect(l.predictKernel).toBe(null); expect(l.compareKernel).toBe(null); l.setupKernels(); expect(l.predictKernel).not.toBe(null); expect(l.compareKernel).not.toBe(null); expect(makeKernel).toHaveBeenCalledWith(predict2D, { functions: [reluActivation.activate], immutable: true, output: [3, 4], }); expect(makeKernel).toHaveBeenCalledWith(compare2D, { functions: [reluActivation.measure], immutable: true, output: [3, 4], }); }); }); describe('3d', () => { it('sets up kernels correctly', () => { const width = 3; const height = 4; const depth = 5; const mockInputLayer = mockLayer({ width, height, depth }); const l = new Relu(mockInputLayer); expect(l.predictKernel).toBe(null); expect(l.compareKernel).toBe(null); l.setupKernels(); expect(l.predictKernel).not.toBe(null); expect(l.compareKernel).not.toBe(null); expect(makeKernel).toHaveBeenCalledWith(predict3D, { functions: [reluActivation.activate], immutable: true, output: [3, 4, 5], }); expect(makeKernel).toHaveBeenCalledWith(compare3D, { functions: [reluActivation.measure], immutable: true, output: [3, 4, 5], }); }); }); }); describe('.predict()', () => { it('calls this.predictKernel() with this.inputLayer.weights', () => { const mockWeights = randos2D(1, 1); const mockInputLayer = mockLayer({ weights: mockWeights, width: 1, height: 1, depth: 1, }); const l = new Relu(mockInputLayer); ((l as unknown) as IWithPredictKernel).predictKernel = jest.fn( (weights) => weights ); l.predict(); expect(l.predictKernel).toBeCalledWith(mockWeights); expect(l.weights).toBe(mockWeights); }); }); describe('.compare()', () => { it('calls this.compareKernel() with this.inputLayer.weights & this.inputLayer.deltas', () => { const mockWeights = randos2D(1, 1); const mockDeltas = randos2D(1, 1); const mockInputDeltas = randos2D(1, 1); const mockInputLayer = mockLayer({ width: 1, height: 1, depth: 1, deltas: mockInputDeltas, }); const l = new Relu(mockInputLayer); l.weights = mockWeights; l.deltas = mockDeltas; const expectedDeltas = randos2D(1, 1); ((l as unknown) as IWithCompareKernel).compareKernel = jest.fn( (weights, deltas) => expectedDeltas ); l.compare(); expect(l.compareKernel).toBeCalledWith(mockWeights, mockDeltas); expect(l.inputLayer.deltas).toBe(expectedDeltas); }); }); describe('relu lambda', () => { test('creates a new instance of Relu', () => { const width = 3; const height = 4; const depth = 5; const mockInputLayer = mockLayer({ width, height, depth }); const praxis = mockPraxis(mockInputLayer); const praxisSettings = {}; const settings: ILayerSettings = { praxisOpts: praxisSettings, initPraxis: jest.fn((settings: typeof praxisSettings) => { return praxis; }), }; const l = relu(mockInputLayer, settings); expect(l.constructor).toBe(Relu); expect(l.width).toBe(width); expect(l.height).toBe(height); expect(l.depth).toBe(depth); expect(l.praxis).toBe(praxis); }); }); }); ================================================ FILE: src/layer/relu.ts ================================================ import { IKernelFunctionThis, IKernelRunShortcut } from 'gpu.js'; import { Activation } from './types'; import { makeKernel, release } from '../utilities/kernel'; import { activate, measure } from '../activation/relu'; import { ILayer, ILayerSettings } from './base-layer'; export function predict2D( this: IKernelFunctionThis, inputs: number[][] ): number { return activate(inputs[this.thread.y][this.thread.x]); } export function compare2D( this: IKernelFunctionThis, weights: number[][], deltas: number[][] ): number { return measure( weights[this.thread.y][this.thread.x], deltas[this.thread.y][this.thread.x] ); } export function predict3D( this: IKernelFunctionThis, inputs: number[][][] ): number { return activate(inputs[this.thread.z][this.thread.y][this.thread.x]); } export function compare3D( this: IKernelFunctionThis, weights: number[][][], deltas: number[][][] ): number { return measure( weights[this.thread.z][this.thread.y][this.thread.x], deltas[this.thread.z][this.thread.y][this.thread.x] ); } export class Relu extends Activation { setupKernels(): void { const { width, height, depth } = this.inputLayer; if (depth > 0) { this.predictKernel = makeKernel(predict3D, { output: [width, height, depth], functions: [activate], immutable: true, }); this.compareKernel = makeKernel(compare3D, { output: [width, height, depth], functions: [measure], immutable: true, }); } else { this.predictKernel = makeKernel(predict2D, { output: [width, height], functions: [activate], immutable: true, }); this.compareKernel = makeKernel(compare2D, { output: [width, height], functions: [measure], immutable: true, }); } } predict(): void { release(this.weights); this.weights = (this.predictKernel as IKernelRunShortcut)( this.inputLayer.weights ); } compare(): void { release(this.inputLayer.deltas); this.inputLayer.deltas = (this.compareKernel as IKernelRunShortcut)( this.weights, this.deltas ); } } export function relu(inputLayer: ILayer, settings?: ILayerSettings): Relu { return new Relu(inputLayer, settings); } ================================================ FILE: src/layer/rnn-cell.test.ts ================================================ import { rnnCell, RecurrentZeros, Add, Random, Zeros, Multiply, Relu, } from './'; import { mockLayer, TestLayer } from '../test-utils'; import { flattenLayers } from '../utilities/flatten-layers'; describe('rnn Cell', () => { it('properly sets width and height', () => { const input = mockLayer({ width: 1, height: 3 }); const settings = { height: 3 }; const recurrentInput = new RecurrentZeros(); const layer = rnnCell(settings, input, recurrentInput); expect(layer.width).toEqual(1); expect(layer.height).toEqual(settings.height); }); it('throws if height is not a number', () => { const input = mockLayer({ width: 1, height: 3 }); const settings = { height: null }; const recurrentInput = new RecurrentZeros(); expect(() => { rnnCell(settings, input, recurrentInput); }).toThrow(); }); describe('when .setDimensions is available', () => { let setDimensionsSpy: jest.SpyInstance; beforeEach(() => { setDimensionsSpy = jest.spyOn(RecurrentZeros.prototype, 'setDimensions'); }); afterEach(() => { setDimensionsSpy.mockRestore(); }); it('is called', () => { const input = mockLayer({ width: 1, height: 3 }); const settings = { height: 33 }; const recurrentInput = new RecurrentZeros(); rnnCell(settings, input, recurrentInput); expect(setDimensionsSpy).toHaveBeenCalledWith(1, 33); }); }); it('properly sets up equation', () => { const input = mockLayer({ width: 1, height: 3 }); const settings = { height: 3 }; const recurrentInput = new RecurrentZeros(); const layer = rnnCell(settings, input, recurrentInput); const list = flattenLayers([layer]); expect(list.length).toBe(10); // model expect(list[0]).toBeInstanceOf(Random); expect(list[0].id).toBe('weight'); expect((list[0] as Random).settings.std).toBe(0.08); expect(list[1]).toBeInstanceOf(TestLayer); expect(list[1].id).toBe('MockLayer'); expect(list[2]).toBeInstanceOf(Multiply); expect(list[3]).toBeInstanceOf(Random); expect(list[3].id).toBe('transition'); expect((list[3] as Random).settings.std).toBe(0.08); expect(list[4]).toBeInstanceOf(RecurrentZeros); expect(list[5]).toBeInstanceOf(Multiply); expect(list[6]).toBeInstanceOf(Add); expect(list[7]).toBeInstanceOf(Zeros); expect(list[7].id).toBe('bias'); expect(list[8]).toBeInstanceOf(Add); expect(list[9]).toBeInstanceOf(Relu); }); }); ================================================ FILE: src/layer/rnn-cell.ts ================================================ import { add } from './add'; import { ILayer, ILayerSettings } from './base-layer'; import { multiply } from './multiply'; import { random } from './random'; import { relu } from './relu'; import { zeros } from './zeros'; import { IRecurrentInput } from './recurrent-input'; export function rnnCell( settings: ILayerSettings, input: ILayer, recurrentInput: IRecurrentInput ): ILayer { const { height } = settings; if (typeof height !== 'number') throw new Error('height not set'); if (recurrentInput.setDimensions) { recurrentInput.setDimensions(1, height); } // wxh const weight = random({ id: 'weight', height, width: input.height, std: 0.08, }); // whh const transition = random({ id: 'transition', height, width: height, std: 0.08, }); // bhh const bias = zeros({ id: 'bias', height }); return relu( add( add(multiply(weight, input), multiply(transition, recurrentInput)), bias ) ); } ================================================ FILE: src/layer/sigmoid.test.ts ================================================ import { gpuMock } from 'gpu-mock.js'; import { Sigmoid, sigmoid, predict2D, predict3D, compare2D, compare3D, } from './sigmoid'; import { IWithCompareKernel, IWithPredictKernel, mockLayer, mockPraxis, shave2D, shave3D, } from '../test-utils'; import * as sigmoidActivation from '../activation/sigmoid'; import { makeKernel } from '../utilities/kernel'; import { ILayerSettings } from './base-layer'; jest.mock('../utilities/kernel', () => { return { setup: jest.fn(), teardown: jest.fn(), makeKernel: jest.fn(() => { return [[1]]; }), release: jest.fn(), clear: jest.fn(), }; }); describe('Sigmoid Layer', () => { describe('predict2D() (forward propagation)', () => { test('can sigmoid a simple matrix', () => { const inputs = [ [0.1, 0.2, 0.3, 0.4], [0.5, 0.6, 0.7, 0.8], [0.9, 1, 1.1, 1.2], ]; const width = 4; const height = 3; const results = gpuMock(predict2D, { output: [width, height] })( inputs ) as Float32Array[]; expect(results.length).toBe(height); expect(results[0].length).toBe(width); expect(shave2D(results)).toEqual( shave2D([ Float32Array.from([0.52497917, 0.54983401, 0.57444251, 0.59868765]), Float32Array.from([0.62245935, 0.64565629, 0.6681878, 0.68997449]), Float32Array.from([0.71094948, 0.7310586, 0.75026011, 0.76852477]), ]) ); }); }); describe('predict3D() (forward propagation)', () => { test('can sigmoid a simple matrix', () => { const inputs = [ [ [0.1, 0.2, 0.3, 0.4], [0.5, 0.6, 0.7, 0.8], [0.9, 1, 1.1, 1.2], ], [ [0.1, 0.2, 0.3, 0.4], [0.5, 0.6, 0.7, 0.8], [0.9, 1, 1.1, 1.2], ], ]; const width = 4; const height = 3; const depth = 2; const results = gpuMock(predict3D, { output: [width, height, depth] })( inputs ) as Float32Array[][]; expect(results.length).toBe(depth); expect(results[0].length).toBe(height); expect(results[0][0].length).toBe(width); expect(shave3D(results)).toEqual( shave3D([ [ Float32Array.from([0.52497917, 0.54983401, 0.57444251, 0.59868765]), Float32Array.from([0.62245935, 0.64565629, 0.6681878, 0.68997449]), Float32Array.from([0.71094948, 0.7310586, 0.75026011, 0.76852477]), ], [ Float32Array.from([0.52497917, 0.54983401, 0.57444251, 0.59868765]), Float32Array.from([0.62245935, 0.64565629, 0.6681878, 0.68997449]), Float32Array.from([0.71094948, 0.7310586, 0.75026011, 0.76852477]), ], ]) ); }); }); describe('compare2D (back propagation)', () => { test('can sigmoid a simple matrix', () => { const inputs = [ [0.1, 0.2, 0.3, 0.4], [0.5, 0.6, 0.7, 0.8], [0.9, 1, 1.1, 1.2], ]; const deltas = [ [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1], ]; const width = 4; const height = 3; const results = gpuMock(compare2D, { output: [width, height] })( inputs, deltas ) as Float32Array[]; expect(results.length).toBe(height); expect(results[0].length).toBe(width); expect(shave2D(results)).toEqual( shave2D([ Float32Array.from([ 0.09000000000000001, 0.16000000000000003, 0.20999999, 0.23999999, ]), Float32Array.from([ 0.25, 0.23999999, 0.20999999, 0.15999999999999998, ]), Float32Array.from([0.08999999999999998, 0.0, -0.11, -0.23999999]), ]) ); }); }); describe('compare3D (back propagation)', () => { test('can sigmoid a simple matrix', () => { const inputs = [ [ [0.1, 0.2, 0.3, 0.4], [0.5, 0.6, 0.7, 0.8], [0.9, 1, 1.1, 1.2], ], [ [0.1, 0.2, 0.3, 0.4], [0.5, 0.6, 0.7, 0.8], [0.9, 1, 1.1, 1.2], ], ]; const deltas = [ [ [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1], ], [ [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1], ], ]; const width = 4; const height = 3; const depth = 2; const results = gpuMock(compare3D, { output: [width, height, depth] })( inputs, deltas ) as Float32Array[][]; expect(results.length).toBe(depth); expect(results[0].length).toBe(height); expect(results[0][0].length).toBe(width); expect(shave3D(results)).toEqual( shave3D([ [ Float32Array.from([ 0.09000000000000001, 0.16000000000000003, 0.20999999, 0.23999999, ]), Float32Array.from([ 0.25, 0.23999999, 0.20999999, 0.15999999999999998, ]), Float32Array.from([0.08999999999999998, 0.0, -0.11, -0.23999999]), ], [ Float32Array.from([ 0.09000000000000001, 0.16000000000000003, 0.20999999, 0.23999999, ]), Float32Array.from([ 0.25, 0.23999999, 0.20999999, 0.15999999999999998, ]), Float32Array.from([0.08999999999999998, 0.0, -0.11, -0.23999999]), ], ]) ); }); }); describe('.setupKernels()', () => { describe('2d', () => { it('sets up kernels correctly', () => { const width = 3; const height = 4; const mockInputLayer = mockLayer({ width, height }); const l = new Sigmoid(mockInputLayer); expect(l.predictKernel).toBe(null); expect(l.compareKernel).toBe(null); l.setupKernels(); expect(l.predictKernel).not.toBe(null); expect(l.compareKernel).not.toBe(null); expect(makeKernel).toHaveBeenCalledWith(predict2D, { functions: [sigmoidActivation.activate], immutable: true, output: [3, 4], }); expect(makeKernel).toHaveBeenCalledWith(compare2D, { functions: [sigmoidActivation.measure], immutable: true, output: [3, 4], }); }); }); describe('3d', () => { it('sets up kernels correctly', () => { const width = 3; const height = 4; const depth = 5; const mockInputLayer = mockLayer({ width, height, depth }); const l = new Sigmoid(mockInputLayer); expect(l.predictKernel).toBe(null); expect(l.compareKernel).toBe(null); l.setupKernels(); expect(l.predictKernel).not.toBe(null); expect(l.compareKernel).not.toBe(null); expect(makeKernel).toHaveBeenCalledWith(predict3D, { functions: [sigmoidActivation.activate], immutable: true, output: [3, 4, 5], }); expect(makeKernel).toHaveBeenCalledWith(compare3D, { functions: [sigmoidActivation.measure], immutable: true, output: [3, 4, 5], }); }); }); }); describe('.predict()', () => { it('calls this.predictKernel() with this.inputLayer.weights', () => { const mockWeights = [[new Float32Array(1)]]; const mockInputLayer = mockLayer({ weights: mockWeights, width: 1, height: 1, depth: 1, }); const l = new Sigmoid(mockInputLayer); ((l as unknown) as IWithPredictKernel).predictKernel = jest.fn( (weights) => weights ); l.predict(); expect(l.predictKernel).toBeCalledWith(mockWeights); expect(l.weights).toBe(mockWeights); }); }); describe('.compare()', () => { it('calls this.compareKernel() with this.inputLayer.weights & this.inputLayer.deltas', () => { const mockWeights = [[new Float32Array(1)]]; const mockDeltas = [[new Float32Array(1)]]; const mockInputLayer = mockLayer({ width: 1, height: 1, depth: 1, }); const l = new Sigmoid(mockInputLayer); l.weights = mockWeights; l.deltas = mockDeltas; ((l as unknown) as IWithCompareKernel).compareKernel = jest.fn( (weights, deltas) => deltas ); l.compare(); expect(l.compareKernel).toBeCalledWith(mockWeights, mockDeltas); expect(l.deltas).toBe(mockDeltas); }); }); describe('sigmoid lambda', () => { test('creates a new instance of Sigmoid and uses settings', () => { const width = 3; const height = 4; const depth = 5; const mockInputLayer = mockLayer({ width, height, depth }); const mockPraxisInstance = mockPraxis(mockInputLayer); const settings: ILayerSettings = { initPraxis: () => mockPraxisInstance }; const l = sigmoid(mockInputLayer, settings); expect(l.constructor).toBe(Sigmoid); expect(l.width).toBe(width); expect(l.height).toBe(height); expect(l.depth).toBe(depth); expect(l.praxis).toBe(mockPraxisInstance); }); }); }); ================================================ FILE: src/layer/sigmoid.ts ================================================ import { ILayer, ILayerSettings } from './base-layer'; import { IKernelFunctionThis, IKernelRunShortcut } from 'gpu.js'; import { Activation } from './types'; import { makeKernel, release } from '../utilities/kernel'; import { activate, measure } from '../activation/sigmoid'; export function predict2D( this: IKernelFunctionThis, inputs: number[][] ): number { return 1 / (1 + Math.exp(-inputs[this.thread.y][this.thread.x])); } export function predict3D( this: IKernelFunctionThis, inputs: number[][][] ): number { return ( 1 / (1 + Math.exp(-inputs[this.thread.z][this.thread.y][this.thread.x])) ); } export function compare2D( this: IKernelFunctionThis, weights: number[][], deltas: number[][] ): number { const weight = weights[this.thread.y][this.thread.x]; const delta = deltas[this.thread.y][this.thread.x]; return weight * (1 - weight) * delta; } export function compare3D( this: IKernelFunctionThis, weights: number[][][], deltas: number[][][] ): number { const weight = weights[this.thread.z][this.thread.y][this.thread.x]; const delta = deltas[this.thread.z][this.thread.y][this.thread.x]; return weight * (1 - weight) * delta; } export class Sigmoid extends Activation { setupKernels(): void { if (this.depth > 0) { this.predictKernel = makeKernel(predict3D, { output: [this.width, this.height, this.depth], functions: [activate], immutable: true, }); this.compareKernel = makeKernel(compare3D, { output: [this.width, this.height, this.depth], functions: [measure], immutable: true, }); } else { this.predictKernel = makeKernel(predict2D, { output: [this.width, this.height], functions: [activate], immutable: true, }); this.compareKernel = makeKernel(compare2D, { output: [this.width, this.height], functions: [measure], immutable: true, }); } } predict(): void { release(this.weights); this.weights = (this.predictKernel as IKernelRunShortcut)( this.inputLayer.weights ); } compare(): void { release(this.inputLayer.deltas); this.inputLayer.deltas = (this.compareKernel as IKernelRunShortcut)( this.weights, this.deltas ); } learn(learningRate?: number): void {} } export function sigmoid( inputLayer: ILayer, settings?: ILayerSettings ): Sigmoid { return new Sigmoid(inputLayer, settings); } ================================================ FILE: src/layer/soft-max.test.ts ================================================ import { GPU } from 'gpu.js'; import { gpuMock } from 'gpu-mock.js'; import { setup, teardown } from '../utilities/kernel'; import { compare, compare2D, compare3D, // getExponentials, getExponentials2D, getExponentials3D, // getMaxValue, getMaxValue2D, getMaxValue3D, // getSum, getSum2D, getSum3D, // predict, predict2D, predict3D, } from './soft-max'; describe('SoftMax', () => { beforeEach(() => { setup( new GPU({ mode: 'cpu', }) ); }); afterEach(() => { teardown(); }); describe('.compare', () => { it('can run on a simple matrix', () => { const exponentials = [1, 2, 3, 4]; const kernel = gpuMock(compare, { output: [4], }); expect(kernel(0, exponentials)).toEqual(Float32Array.from([-0, 2, 3, 4])); expect(kernel(1, exponentials)).toEqual(Float32Array.from([1, 1, 3, 4])); expect(kernel(2, exponentials)).toEqual(Float32Array.from([1, 2, 2, 4])); expect(kernel(3, exponentials)).toEqual(Float32Array.from([1, 2, 3, 3])); }); }); describe('.compare2D', () => { it('can run on a simple matrix', () => { const exponentials = [ [1, 2], [3, 4], ]; const kernel = gpuMock(compare2D, { output: [2, 2], }); expect(kernel(0, exponentials)).toEqual([ Float32Array.from([-0, 2]), Float32Array.from([3, 4]), ]); expect(kernel(1, exponentials)).toEqual([ Float32Array.from([1, 1]), Float32Array.from([3, 4]), ]); expect(kernel(2, exponentials)).toEqual([ Float32Array.from([1, 2]), Float32Array.from([2, 4]), ]); expect(kernel(3, exponentials)).toEqual([ Float32Array.from([1, 2]), Float32Array.from([3, 3]), ]); }); }); describe('.compare3D', () => { it('can run on a simple matrix', () => { const exponentials = [ [ [1, 2], [3, 4], ], [ [5, 6], [7, 8], ], ]; const kernel = gpuMock(compare3D, { output: [2, 2, 2], }); expect(kernel(0, exponentials)).toEqual([ [Float32Array.from([-0, 2]), Float32Array.from([3, 4])], [Float32Array.from([5, 6]), Float32Array.from([7, 8])], ]); expect(kernel(1, exponentials)).toEqual([ [Float32Array.from([1, 1]), Float32Array.from([3, 4])], [Float32Array.from([5, 6]), Float32Array.from([7, 8])], ]); expect(kernel(2, exponentials)).toEqual([ [Float32Array.from([1, 2]), Float32Array.from([2, 4])], [Float32Array.from([5, 6]), Float32Array.from([7, 8])], ]); expect(kernel(3, exponentials)).toEqual([ [Float32Array.from([1, 2]), Float32Array.from([3, 3])], [Float32Array.from([5, 6]), Float32Array.from([7, 8])], ]); expect(kernel(4, exponentials)).toEqual([ [Float32Array.from([1, 2]), Float32Array.from([3, 4])], [Float32Array.from([4, 6]), Float32Array.from([7, 8])], ]); expect(kernel(5, exponentials)).toEqual([ [Float32Array.from([1, 2]), Float32Array.from([3, 4])], [Float32Array.from([5, 5]), Float32Array.from([7, 8])], ]); expect(kernel(6, exponentials)).toEqual([ [Float32Array.from([1, 2]), Float32Array.from([3, 4])], [Float32Array.from([5, 6]), Float32Array.from([6, 8])], ]); expect(kernel(7, exponentials)).toEqual([ [Float32Array.from([1, 2]), Float32Array.from([3, 4])], [Float32Array.from([5, 6]), Float32Array.from([7, 7])], ]); }); }); describe('.getExponentials2D', () => { it('can run on a simple matrix', () => { const weights = [ [1, 2], [3, 4], ]; const kernel = gpuMock(getExponentials2D, { output: [2, 2], }); const result = kernel(weights, [0]); expect(result).toEqual([ new Float32Array([Math.exp(1), Math.exp(2)]), new Float32Array([Math.exp(3), Math.exp(4)]), ]); }); it('can subtract maxInput and run on a simple matrix', () => { const weights = [ [1, 2], [3, 4], ]; const kernel = gpuMock(getExponentials2D, { output: [2, 2], }); const result = kernel(weights, [4]); expect(result).toEqual([ new Float32Array([Math.exp(1 - 4), Math.exp(2 - 4)]), new Float32Array([Math.exp(3 - 4), Math.exp(4 - 4)]), ]); }); }); describe('.getExponentials3D', () => { it('can run on a simple matrix', () => { const weights = [ [ [1, 2], [3, 4], ], [ [5, 6], [7, 8], ], ]; const kernel = gpuMock(getExponentials3D, { output: [2, 2, 2], }); const result = kernel(weights, [0]); expect(result).toEqual([ [ new Float32Array([Math.exp(1), Math.exp(2)]), new Float32Array([Math.exp(3), Math.exp(4)]), ], [ new Float32Array([Math.exp(5), Math.exp(6)]), new Float32Array([Math.exp(7), Math.exp(8)]), ], ]); }); it('can subtract maxInput and run on a simple matrix', () => { const weights = [ [ [1, 2], [3, 4], ], [ [5, 6], [7, 8], ], ]; const kernel = gpuMock(getExponentials3D, { output: [2, 2, 2], }); const result = kernel(weights, [4]); expect(result).toEqual([ [ new Float32Array([Math.exp(1 - 4), Math.exp(2 - 4)]), new Float32Array([Math.exp(3 - 4), Math.exp(4 - 4)]), ], [ new Float32Array([Math.exp(5 - 4), Math.exp(6 - 4)]), new Float32Array([Math.exp(7 - 4), Math.exp(8 - 4)]), ], ]); }); }); describe('.getMaxValue2D', () => { it('can run on a simple matrix', () => { const weights = [ [1, 2], [3, 4], ]; const kernel = gpuMock(getMaxValue2D, { output: [1], constants: { inputWidth: 2, inputHeight: 2, }, }); const result = kernel(weights); expect(result).toEqual(Float32Array.from([4])); }); }); describe('.getMaxValue3D', () => { it('can run on a simple matrix', () => { const weights = [ [ [1, 2], [3, 4], ], [ [5, 6], [7, 8], ], ]; const kernel = gpuMock(getMaxValue3D, { output: [1], constants: { inputWidth: 2, inputHeight: 2, inputDepth: 2, }, }); const result = kernel(weights); expect(result).toEqual(Float32Array.from([8])); }); }); describe('.getSum2D', () => { it('can run on a simple matrix', () => { const weights = [ [1, 2], [3, 4], ]; const kernel = gpuMock(getSum2D, { output: [1], constants: { inputWidth: 2, inputHeight: 2, }, }); const result = kernel(weights); expect(result).toEqual(Float32Array.from([10])); }); }); describe('.getSum3D', () => { it('can run on a simple matrix', () => { const weights = [ [ [1, 2], [3, 4], ], [ [5, 6], [7, 8], ], ]; const kernel = gpuMock(getSum3D, { output: [1], constants: { inputWidth: 2, inputHeight: 2, inputDepth: 2, }, }); const result = kernel(weights); expect(result).toEqual(Float32Array.from([36])); }); }); describe('.predict2D', () => { it('can run on a simple matrix', () => { const weights = [ [1, 2], [3, 4], ]; const kernel = gpuMock(predict2D, { output: [2, 2], }); const result = kernel(weights, [2]); expect(result).toEqual([ Float32Array.from([0.5, 1]), Float32Array.from([1.5, 2]), ]); }); }); describe('.predict3D', () => { it('can run on a simple matrix', () => { const weights = [ [ [1, 2], [3, 4], ], [ [5, 6], [7, 8], ], ]; const kernel = gpuMock(predict3D, { output: [2, 2, 2], }); const result = kernel(weights, [2]); expect(result).toEqual([ [Float32Array.from([0.5, 1]), Float32Array.from([1.5, 2])], [Float32Array.from([2.5, 3]), Float32Array.from([3.5, 4])], ]); }); }); }); ================================================ FILE: src/layer/soft-max.ts ================================================ import { IConstantsThis, IKernelFunctionThis, IKernelRunShortcut, KernelOutput, Texture, } from 'gpu.js'; import { makeKernel, release, clone } from '../utilities/kernel'; import { randos, randos2D, randos3D } from '../utilities/randos'; import { zeros } from '../utilities/zeros'; import { zeros2D } from '../utilities/zeros-2d'; import { zeros3D } from '../utilities/zeros-3d'; import { ILayer, ILayerSettings } from './base-layer'; import { Modifier } from './modifier'; interface ISoftMaxConstants extends IConstantsThis { inputWidth: number; } export function getMaxValue( this: IKernelFunctionThis, inputs: number[] ): number { let maxInput = -Infinity; for (let x = 0; x < this.constants.inputWidth; x++) { const input = inputs[x]; if (input > maxInput) { maxInput = input; } } return maxInput; } export function getMaxValue2D( this: IKernelFunctionThis, inputs: number[][] ): number { let maxInput = -Infinity; for (let y = 0; y < this.constants.inputHeight; y++) { for (let x = 0; x < this.constants.inputWidth; x++) { const input = inputs[y][x]; if (input > maxInput) { maxInput = input; } } } return maxInput; } export function getMaxValue3D( this: IKernelFunctionThis, inputs: number[][][] ): number { let maxInput = -Infinity; for (let z = 0; z < this.constants.inputDepth; z++) { for (let y = 0; y < this.constants.inputHeight; y++) { for (let x = 0; x < this.constants.inputWidth; x++) { const input = inputs[z][y][x]; if (input > maxInput) { maxInput = input; } } } } return maxInput; } export function getSum( this: IKernelFunctionThis, inputs: number[] ): number { let sum = 0; for (let x = 0; x < this.constants.inputWidth; x++) { sum += inputs[x]; } return sum; } export function getSum2D( this: IKernelFunctionThis, inputs: number[][] ): number { let sum = 0; for (let y = 0; y < this.constants.inputHeight; y++) { for (let x = 0; x < this.constants.inputWidth; x++) { sum += inputs[y][x]; } } return sum; } export function getSum3D( this: IKernelFunctionThis, inputs: number[][][] ): number { let sum = 0; for (let z = 0; z < this.constants.inputDepth; z++) { for (let y = 0; y < this.constants.inputHeight; y++) { for (let x = 0; x < this.constants.inputWidth; x++) { sum += inputs[z][y][x]; } } } return sum; } export function getExponentials( this: IKernelFunctionThis, inputs: number[], maxInput: number[] ): number { return Math.exp(inputs[this.thread.x] - maxInput[0]); } export function getExponentials2D( this: IKernelFunctionThis, inputs: number[][], maxInput: number[] ): number { return Math.exp(inputs[this.thread.y][this.thread.x] - maxInput[0]); } export function getExponentials3D( this: IKernelFunctionThis, inputs: number[][][], maxInput: number[] ): number { return Math.exp( inputs[this.thread.z][this.thread.y][this.thread.x] - maxInput[0] ); } export function predict( this: IKernelFunctionThis, exponentials: number[], exponentialsSum: number[] ): number { return exponentials[this.thread.x] / exponentialsSum[0]; } export function predict2D( this: IKernelFunctionThis, exponentials: number[][], exponentialsSum: number[] ): number { return exponentials[this.thread.y][this.thread.x] / exponentialsSum[0]; } export function predict3D( this: IKernelFunctionThis, exponentials: number[][][], exponentialsSum: number[] ): number { return ( exponentials[this.thread.z][this.thread.y][this.thread.x] / exponentialsSum[0] ); } export function compare( this: IKernelFunctionThis, target: number, exponentials: number[] ): number { let indicator = 0; if (this.thread.x === target) { indicator = 1; } return -(indicator - exponentials[this.thread.x]); } export function compare2D( this: IKernelFunctionThis, target: number, exponentials: number[][] ): number { let indicator = 0; const index = this.thread.x + this.thread.y * this.output.x; if (index === target) { indicator = 1; } return -(indicator - exponentials[this.thread.y][this.thread.x]); } export function compare3D( this: IKernelFunctionThis, target: number, exponentials: number[][][] ): number { let indicator = 0; const index = this.thread.x + this.thread.y * this.output.x + this.thread.z * this.output.x * this.output.y; if (index === target) { indicator = 1; } return -( indicator - exponentials[this.thread.z][this.thread.y][this.thread.x] ); } export function loss(): number { return -Math.log(0); } // TODO: handle: `return -Math.log(this.es[y]);` in learn export class SoftMax extends Modifier { getExponentialsKernel: IKernelRunShortcut | null; getMaxValueKernel: IKernelRunShortcut | null; getSumKernel: IKernelRunShortcut | null; errors: KernelOutput | null = null; constructor(inputLayer: ILayer, settings?: ILayerSettings) { super(inputLayer, settings); this.getExponentialsKernel = null; this.getMaxValueKernel = null; this.getSumKernel = null; this.validate(); if (this.depth > 0) { this.weights = randos3D(this.width, this.height, this.depth); this.deltas = zeros3D(this.width, this.height, this.depth); } else if (this.height > 0) { this.weights = randos2D(this.width, this.height); this.deltas = zeros2D(this.width, this.height); } else { this.weights = randos(this.width); this.deltas = zeros(this.width); } } setupKernels(): void { const { width, height, depth } = this; if (depth > 0) { this.getExponentialsKernel = makeKernel(getExponentials3D, { output: [width, height, depth], }); this.getMaxValueKernel = makeKernel(getMaxValue3D, { output: [1, 1, 1], constants: { inputWidth: width, inputHeight: height, inputDepth: depth, }, }); this.getSumKernel = makeKernel(getSum3D, { output: [1, 1, 1], constants: { inputWidth: width, inputHeight: height, inputDepth: depth, }, }); this.predictKernel = makeKernel(predict3D, { output: [width, height, depth], }); this.compareKernel = makeKernel(compare3D, { output: [width, height, depth], immutable: true, }); } else { this.getExponentialsKernel = makeKernel(getExponentials, { output: [width, height], }); this.getMaxValueKernel = makeKernel(getMaxValue2D, { output: [1, 1], constants: { inputWidth: width, inputHeight: height, }, }); this.getSumKernel = makeKernel(getSum2D, { output: [1, 1], constants: { inputWidth: width, inputHeight: height, }, }); this.predictKernel = makeKernel(predict2D, { output: [width, height], }); this.compareKernel = makeKernel(compare2D, { output: [width, height], immutable: true, }); } } predict(): void { const maxValue = (this.getMaxValueKernel as IKernelRunShortcut)( this.inputLayer.weights ); const exponentials = (this.getExponentialsKernel as IKernelRunShortcut)( this.inputLayer.weights, maxValue ); const exponentialsSum = (this.getSumKernel as IKernelRunShortcut)( exponentials ); this.weights = (this.predictKernel as IKernelRunShortcut)( exponentials, exponentialsSum ); } compare(targetValues: KernelOutput): void { const { deltas, errors } = this; this.errors = (this.compareKernel as IKernelRunShortcut)( (targetValues as number[])[0], deltas ); this.deltas = clone(this.errors); release(deltas); release(errors as Texture); const inputLayerDeltas = this.inputLayer.deltas; this.inputLayer.deltas = clone(this.deltas); release(inputLayerDeltas); } } export function softMax( inputLayer: ILayer, settings?: ILayerSettings ): SoftMax { return new SoftMax(inputLayer, settings); } ================================================ FILE: src/layer/svm.ts ================================================ import { KernelOutput } from 'gpu.js'; import { BaseLayer, ILayer, ILayerSettings } from './base-layer'; import { clone, release } from '../utilities/kernel'; export class SVM extends BaseLayer { inputLayer: ILayer; constructor(inputLayer: ILayer, settings: ILayerSettings) { super(settings); this.inputLayer = inputLayer; } predict(): void { release(this.weights); this.weights = clone(this.inputLayer.weights as KernelOutput); this.validate(); } learn(): void { // throw new Error(`${this.constructor.name}-learn is not yet implemented`) } } // function learn(target) { // if (y === i) { // continue; // } // const ydiff = -yscore + x.w[i] + margin; // if (ydiff > 0) { // // violating dimension, apply loss // x.dw[i] += 1; // x.dw[y] -= 1; // loss += ydiff; // } // } export function svm(inputLayer: ILayer, settings: ILayerSettings): SVM { return new SVM(inputLayer, settings); } ================================================ FILE: src/layer/tanh.test.ts ================================================ import { GPU } from 'gpu.js'; import { gpuMock } from 'gpu-mock.js'; import { Tanh, tanh, predict2D, predict3D, compare2D, compare3D } from './tanh'; import * as tanhActivation from '../activation/tanh'; import { IWithCompareKernel, IWithPredictKernel, mockLayer, mockPraxis, shave2D, shave3D, } from '../test-utils'; import { makeKernel, setup, teardown } from '../utilities/kernel'; import { randos2D } from '../utilities/randos'; import { ILayerSettings } from './base-layer'; jest.mock('../utilities/kernel', () => { return { setup: jest.fn(), teardown: jest.fn(), makeKernel: jest.fn(() => { return [[1]]; }), release: jest.fn(), clear: jest.fn(), }; }); describe('Tanh Layer', () => { describe('predict2D() (forward propagation)', () => { test('can tanh a simple matrix', () => { const inputs = [ [0.1, 0.2, 0.3, 0.4], [0.5, 0.6, 0.7, 0.8], [0.9, 1, 1.1, 1.2], ]; const width = 4; const height = 3; const results = gpuMock(predict2D, { output: [width, height] })( inputs ) as Float32Array[]; expect(results.length).toBe(height); expect(results[0].length).toBe(width); expect(shave2D(results)).toEqual( shave2D([ Float32Array.from([0.099668, 0.19737533, 0.29131261, 0.37994897]), Float32Array.from([0.46211717, 0.53704959, 0.60436779, 0.66403675]), Float32Array.from([0.71629786, 0.76159418, 0.80049902, 0.83365458]), ]) ); }); }); describe('predict3D() (forward propagation)', () => { test('can tanh a simple matrix', () => { const inputs = [ [ [0.1, 0.2, 0.3, 0.4], [0.5, 0.6, 0.7, 0.8], [0.9, 1, 1.1, 1.2], ], [ [0.1, 0.2, 0.3, 0.4], [0.5, 0.6, 0.7, 0.8], [0.9, 1, 1.1, 1.2], ], ]; const width = 4; const height = 3; const depth = 2; const results = gpuMock(predict3D, { output: [width, height, depth] })( inputs ) as Float32Array[][]; expect(results.length).toBe(depth); expect(results[0].length).toBe(height); expect(results[0][0].length).toBe(width); expect(shave3D(results)).toEqual( shave3D([ [ Float32Array.from([0.099668, 0.19737533, 0.29131261, 0.37994897]), Float32Array.from([0.46211717, 0.53704959, 0.60436779, 0.66403675]), Float32Array.from([0.71629786, 0.76159418, 0.80049902, 0.83365458]), ], [ Float32Array.from([0.099668, 0.19737533, 0.29131261, 0.37994897]), Float32Array.from([0.46211717, 0.53704959, 0.60436779, 0.66403675]), Float32Array.from([0.71629786, 0.76159418, 0.80049902, 0.83365458]), ], ]) ); }); }); describe('compare2D() (back propagation)', () => { test('can tanh a simple matrix', () => { const inputs = [ [0.1, 0.2, 0.3, 0.4], [0.5, 0.6, 0.7, 0.8], [0.9, 1, 1.1, 1.2], ]; const deltas = [ [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1], ]; const width = 4; const height = 3; const results = gpuMock(compare2D, { output: [width, height] })( inputs, deltas ) as Float32Array[]; expect(results.length).toBe(height); expect(results[0].length).toBe(width); expect(shave2D(results)).toEqual( shave2D([ Float32Array.from([0.99000001, 0.95999998, 0.91000003, 0.83999997]), Float32Array.from([0.75, 0.63999999, 0.50999999, 0.36000001]), Float32Array.from([0.19, 0.0, -0.20999999, -0.44]), ]) ); }); }); describe('compare3D() (back propagation)', () => { test('can tanh a simple matrix', () => { const inputs = [ [ [0.1, 0.2, 0.3, 0.4], [0.5, 0.6, 0.7, 0, 8], [0.9, 1, 1.1, 1.2], ], [ [0.1, 0.2, 0.3, 0.4], [0.5, 0.6, 0.7, 0, 8], [0.9, 1, 1.1, 1.2], ], ]; const deltas = [ [ [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1], ], [ [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1], ], ]; const width = 4; const height = 3; const depth = 2; const results = gpuMock(compare3D, { output: [width, height, depth] })( inputs, deltas ) as Float32Array[][]; expect(results.length).toBe(depth); expect(results[0].length).toBe(height); expect(results[0][0].length).toBe(width); expect(shave3D(results)).toEqual( shave3D([ [ Float32Array.from([0.99000001, 0.95999998, 0.91000003, 0.83999997]), Float32Array.from([0.75, 0.63999999, 0.50999999, 1]), Float32Array.from([0.19, 0.0, -0.20999999, -0.44]), ], [ Float32Array.from([0.99000001, 0.95999998, 0.91000003, 0.83999997]), Float32Array.from([0.75, 0.63999999, 0.50999999, 1]), Float32Array.from([0.19, 0.0, -0.20999999, -0.44]), ], ]) ); }); }); describe('.setupKernels()', () => { beforeEach(() => { setup( new GPU({ mode: 'cpu', }) ); }); afterEach(() => { teardown(); }); describe('2d', () => { it('sets up kernels correctly', () => { const width = 3; const height = 4; const mockInputLayer = mockLayer({ width, height }); const l = new Tanh(mockInputLayer); expect(l.predictKernel).toBe(null); expect(l.compareKernel).toBe(null); l.setupKernels(); expect(l.predictKernel).not.toBe(null); expect(l.compareKernel).not.toBe(null); expect(makeKernel).toHaveBeenCalledWith(predict2D, { functions: [tanhActivation.activate], immutable: true, output: [3, 4], }); expect(makeKernel).toHaveBeenCalledWith(compare2D, { functions: [tanhActivation.measure], immutable: true, output: [3, 4], }); }); }); describe('3d', () => { it('sets up kernels correctly', () => { const width = 3; const height = 4; const depth = 5; const mockInputLayer = mockLayer({ width, height, depth }); const l = new Tanh(mockInputLayer); expect(l.predictKernel).toBe(null); expect(l.compareKernel).toBe(null); l.setupKernels(); expect(l.predictKernel).not.toBe(null); expect(l.compareKernel).not.toBe(null); expect(makeKernel).toHaveBeenCalledWith(predict3D, { functions: [tanhActivation.activate], immutable: true, output: [3, 4, 5], }); expect(makeKernel).toHaveBeenCalledWith(compare3D, { functions: [tanhActivation.measure], immutable: true, output: [3, 4, 5], }); }); }); }); describe('.predict()', () => { it('calls this.predictKernel() with this.inputLayer.weights', () => { const mockWeights = randos2D(1, 1); const mockInputLayer = mockLayer({ weights: mockWeights, width: 1, height: 1, depth: 1, }); const l = new Tanh(mockInputLayer); ((l as unknown) as IWithPredictKernel).predictKernel = jest.fn( (weights) => weights ); l.predict(); expect(l.predictKernel).toBeCalledWith(mockWeights); expect(l.weights).toBe(mockWeights); }); }); describe('.compare()', () => { it('calls this.compareKernel() with this.inputLayer.weights & this.inputLayer.deltas', () => { const mockWeights = randos2D(1, 1); const mockDeltas = randos2D(1, 1); const mockInputLayer = mockLayer({ width: 1, height: 1, depth: 1, }); const l = new Tanh(mockInputLayer); l.weights = mockWeights; l.deltas = mockDeltas; const expected = randos2D(1, 1); ((l as unknown) as IWithCompareKernel).compareKernel = jest.fn( (weights, deltas) => expected ); l.compare(); expect(l.compareKernel).toBeCalledWith(mockWeights, mockDeltas); expect(l.inputLayer.deltas).toBe(expected); }); }); describe('tanh lambda', () => { test('creates a new instance of Tanh', () => { const width = 3; const height = 4; const depth = 5; const mockInputLayer = mockLayer({ width, height, depth }); const mockPraxisInstance = mockPraxis(mockInputLayer); const settings: ILayerSettings = { initPraxis: () => mockPraxisInstance }; const l = tanh(mockInputLayer, settings); expect(l.constructor).toBe(Tanh); expect(l.width).toBe(width); expect(l.height).toBe(height); expect(l.depth).toBe(depth); expect(l.praxis).toBe(mockPraxisInstance); }); }); }); ================================================ FILE: src/layer/tanh.ts ================================================ import { IKernelFunctionThis, IKernelRunShortcut } from 'gpu.js'; import { Activation } from './activation'; import { activate, measure } from '../activation/tanh'; import { release, makeKernel } from '../utilities/kernel'; import { ILayer, ILayerSettings } from './base-layer'; export function predict2D( this: IKernelFunctionThis, inputs: number[][] ): number { return activate(inputs[this.thread.y][this.thread.x]); } export function predict3D( this: IKernelFunctionThis, inputs: number[][][] ): number { return activate(inputs[this.thread.z][this.thread.y][this.thread.x]); } export function compare2D( this: IKernelFunctionThis, weights: number[][], errors: number[][] ): number { return measure( weights[this.thread.y][this.thread.x], errors[this.thread.y][this.thread.x] ); } export function compare3D( this: IKernelFunctionThis, weights: number[][][], errors: number[][][] ): number { return measure( weights[this.thread.z][this.thread.y][this.thread.x], errors[this.thread.z][this.thread.y][this.thread.x] ); } export class Tanh extends Activation { setupKernels(): void { if (this.depth > 0) { this.predictKernel = makeKernel(predict3D, { output: [this.width, this.height, this.depth], functions: [activate], immutable: true, }); this.compareKernel = makeKernel(compare3D, { output: [this.width, this.height, this.depth], functions: [measure], immutable: true, }); } else { this.predictKernel = makeKernel(predict2D, { output: [this.width, this.height], functions: [activate], immutable: true, }); this.compareKernel = makeKernel(compare2D, { output: [this.width, this.height], functions: [measure], immutable: true, }); } } predict(): void { release(this.weights); this.weights = (this.predictKernel as IKernelRunShortcut)( this.inputLayer.weights ); } compare(): void { release(this.inputLayer.deltas); this.inputLayer.deltas = (this.compareKernel as IKernelRunShortcut)( this.weights, this.deltas ); } } export function tanh(inputLayer: ILayer, settings?: ILayerSettings): Tanh { return new Tanh(inputLayer, settings); } ================================================ FILE: src/layer/target.test.ts ================================================ import { GPU } from 'gpu.js'; import { compare1D, compare2D, Target } from './target'; import { setup, teardown, makeKernel } from '../utilities/kernel'; import { IWithCompareKernel, mockLayer } from '../test-utils'; jest.mock('../utilities/kernel', () => { return { setup: jest.fn(), teardown: jest.fn(), makeKernel: jest.fn(() => { return [[1]]; }), release: jest.fn(), clear: jest.fn(), clone: jest.fn(), }; }); describe('Target Layer', () => { beforeEach(() => { setup( new GPU({ mode: 'cpu', }) ); // Clear the mock call count before each test (makeKernel as jest.Mock).mockClear(); }); afterEach(() => { teardown(); }); test('is fully back propagating values to deltas', () => { const input = mockLayer({ width: 1, height: 1, weights: [[1]], deltas: [[0]], }); const target = new Target({ width: 1, height: 1 }, input); target.validate(); target.setupKernels(); target.predict(); ((target as unknown) as IWithCompareKernel).compareKernel = jest.fn(() => [ new Float32Array([1]), ]); target.compare([[0]]); expect(target.deltas).toEqual([new Float32Array([1])]); }); test('uses compare1D when width = 1', () => { const target = new Target( { height: 10, width: 1 }, mockLayer({ height: 10, width: 1 }) ); target.setupKernels(); expect(makeKernel).toBeCalledTimes(1); expect(makeKernel).toHaveBeenCalledWith(compare1D, { output: [1, 10], immutable: true, }); }); test('uses compare2D when width > 1', () => { const target = new Target( { width: 10, height: 10 }, mockLayer({ height: 10, width: 10 }) ); target.setupKernels(); expect(makeKernel).toBeCalledTimes(1); expect(makeKernel).toHaveBeenCalledWith(compare2D, { output: [10, 10], immutable: true, }); }); }); ================================================ FILE: src/layer/target.ts ================================================ import { IKernelFunctionThis, IKernelRunShortcut, KernelOutput } from 'gpu.js'; import { makeKernel, release, clone } from '../utilities/kernel'; import { zeros } from '../utilities/zeros'; import { zeros2D } from '../utilities/zeros-2d'; import { BaseLayer, ILayer, ILayerSettings } from './base-layer'; export function compare1D( this: IKernelFunctionThis, weights: number[][], targetValues: number[] ): number { return weights[this.thread.y][this.thread.x] - targetValues[this.thread.x]; } export function compare2D( this: IKernelFunctionThis, weights: number[][], targetValues: number[][] ): number { return ( weights[this.thread.y][this.thread.x] - targetValues[this.thread.y][this.thread.x] ); } export type TargetType = new ( settings: Partial, inputLayer: ILayer ) => ILayer; export class Target extends BaseLayer { errors: KernelOutput; inputLayer: ILayer; constructor(settings: Partial, inputLayer: ILayer) { super(settings); this.inputLayer = inputLayer; this.validate(); if (this.depth) { throw new Error('Target layer not implemented for depth'); } else if (this.height) { this.weights = zeros2D(this.width, this.height); this.deltas = zeros2D(this.width, this.height); this.errors = zeros2D(this.width, this.height); } else { this.weights = zeros(this.width); this.deltas = zeros(this.width); this.errors = zeros(this.width); } } setupKernels(): void { if (this.width === 1) { this.compareKernel = makeKernel(compare1D, { output: [this.width, this.height], immutable: true, }); } else { this.compareKernel = makeKernel(compare2D, { output: [this.width, this.height], immutable: true, }); } } predict(): void { // TODO: should we clone here? // NOTE: this looks like it shouldn't be, but the weights are immutable, and this is where they are reused. release(this.weights); this.weights = clone(this.inputLayer.weights as KernelOutput); } compare(targetValues: KernelOutput): void { // this is where weights attach to deltas // deltas will be zero on learn, so save it in error for comparing to mse later release(this.deltas); release(this.errors); release(this.inputLayer.deltas); this.deltas = (this.compareKernel as IKernelRunShortcut)( this.weights, targetValues ); this.inputLayer.deltas = clone(this.deltas); this.errors = clone(this.deltas); } setupPraxis(): void {} } export function target(settings: ILayerSettings, inputLayer: ILayer): Target { return new Target(settings, inputLayer); } ================================================ FILE: src/layer/transpose.ts ================================================ import { IKernelFunctionThis, IKernelRunShortcut } from 'gpu.js'; import { makeKernel } from '../utilities/kernel'; import { ILayer } from './base-layer'; import { Modifier } from './types'; export function predict(this: IKernelFunctionThis, value: number[][]): number { return value[this.thread.x][this.thread.y]; } const compare = predict; export class Transpose extends Modifier { get width(): number { return this.inputLayer.height; } get height(): number { return this.inputLayer.width; } constructor(inputLayer: ILayer) { super(inputLayer); this.validate(); } setupKernels(): void { this.predictKernel = makeKernel(predict, { output: [this.height, this.width], }); this.compareKernel = makeKernel(compare, { output: [this.width, this.height], }); } predict(): void { this.weights = (this.predictKernel as IKernelRunShortcut)( this.inputLayer.weights ); } compare(): void { this.inputLayer.deltas = (this.compareKernel as IKernelRunShortcut)( this.deltas ); } } export function transpose(inputLayer: ILayer): Transpose { return new Transpose(inputLayer); } ================================================ FILE: src/layer/types.ts ================================================ import { release } from '../utilities/kernel'; import { BaseLayer, ILayer, ILayerSettings } from './base-layer'; export { Activation } from './activation'; export { Filter } from './filter'; export { Internal } from './internal'; export { Modifier } from './modifier'; export { Operator } from './operator'; export { Target } from './target'; // eslint-disable-next-line @typescript-eslint/no-extraneous-class export class InternalModel {} export type EntryPointType = new (settings: Partial) => ILayer; // eslint-disable-next-line @typescript-eslint/no-extraneous-class export class EntryPoint extends BaseLayer {} // eslint-disable-next-line @typescript-eslint/no-extraneous-class export class Model extends BaseLayer { learn(learningRate?: number): void { // TODO: do we need to release here? const { weights: oldWeights } = this; if (!this.praxis) throw new Error('this.praxis not defined'); this.weights = this.praxis.run(this, learningRate as number); release(oldWeights); } } ================================================ FILE: src/layer/zeros.ts ================================================ import { zeros2D } from '../utilities/zeros-2d'; import { Model } from './types'; import { ILayerSettings } from './base-layer'; export class Zeros extends Model { constructor(settings: ILayerSettings) { super(settings); this.validate(); this.weights = zeros2D(this.width, this.height); this.deltas = zeros2D(this.width, this.height); } predict(): void { // throw new Error(`${this.constructor.name}-predict is not yet implemented`) } compare(): void { // throw new Error(`${this.constructor.name}-compare is not yet implemented`) } // learn(): void {} } export function zeros(settings: ILayerSettings): Zeros { return new Zeros(settings); } ================================================ FILE: src/likely.test.ts ================================================ import { likely } from './likely'; import { NeuralNetwork } from './neural-network'; /** * Return 0 or 1 for '#' */ function integer(character: string): number { if (character === '#') return 1; return 0; } /** * Turn the # into 1s and . into 0s. for whole string */ function character(string: string): number[] { return string.trim().split('').map(integer); } describe('likely', () => { const a = character( '.#####.' + '#.....#' + '#.....#' + '#######' + '#.....#' + '#.....#' + '#.....#' ); const b = character( '######.' + '#.....#' + '#.....#' + '######.' + '#.....#' + '#.....#' + '######.' ); const c = character( '#######' + '#......' + '#......' + '#......' + '#......' + '#......' + '#######' ); /** * Learn the letters A through C. */ const net = new NeuralNetwork(); net.train([ { input: a, output: { a: 1 } }, { input: b, output: { b: 1 } }, { input: c, output: { c: 1 } }, ]); it('should be able to find a', () => { /** * Predict the letter A, even with a pixel off. */ const result = likely( character( '.#####.' + '#.....#' + '#.....#' + '###.###' + '#.....#' + '#.....#' + '#.....#' ), net ); expect(result).toBe('a'); }); it('should be able to find b', () => { /** * Predict the letter B, even with a pixel off. */ const result = likely( character( '######.' + '#.....#' + '#.....#' + '######.' + '#..#..#' + '#.....#' + '###.##.' ), net ); expect(result).toBe('b'); }); it('should be able to find c', () => { /** * Predict the letter C, even with a pixel off. */ const result = likely( character( '#######' + '#......' + '#......' + '#......' + '#......' + '##.....' + '#######' ), net ); expect(result).toBe('c'); }); }); ================================================ FILE: src/likely.ts ================================================ export interface ILikelyNet { run: (input: InputType) => OutputType; } export function likely< NetworkType extends ILikelyNet< Parameters[0], ReturnType > >( input: Parameters[0], net: NetworkType ): ReturnType | null { if (!net) { throw new TypeError( `Required parameter 'net' is of type ${typeof net}. Must be of type 'brain.NeuralNetwork'` ); } const output = net.run(input); let maxProp = null; let maxValue = -1; Object.entries(output as number[]).forEach(([key, value]) => { if ( typeof value !== 'undefined' && typeof value === 'number' && value > maxValue ) { maxProp = key; maxValue = value; } }); return maxProp; } ================================================ FILE: src/lookup.test.ts ================================================ import { lookup } from './lookup'; describe('lookup', () => { it('toHash()', () => { const lup = lookup.toHash({ a: 6, b: 7, c: 8 }); expect(lup).toEqual({ a: 0, b: 1, c: 2 }); }); it('toTable()', () => { const lup = lookup.toTable([ { x: 0, y: 0 }, { x: 1, z: 0 }, { q: 0 }, { x: 1, y: 1 }, ]); expect(lup).toEqual({ x: 0, y: 1, z: 2, q: 3 }); }); it('toArray()', () => { const lup = { a: 0, b: 1, c: 2 }; const array = lookup.toArray(lup, { b: 8, notinlookup: 9 }, 3); expect(array).toEqual(Float32Array.from([0, 8, 0])); }); it('toObject()', () => { const lup = { b: 1, a: 0, c: 2 }; const hash = lookup.toObject(lup, [0, 9, 8]); expect(hash).toEqual({ a: 0, b: 9, c: 8 }); }); describe('dataShape', () => { describe('collection usage', () => { it('can identify array,array,number', () => { const individual = lookup.dataShape([0]); const collection = lookup.dataShape([[0]]); expect(individual).toEqual(['array', 'number']); expect(collection).toEqual(['array', 'array', 'number']); }); it('can identify array,array,array,number', () => { const individual = lookup.dataShape([[0]]); const collection = lookup.dataShape([[[0]]]); expect(individual).toEqual(['array', 'array', 'number']); expect(collection).toEqual(['array', 'array', 'array', 'number']); }); it('can identify array,object,number', () => { const individual = lookup.dataShape({ one: 0 }); const collection = lookup.dataShape([{ one: 0 }]); expect(individual).toEqual(['object', 'number']); expect(collection).toEqual(['array', 'object', 'number']); }); it('can identify array,array,object,number', () => { const individual = lookup.dataShape([{ one: 0 }]); const collection = lookup.dataShape([[{ one: 0 }]]); expect(individual).toEqual(['array', 'object', 'number']); expect(collection).toEqual(['array', 'array', 'object', 'number']); }); it('can identify array,datum,array,number', () => { const individual = lookup.dataShape({ input: [0], output: [0] }); const collection = lookup.dataShape([{ input: [0], output: [0] }]); expect(individual).toEqual(['datum', 'array', 'number']); expect(collection).toEqual(['array', 'datum', 'array', 'number']); }); it('can identify array,datum,object,number', () => { const individual = lookup.dataShape({ input: { one: 0 }, output: { none: 0 }, }); const collection = lookup.dataShape([ { input: { one: 0 }, output: { none: 0 } }, ]); expect(individual).toEqual(['datum', 'object', 'number']); expect(collection).toEqual(['array', 'datum', 'object', 'number']); }); it('can identify array,datum,array,array,number', () => { const individual = lookup.dataShape({ input: [[0]], output: [[0]] }); const collection = lookup.dataShape([{ input: [[0]], output: [[0]] }]); expect(individual).toEqual(['datum', 'array', 'array', 'number']); expect(collection).toEqual([ 'array', 'datum', 'array', 'array', 'number', ]); }); it('can identify array,datum,array,object,number', () => { const individual = lookup.dataShape({ input: [{ one: 0 }], output: [{ one: 0 }], }); const collection = lookup.dataShape([ { input: [{ one: 0 }], output: [{ one: 0 }] }, ]); expect(individual).toEqual(['datum', 'array', 'object', 'number']); expect(collection).toEqual([ 'array', 'datum', 'array', 'object', 'number', ]); }); }); }); }); ================================================ FILE: src/lookup.ts ================================================ import { KernelOutput } from 'gpu.js'; export interface INumberHash { [character: string]: number; } export interface INumberArray { length: number; buffer?: ArrayBuffer; [index: number]: number; } export type InputOutputValue = INumberArray | Partial; export interface ITrainingDatum { input: InputOutputValue | InputOutputValue[] | KernelOutput; output: InputOutputValue | InputOutputValue[] | KernelOutput; } export type FormattableData = | number | ITrainingDatum | InputOutputValue | InputOutputValue[]; /* Functions for turning sparse hashes into arrays and vice versa */ export const lookup = { /** * Performs `[{a: 1}, {b: 6, c: 7}] -> {a: 0, b: 1, c: 2}` * @param {Object} hashes * @returns {Object} */ toTable(hashes: INumberHash[]): INumberHash { const hash = hashes.reduce((memo, hash) => { return Object.assign(memo, hash); }, {}); return lookup.toHash(hash); }, /** * Performs `[{a: 1}, {b: 6, c: 7}] -> {a: 0, b: 1, c: 2}` */ toTable2D(objects2D: INumberHash[][]): INumberHash { const table: INumberHash = {}; let valueIndex = 0; for (let i = 0; i < objects2D.length; i++) { const objects = objects2D[i]; for (let j = 0; j < objects.length; j++) { const object = objects[j]; for (const p in object) { if (object.hasOwnProperty(p) && !table.hasOwnProperty(p)) { table[p] = valueIndex++; } } } } return table; }, toInputTable2D( data: Array<{ input: Array<{ [key: string]: number }> }> ): INumberHash { const table: INumberHash = {}; let tableIndex = 0; for (let dataIndex = 0; dataIndex < data.length; dataIndex++) { const input = data[dataIndex].input; for (let i = 0; i < input.length; i++) { const object = input[i]; for (const p in object) { if (!object.hasOwnProperty(p)) continue; if (!table.hasOwnProperty(p)) { table[p] = tableIndex++; } } } } return table; }, toOutputTable2D( data: Array<{ output: Array<{ [key: string]: number }> }> ): INumberHash { const table: INumberHash = {}; let tableIndex = 0; for (let dataIndex = 0; dataIndex < data.length; dataIndex++) { const output = data[dataIndex].output; for (let i = 0; i < output.length; i++) { const object = output[i]; for (const p in object) { if (!object.hasOwnProperty(p)) continue; if (!table.hasOwnProperty(p)) { table[p] = tableIndex++; } } } } return table; }, /** * performs `{a: 6, b: 7} -> {a: 0, b: 1}` */ toHash(hash: INumberHash): INumberHash { const lookup: INumberHash = {}; let index = 0; const keys = Object.keys(hash); for (let i = 0; i < keys.length; i++) { lookup[keys[i]] = index++; } return lookup; }, /** * performs `{a: 0, b: 1}, {a: 6} -> [6, 0]` */ toArray( lookup: INumberHash, object: INumberHash, arrayLength: number ): Float32Array { const result = new Float32Array(arrayLength); for (const p in lookup) { if (!lookup.hasOwnProperty(p)) continue; result[lookup[p]] = object.hasOwnProperty(p) ? object[p] : 0; } return result; }, toArrayShort(lookup: INumberHash, object: INumberHash): Float32Array { const result = []; for (const p in lookup) { if (!lookup.hasOwnProperty(p)) continue; if (!object.hasOwnProperty(p)) break; result[lookup[p]] = object[p]; } return Float32Array.from(result); }, toArrays( lookup: INumberHash, objects: INumberHash[], arrayLength: number ): Float32Array[] { const result = []; for (let i = 0; i < objects.length; i++) { result.push(this.toArray(lookup, objects[i], arrayLength)); } return result; }, /** * performs `{a: 0, b: 1}, [6, 7] -> {a: 6, b: 7}` * @param {Object} lookup * @param {Array} array * @returns {Object} */ toObject(lookup: INumberHash, array: number[] | Float32Array): INumberHash { const object: INumberHash = {}; for (const p in lookup) { if (!lookup.hasOwnProperty(p)) continue; object[p] = array[lookup[p]]; } return object; }, toObjectPartial( lookup: INumberHash, array: number[] | Float32Array, offset = 0, limit = 0 ): INumberHash { const object: INumberHash = {}; let i = 0; for (const p in lookup) { if (!lookup.hasOwnProperty(p)) continue; if (offset > 0) { if (i++ < offset) continue; } if (limit > 0) { if (i++ >= limit) continue; } object[p] = array[lookup[p] - offset]; } return object; }, dataShape(data: FormattableData[] | FormattableData): string[] { const shape = []; let lastData; if (data.hasOwnProperty('input')) { shape.push('datum'); lastData = (data as ITrainingDatum).input; } else if (Array.isArray(data)) { if ( (data as ITrainingDatum[])[0] && (data as ITrainingDatum[])[0].input ) { shape.push('array', 'datum'); lastData = (data as ITrainingDatum[])[0].input; } else if (Array.isArray(data[0])) { shape.push('array'); lastData = data[0]; } else { lastData = data as | InputOutputValue | InputOutputValue[] | InputOutputValue[][]; } } else { lastData = data as | InputOutputValue | InputOutputValue[] | InputOutputValue[][]; } let p; while (lastData) { p = Object.keys(lastData)[0]; if ( Array.isArray(lastData) || typeof (lastData as Float32Array).buffer === 'object' ) { shape.push('array'); const possibleNumber: | number | INumberArray = (lastData as INumberArray[])[parseInt(p)]; if (typeof possibleNumber === 'number') { shape.push('number'); break; } else { lastData = possibleNumber; } } else if ( typeof lastData === 'object' && typeof (lastData as Float32Array).buffer !== 'object' ) { shape.push('object'); const possibleNumber: number | INumberHash = (lastData as INumberHash)[ p ]; if (typeof possibleNumber === 'number') { shape.push('number'); break; } else { lastData = possibleNumber; } } else { throw new Error('unhandled signature'); } } return shape; }, addKeys(value: number[] | INumberHash, table: INumberHash): INumberHash { if (Array.isArray(value)) return table; let i = Object.keys(table).length; for (const p in value) { if (!value.hasOwnProperty(p)) continue; if (table.hasOwnProperty(p)) continue; table[p] = i++; } return table; }, }; ================================================ FILE: src/neural-network-gpu.end-to-end.test.ts ================================================ import { NeuralNetworkGPU } from './neural-network-gpu'; import { xorTrainingData } from './test-utils'; describe('NeuralNetworkGPU Class: End to End', () => { it('can learn xor', () => { const net = new NeuralNetworkGPU(); const status = net.train(xorTrainingData, { iterations: 5000, errorThresh: 0.01, }); expect(status.error).toBeLessThanOrEqual(0.01); expect(status.iterations).toBeLessThanOrEqual(5000); }); }); ================================================ FILE: src/neural-network-gpu.test.ts ================================================ import { Texture } from 'gpu.js'; import { NeuralNetwork } from './neural-network'; import { NeuralNetworkGPU } from './neural-network-gpu'; import { xorTrainingData } from './test-utils'; describe('NeuralNetworkGPU', () => { describe('run', () => { describe('when input is not same as options.inputSize', () => { it('throws', () => { const net = new NeuralNetworkGPU({ inputSize: 1, hiddenLayers: [1], outputSize: 1, }); net.train([{ input: [1], output: [1] }], { iterations: 1 }); expect(() => { net.run([1, 1]); }).toThrow(); }); }); describe('when input is same as options.inputSize', () => { it('throws', () => { const net = new NeuralNetworkGPU({ inputSize: 1, hiddenLayers: [1], outputSize: 1, }); net.train([{ input: [1], output: [1] }], { iterations: 1 }); expect(() => { net.run([1]); }).not.toThrow(); }); }); }); describe('.toJSON()', () => { it('can serialize & deserialize JSON', () => { const net = new NeuralNetworkGPU(); net.train(xorTrainingData, { iterations: 1 }); const target = xorTrainingData.map((datum) => net.run(datum.input)); const json = net.toJSON(); const net2 = new NeuralNetworkGPU(); net2.fromJSON(json); const output = xorTrainingData.map((datum) => net2.run(datum.input)); expect(output).toEqual(target); }); it('can serialize from NeuralNetworkGPU & deserialize to NeuralNetwork', () => { const net = new NeuralNetworkGPU(); net.train(xorTrainingData, { iterations: 1 }); const target = xorTrainingData.map((datum) => net.run(datum.input)); const json = net.toJSON(); const net2 = new NeuralNetwork(); net2.fromJSON(json); for (let i = 0; i < xorTrainingData.length; i++) { // there is a wee bit of loss going from GPU to CPU expect(net2.run(xorTrainingData[i].input)[0]).toBeCloseTo( target[i][0], 5 ); } }); it('can serialize from NeuralNetwork & deserialize to NeuralNetworkGPU', () => { const net = new NeuralNetwork(); net.train(xorTrainingData, { iterations: 1 }); const target = xorTrainingData.map((datum) => net.run(datum.input)); const json = net.toJSON(); const net2 = new NeuralNetworkGPU(); net2.fromJSON(json); for (let i = 0; i < xorTrainingData.length; i++) { // there is a wee bit of loss going from CPU to GPU expect(net2.run(xorTrainingData[i].input)[0]).toBeCloseTo( target[i][0], 5 ); } }); describe('mocked GPU mode', () => { it('converts Textures to Arrays of numbers', () => { const net = new NeuralNetworkGPU(); const isUsingGPU = net.gpu.mode === 'gpu'; // When running in a machine with GPU, will the type will be `Texture` // The CI is running on machines WITHOUT GPUs, so in the case of mocking the GPU the return type will be a bit different const expectedWeightsType = isUsingGPU ? Texture : Array; const expectedBiasesType = isUsingGPU ? Texture : Float32Array; net.train( [ { input: [1, 2], output: [3] }, { input: [2, 1], output: [0] }, { input: [3, 1], output: [1] }, ], { iterations: 1 } ); expect(net.weights.length).toBe(3); for (let i = 1; i < net.weights.length; i++) { expect(net.weights[i]).toBeInstanceOf(expectedWeightsType); } expect(net.biases.length).toBe(3); for (let i = 1; i < net.biases.length; i++) { expect(net.biases[i]).toBeInstanceOf(expectedBiasesType); } const json = net.toJSON(); expect(json.layers.length).toBe(3); for (let i = 1; i < json.layers.length; i++) { const layer = json.layers[i]; expect(layer.weights).toBeInstanceOf(Array); expect(layer.weights[0]).toBeInstanceOf(Array); expect(typeof layer.weights[0][0]).toBe('number'); expect(layer.biases).toBeInstanceOf(Array); expect(typeof layer.biases[0]).toBe('number'); } }); }); }); describe.skip('.toFunction()', () => { it('creates a function equivalent to that of NeuralNetwork', () => { const net = new NeuralNetwork(); net.train(xorTrainingData, { iterations: 5000, errorThresh: 0.01 }); const run = net.toFunction(); const target = xorTrainingData.map((datum) => run(datum.input)); const json = net.toJSON(); const net2 = new NeuralNetworkGPU(); net2.fromJSON(json); const run2 = net2.toFunction(); const output = xorTrainingData.map((datum) => run2(datum.input)); expect(output).toEqual(target); }); }); describe('.trainPattern()', () => { let mockAdjustWeights: jest.SpyInstance; beforeEach(() => { mockAdjustWeights = jest.spyOn( NeuralNetworkGPU.prototype, 'adjustWeights' ); }); afterEach(() => { mockAdjustWeights.mockRestore(); }); describe('when called with logErrorRate = falsey', () => { let runInputSpy: jest.SpyInstance; let calculateDeltasSpy: jest.SpyInstance; let getMSESpy: jest.SpyInstance; afterEach(() => { if (runInputSpy) runInputSpy.mockRestore(); if (calculateDeltasSpy) calculateDeltasSpy.mockRestore(); if (getMSESpy) getMSESpy.mockRestore(); }); it('calls .runInput(), .calculateDeltas(), and .adjustWeights()', () => { const net = new NeuralNetworkGPU({ inputSize: 1, hiddenLayers: [2], outputSize: 3, }); net.initialize(); runInputSpy = jest.spyOn(net, 'runInput'); calculateDeltasSpy = jest.spyOn(net, 'calculateDeltas'); getMSESpy = jest.spyOn(net, 'getMSE'); net.trainPattern({ input: [123], output: [321] }); expect(runInputSpy).toBeCalled(); expect(runInputSpy.mock.calls[0][0]).toEqual([123]); expect(calculateDeltasSpy).toBeCalled(); expect(calculateDeltasSpy.mock.calls[0][0]).toEqual([321]); expect(mockAdjustWeights).toBeCalled(); expect(getMSESpy).not.toBeCalled(); }); }); describe('when called with logErrorRate = truthy', () => { let runInputSpy: jest.SpyInstance; let calculateDeltasSpy: jest.SpyInstance; let getMSESpy: jest.SpyInstance; afterEach(() => { if (runInputSpy) runInputSpy.mockRestore(); if (calculateDeltasSpy) calculateDeltasSpy.mockRestore(); if (getMSESpy) getMSESpy.mockRestore(); }); it('calls .runInput(), .calculateDeltas(), and .adjustWeights()', () => { const net = new NeuralNetworkGPU({ inputSize: 1, hiddenLayers: [2], outputSize: 3, }); net.initialize(); runInputSpy = jest.spyOn(net, 'runInput'); calculateDeltasSpy = jest.spyOn(net, 'calculateDeltas'); getMSESpy = jest.spyOn(net, 'getMSE'); net.trainPattern({ input: [123], output: [321] }, true); expect(runInputSpy).toBeCalled(); expect(runInputSpy.mock.calls[0][0]).toEqual([123]); expect(calculateDeltasSpy).toBeCalled(); expect(calculateDeltasSpy.mock.calls[0][0]).toEqual([321]); expect(mockAdjustWeights).toBeCalled(); expect(getMSESpy).toBeCalled(); }); }); }); }); ================================================ FILE: src/neural-network-gpu.ts ================================================ import { alias, GPU, GPUFunction, IKernelFunctionThis, IKernelMapRunShortcut, IMappedKernelResult, KernelOutput, Texture, utils, } from 'gpu.js'; import { ITrainingStatus } from './feed-forward'; import { INumberHash, lookup } from './lookup'; import { IJSONLayer, INeuralNetworkData, INeuralNetworkDatum, INeuralNetworkJSON, INeuralNetworkOptions, INeuralNetworkPreppedTrainingData, INeuralNetworkTrainOptions, NeuralNetwork, } from './neural-network'; import { release } from './utilities/kernel'; export interface INeuralNetworkGPUDatumFormatted { input: KernelOutput; output: KernelOutput; } export interface INeuralNetworkGPUPreppedTrainingData extends INeuralNetworkPreppedTrainingData { status: ITrainingStatus; endTime: number; } interface ISizedKernelThis extends IKernelFunctionThis { constants: { size: number; }; } function weightedSumSigmoid( this: ISizedKernelThis, weights: number[][], biases: number[], inputs: number[] ): number { let sum = biases[this.thread.x]; for (let k = 0; k < this.constants.size; k++) { sum += weights[this.thread.x][k] * inputs[k]; } // sigmoid return 1 / (1 + Math.exp(-sum)); } function weightedSumRelu( this: ISizedKernelThis, weights: number[][], biases: number[], inputs: number[] ): number { let sum = biases[this.thread.x]; for (let k = 0; k < this.constants.size; k++) { sum += weights[this.thread.x][k] * inputs[k]; } // relu return sum < 0 ? 0 : sum; } function weightedSumLeakyRelu( this: ISizedKernelThis, weights: number[][], biases: number[], inputs: number[] ): number { let sum = biases[this.thread.x]; for (let k = 0; k < this.constants.size; k++) { sum += weights[this.thread.x][k] * inputs[k]; } // leaky relu return sum < 0 ? 0 : 0.01 * sum; } function weightedSumTanh( this: ISizedKernelThis, weights: number[][], biases: number[], inputs: number[] ): number { let sum = biases[this.thread.x]; for (let k = 0; k < this.constants.size; k++) { sum += weights[this.thread.x][k] * inputs[k]; } // tanh return Math.tanh(sum); } function calcErrorOutput(output: number, target: number): number { return target - output; } function calcDeltasSigmoid(error: number, output: number): number { // sigmoid derivative return error * output * (1 - output); } function calcDeltasRelu(error: number, output: number): number { // relu derivative return output > 0 ? error : 0; } function calcDeltasLeakyRelu(error: number, output: number): number { // leaky relu derivative return output > 0 ? error : 0.01 * error; } function calcDeltasTanh(error: number, output: number): number { // tanh derivative return (1 - output * output) * error; } function calcError( x: number, size: number, nextWeights: number[][], nextDeltas: number[] ): number { let error = 0; for (let k = 0; k < size; k++) { error += nextDeltas[k] * nextWeights[k][x]; } return error; } interface ILearningKernelThis extends IKernelFunctionThis { constants: { momentum: number; learningRate: number; }; } function calcChanges( learningRate: number, momentum: number, previousChange: number, delta: number, previousOutput: number ): number { return learningRate * delta * previousOutput + momentum * previousChange; } function addWeights(change: number, weight: number): number { return change + weight; } function addBiases( this: ILearningKernelThis, biases: number[], deltas: number[] ): number { return ( biases[this.thread.x] + deltas[this.thread.x] * this.constants.learningRate ); } // mean squared error, reimplemented for GPU function mse(this: ISizedKernelThis, errors: number[]): number { let sum = 0; for (let i = 0; i < this.constants.size; i++) { sum += errors[i] ** 2; } return sum / this.constants.size; } export interface INeuralNetworkGPUOptions extends INeuralNetworkOptions { mode?: 'cpu' | 'gpu'; } export type BackPropagateOutput = ( this: IKernelFunctionThis, outputs: KernelOutput, targets: KernelOutput ) => { result: KernelOutput; error: KernelOutput }; export type BackPropagateLayer = ( this: IKernelFunctionThis, weights: KernelOutput, outputs: KernelOutput, deltas: KernelOutput ) => { result: KernelOutput; error: KernelOutput }; export class NeuralNetworkGPU< InputType extends INeuralNetworkData, OutputType extends INeuralNetworkData > extends NeuralNetwork { gpu: GPU; texturizeInputData: (value: KernelOutput) => KernelOutput = () => { throw new Error('not yet setup'); }; forwardPropagate: Array< ( weights: KernelOutput, biases: KernelOutput, inputs: KernelOutput ) => KernelOutput > = []; backwardPropagate: Array = []; changesPropagate: Array< (( this: IKernelFunctionThis<{ size: number; learningRate: number; momentum: number; }>, previousOutputs: number[], deltas: number[], weights: number[][], previousChanges: number[][] ) => IMappedKernelResult) & IKernelMapRunShortcut<{ weights: number[][]; changes: number[][] }> > = []; biasesPropagate: Array< (biases: KernelOutput, deltas: KernelOutput) => KernelOutput > = []; getMSE: (error: KernelOutput) => KernelOutput = () => { throw new Error('not yet setup'); }; _addMSE: (sum: KernelOutput, error: KernelOutput) => KernelOutput = () => { throw new Error('not yet setup'); }; _divideMSESum: (length: number, sum: KernelOutput) => KernelOutput = () => { throw new Error('not yet setup'); }; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error outputs: KernelOutput[] = []; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error deltas: KernelOutput[] = []; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error errors: KernelOutput[] = []; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error weights: KernelOutput[] = []; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error changes: KernelOutput[] = []; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error biases: KernelOutput[] = []; constructor(options: Partial = {}) { super(options); this.errorCheckInterval = 100; this.gpu = new GPU({ mode: options.mode }); } initialize(): void { super.initialize(); this.buildRunInput(); this.buildCalculateDeltas(); this.buildGetChanges(); this.buildChangeBiases(); this.buildGetMSE(); } setActivation(): void {} // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error trainPattern( value: INeuralNetworkGPUDatumFormatted, logErrorRate?: boolean ): KernelOutput | null { // forward propagate this.runInput(value.input); // back propagate this.calculateDeltas(value.output); this.adjustWeights(); if (logErrorRate) { return this.getMSE(this.errors[this.outputLayer]); } return null; } calculateTrainingError(data: INeuralNetworkGPUDatumFormatted[]): number { let sum = new Float32Array([0]) as KernelOutput; for (let i = 0; i < data.length; ++i) { const prevSum = sum; const error = this.trainPattern(data[i], true) as KernelOutput; sum = this._addMSE(sum, error); release(error); release(prevSum); } const result = this._divideMSESum(data.length, sum); release(sum); return (result instanceof Texture ? (result.toArray() as number[]) : (result as number[]))[0]; } adjustWeights(): void { this.getChanges(); this.changeBiases(); } buildRunInput(): void { let weightedSum = null; switch (this.trainOpts.activation) { case 'sigmoid': weightedSum = weightedSumSigmoid; break; case 'relu': weightedSum = weightedSumRelu; break; case 'leaky-relu': weightedSum = weightedSumLeakyRelu; break; case 'tanh': weightedSum = weightedSumTanh; break; default: throw new Error( `Unknown activation ${this.trainOpts.activation}. Available activations are: 'sigmoid', 'relu', 'leaky-relu', 'tanh'` ); } for (let layer = 1; layer <= this.outputLayer; layer++) { this.forwardPropagate[layer] = this.gpu.createKernel(weightedSum, { output: [this.sizes[layer]], pipeline: true, constants: { size: this.sizes[layer - 1], }, immutable: true, }); } this.texturizeInputData = this.gpu.createKernel( function (value: number[]): number { return value[this.thread.x]; }, { output: [this.sizes[1]], pipeline: true, immutable: true, } ); } // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error runInput = (input: KernelOutput): KernelOutput => { let output; this.outputs[0] = input; for (let layer = 1; layer <= this.outputLayer; layer++) { release(this.outputs[layer]); this.outputs[layer] = this.forwardPropagate[layer]( this.weights[layer], this.biases[layer], input ); output = input = this.outputs[layer]; } return output; }; buildCalculateDeltas(): void { let calcDeltas: GPUFunction<[number, number]>; switch (this.trainOpts.activation) { case 'sigmoid': calcDeltas = calcDeltasSigmoid; break; case 'relu': calcDeltas = calcDeltasRelu; break; case 'leaky-relu': calcDeltas = calcDeltasLeakyRelu; break; case 'tanh': calcDeltas = calcDeltasTanh; break; default: throw new Error( `Unknown activation ${this.trainOpts.activation}. Available activations are: 'sigmoid', 'relu', 'leaky-relu', 'tanh'` ); } calcDeltas = alias( utils.getMinifySafeName(() => calcDeltas), calcDeltas ); this.gpu.addFunction(calcDeltas); for (let layer = this.outputLayer; layer > 0; layer--) { if (layer === this.outputLayer) { // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error this.backwardPropagate[this.outputLayer] = this.gpu.createKernelMap( { error: calcErrorOutput, }, function ( this: IKernelFunctionThis, outputs: number[], targets: number[] ): number { const output = outputs[this.thread.x]; const target = targets[this.thread.x]; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error return calcDeltas(calcErrorOutput(output, target), output); }, { output: [this.sizes[this.outputLayer]], pipeline: true, immutable: true, } ); } else { // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error this.backwardPropagate[layer] = this.gpu.createKernelMap( { error: calcError, }, function ( this: ISizedKernelThis, nextWeights: number[][], outputs: number[], nextDeltas: number[] ): number { const output = outputs[this.thread.x]; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error return calcDeltas( calcError( this.thread.x, this.constants.size, nextWeights, nextDeltas ), output ); }, { output: [this.sizes[layer]], pipeline: true, constants: { size: this.sizes[layer + 1], }, immutable: true, } ); } } } calculateDeltas = (target: KernelOutput): void => { for (let layer = this.outputLayer; layer > 0; layer--) { release(this.deltas[layer]); release(this.errors[layer]); let output; if (layer === this.outputLayer) { // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error output = this.backwardPropagate[layer](this.outputs[layer], target); } else { // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error output = this.backwardPropagate[layer]( this.weights[layer + 1], this.outputs[layer], this.deltas[layer + 1] ); } this.deltas[layer] = output.result; this.errors[layer] = output.error; } }; buildGetChanges(): void { for (let layer = 1; layer <= this.outputLayer; layer++) { // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error this.changesPropagate[layer] = this.gpu.createKernelMap( { weights: addWeights, changes: calcChanges, }, function ( this: IKernelFunctionThis<{ size: number; learningRate: number; momentum: number; }>, previousOutputs: number[], deltas: number[], weights: number[][], previousChanges: number[][] ) { const change = calcChanges( this.constants.learningRate, this.constants.momentum, previousChanges[this.thread.y][this.thread.x], deltas[this.thread.y], previousOutputs[this.thread.x] ); return addWeights(change, weights[this.thread.y][this.thread.x]); }, { output: [this.sizes[layer - 1], this.sizes[layer]], pipeline: true, constants: { size: this.sizes[layer - 1], learningRate: this.trainOpts.learningRate, momentum: this.trainOpts.momentum, }, immutable: true, } ); } } getChanges(): void { for (let layer = 1; layer <= this.outputLayer; layer++) { const weights = this.weights[layer]; const changes = this.changes[layer]; const output = this.changesPropagate[layer]( this.outputs[layer - 1], this.deltas[layer], weights, changes ); release(weights); release(changes); this.weights[layer] = output.weights; this.changes[layer] = output.changes; release(output.result); } } buildChangeBiases(): void { for (let layer = 1; layer <= this.outputLayer; layer++) { this.biasesPropagate[layer] = this.gpu.createKernel(addBiases, { output: [this.sizes[layer]], pipeline: true, constants: { learningRate: this.trainOpts.learningRate, }, immutable: true, }); } } changeBiases(): void { for (let layer = 1; layer <= this.outputLayer; layer++) { const biases = this.biases[layer]; this.biases[layer] = this.biasesPropagate[layer]( biases, this.deltas[layer] ); release(biases); } } buildGetMSE(): void { this.getMSE = this.gpu.createKernel(mse, { output: [1], constants: { size: this.sizes[this.outputLayer], }, pipeline: true, immutable: true, }); this._addMSE = this.gpu.createKernel( function (value1: number[], value2: number[]): number { return value1[0] + value2[0]; }, { output: [1], pipeline: true, immutable: true, } ); this._divideMSESum = this.gpu.createKernel( function (length: number, mseSum: number[]): number { const value = mseSum[0]; if (value > 0) { return value / length; } return 0; }, { output: [1], } ); } run(input: InputType): OutputType { if (!this.isRunnable) { throw new Error('network not runnable'); } let formattedInput: Float32Array; if (this.inputLookup) { formattedInput = lookup.toArray( this.inputLookup, (input as unknown) as INumberHash, this.inputLookupLength ); } else { formattedInput = (input as unknown) as Float32Array; } this.validateInput(formattedInput); const outputTextures = this.runInput(formattedInput); const output = outputTextures instanceof Texture ? outputTextures.toArray() : outputTextures; if (this.outputLookup) { return (lookup.toObject( this.outputLookup, output as Float32Array ) as unknown) as OutputType; } return (output as unknown) as OutputType; } // @ts-expect-error the underlying network works as normal, but we are working on the GPU prepTraining( data: Array>, options: Partial = {} ): INeuralNetworkGPUPreppedTrainingData { this.updateTrainingOptions(options); const preparedData = this.formatData(data); const endTime = Date.now() + this.trainOpts.timeout; const status = { error: 1, iterations: 0, }; this.verifyIsInitialized(preparedData); const texturizeOutputData = this.gpu.createKernel( function (value: number[]): number { return value[this.thread.x]; }, { output: [preparedData[0].output.length], pipeline: true, immutable: true, } ); return { preparedData: preparedData.map((set) => ({ input: this.texturizeInputData(set.input), output: texturizeOutputData(set.output), })), status, endTime, }; } // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error toFunction(): (input: InputType) => OutputType { throw new Error( `${this.constructor.name}-toFunction is not yet implemented` ); } toJSON(): INeuralNetworkJSON { if (this.sizes === null) { this.initialize(); } // use Array.from, keeping json small const jsonLayerWeights = this.weights.map((layerWeights) => { return (layerWeights instanceof Texture ? (layerWeights.toArray() as Float32Array[]) : (layerWeights as Float32Array[]) ).map((layerWeights) => Array.from(layerWeights)); }); const jsonLayerBiases = this.biases.map((layerBiases) => Array.from( layerBiases instanceof Texture ? (layerBiases.toArray() as Float32Array) : (layerBiases as Float32Array) ) ); const jsonLayers: IJSONLayer[] = []; for (let i = 0; i <= this.outputLayer; i++) { jsonLayers.push({ weights: jsonLayerWeights[i] ?? [], biases: jsonLayerBiases[i] ?? [], }); } return { type: 'NeuralNetworkGPU', sizes: [...this.sizes], layers: jsonLayers, inputLookup: this.inputLookup ? { ...this.inputLookup } : null, inputLookupLength: this.inputLookupLength, outputLookup: this.outputLookup ? { ...this.outputLookup } : null, outputLookupLength: this.outputLookupLength, options: { ...this.options }, trainOpts: this.getTrainOptsJSON(), }; } } ================================================ FILE: src/neural-network-types.ts ================================================ /** TODO: The following should be moved to neural-network.ts once that is converted to typescript. Added here until neural-network.js is converted */ export interface INeuralNetworkOptions { /** * @default 0.5 */ binaryThresh?: number; /** * array of int for the sizes of the hidden layers in the network * * @default [3] */ hiddenLayers?: number[]; } export interface INeuralNetworkTrainingOptions { /** * the maximum times to iterate the training data --> number greater than 0 * @default 20000 */ iterations?: number; /** * the acceptable error percentage from training data --> number between 0 and 1 * @default 0.005 */ errorThresh?: number; /** * true to use console.log, when a function is supplied it is used --> Either true or a function * @default false */ log?: boolean | INeuralNetworkTrainingCallback; /** * iterations between logging out --> number greater than 0 * @default 10 */ logPeriod?: number; /** * scales with delta to effect training rate --> number between 0 and 1 * @default 0.3 */ learningRate?: number; /** * scales with next layer's change value --> number between 0 and 1 * @default 0.1 */ momentum?: number; /** * a periodic call back that can be triggered while training --> null or function * @default null */ callback?: INeuralNetworkTrainingCallback | number; /** * the number of iterations through the training data between callback calls --> number greater than 0 * @default 10 */ callbackPeriod?: number; /** * the max number of milliseconds to train for --> number greater than 0 * @default Infinity */ timeout?: number; praxis?: null | 'adam'; } export type INeuralNetworkTrainingCallback = ( state: INeuralNetworkState ) => void; export interface INeuralNetworkState { iterations: number; error: number; } export interface INeuralNetworkTestResult { misclasses: unknown[]; error: number; total: number; } export interface INeuralNetworkBinaryTestResult extends INeuralNetworkTestResult { trueNeg: number; truePos: number; falseNeg: number; falsePos: number; precision: number; recall: number; accuracy: number; } ================================================ FILE: src/neural-network.bitwise.test.ts ================================================ import { NeuralNetwork } from './neural-network'; describe('NeuralNetwork bitwise', () => { describe('bitwise functions sync training', () => { function testBitwise( data: Array<{ input: number[]; output: number[]; }> ) { const net = new NeuralNetwork(); net.train(data, { errorThresh: 0.003 }); data.forEach((d) => { const actual = net.run(d.input); const expected = d.output; expect(actual[0]).toBeCloseTo(expected[0], 0.05); }); } it('NOT function', () => { const not = [ { input: [0], output: [1] }, { input: [1], output: [0] }, ]; testBitwise(not); }); it('XOR function', () => { const xor = [ { input: [0.001, 0.001], output: [0.001] }, { input: [0.001, 1], output: [1] }, { input: [1, 0.001], output: [1] }, { input: [1, 1], output: [0.001] }, ]; testBitwise(xor); }); it('OR function', () => { const or = [ { input: [0, 0], output: [0] }, { input: [0, 1], output: [1] }, { input: [1, 0], output: [1] }, { input: [1, 1], output: [1] }, ]; testBitwise(or); }); it('AND function', () => { const and = [ { input: [0, 0], output: [0] }, { input: [0, 1], output: [0] }, { input: [1, 0], output: [0] }, { input: [1, 1], output: [1] }, ]; testBitwise(and); }); }); describe('bitwise using adam praxis functions sync training', () => { function testBitwiseAdam( data: Array<{ input: number[]; output: number[]; }> ) { const net = new NeuralNetwork(); net.train(data, { errorThresh: 0.003, learningRate: 0.05, praxis: 'adam', }); data.forEach((d) => { const actual = net.run(d.input); const expected = d.output; expect(actual[0]).toBeCloseTo(expected[0], 0.05); }); } it('NOT function', () => { const not = [ { input: [0], output: [1] }, { input: [1], output: [0] }, ]; testBitwiseAdam(not); }); it('XOR function', () => { const xor = [ { input: [0.001, 0.001], output: [0.001] }, { input: [0.001, 1], output: [1] }, { input: [1, 0.001], output: [1] }, { input: [1, 1], output: [0.001] }, ]; testBitwiseAdam(xor); }); it('OR function', () => { const or = [ { input: [0, 0], output: [0] }, { input: [0, 1], output: [1] }, { input: [1, 0], output: [1] }, { input: [1, 1], output: [1] }, ]; testBitwiseAdam(or); }); it('AND function', () => { const and = [ { input: [0, 0], output: [0] }, { input: [0, 1], output: [0] }, { input: [1, 0], output: [0] }, { input: [1, 1], output: [1] }, ]; testBitwiseAdam(and); }); }); describe('bitwise using async training', () => { async function testBitwiseAsync( data: Array<{ input: number[]; output: number[]; }> ) { const net = new NeuralNetwork(); await net.trainAsync(data, { errorThresh: 0.003, timeout: 4800 }); data.forEach((d) => { const actual = net.run(d.input); const expected = d.output; expect(actual[0]).toBeCloseTo(expected[0], 0.05); }); } it('NOT function', async () => { const not = [ { input: [0], output: [1] }, { input: [1], output: [0] }, ]; await testBitwiseAsync(not); }); it('XOR function', async () => { const xor = [ { input: [0.001, 0.001], output: [0.001] }, { input: [0.001, 1], output: [1] }, { input: [1, 0.001], output: [1] }, { input: [1, 1], output: [0.001] }, ]; await testBitwiseAsync(xor); }); it('OR function', async () => { const or = [ { input: [0, 0], output: [0] }, { input: [0, 1], output: [1] }, { input: [1, 0], output: [1] }, { input: [1, 1], output: [1] }, ]; await testBitwiseAsync(or); }); it('AND function', async () => { const and = [ { input: [0, 0], output: [0] }, { input: [0, 1], output: [0] }, { input: [1, 0], output: [0] }, { input: [1, 1], output: [1] }, ]; await testBitwiseAsync(and); }); }); }); ================================================ FILE: src/neural-network.json.test.ts ================================================ import { NeuralNetwork, defaults, trainDefaults } from './neural-network'; describe('JSON', () => { describe('.toJSON() serialization', () => { describe('json.sizes', () => { it('copies json.sizes correctly [1,2,3]', () => { const net = new NeuralNetwork(); net.sizes = [1, 2, 3]; const json = net.toJSON(); expect(json.sizes).toEqual([1, 2, 3]); }); it('copies json.sizes correctly [3,2,1]', () => { const net = new NeuralNetwork(); net.sizes = [3, 2, 1]; const json = net.toJSON(); expect(json.sizes).toEqual([3, 2, 1]); }); }); describe('hidden layers', () => { it('copies biases correctly', () => { const net = new NeuralNetwork({ hiddenLayers: [3] }); net.verifyIsInitialized([ { input: Float32Array.from([1, 2]), output: Float32Array.from([1, 2, 3]), }, ]); const json = net.toJSON(); const layer1Biases = net.biases[1]; const jsonLayer1Bias = json.layers[1].biases; expect(jsonLayer1Bias.length).toBe(layer1Biases.length); expect(jsonLayer1Bias[0]).toBe(layer1Biases[0]); expect(jsonLayer1Bias[1]).toBe(layer1Biases[1]); expect(jsonLayer1Bias[2]).toBe(layer1Biases[2]); }); it('copies weights correctly', () => { const net = new NeuralNetwork({ hiddenLayers: [3] }); net.verifyIsInitialized([ { input: Float32Array.from([1, 2]), output: Float32Array.from([1, 2, 3]), }, ]); const json = net.toJSON(); const layer1Weights = net.weights[1]; const jsonLayer1Weights = json.layers[1].weights; expect(jsonLayer1Weights.length).toBe(layer1Weights.length); expect(jsonLayer1Weights[0]).toEqual(Array.from(layer1Weights[0])); expect(jsonLayer1Weights[1]).toEqual(Array.from(layer1Weights[1])); expect(jsonLayer1Weights[2]).toEqual(Array.from(layer1Weights[2])); }); }); describe('output layer', () => { it('copies biases correctly', () => { const net = new NeuralNetwork({ hiddenLayers: [3] }); net.verifyIsInitialized([ { input: Float32Array.from([1, 2]), output: Float32Array.from([1, 2, 3]), }, ]); const json = net.toJSON(); const layer2Biases = net.biases[2]; const jsonLayer2Biases = json.layers[2].biases; expect(jsonLayer2Biases.length).toBe(layer2Biases.length); expect(jsonLayer2Biases[0]).toBe(layer2Biases[0]); expect(jsonLayer2Biases[1]).toBe(layer2Biases[1]); expect(jsonLayer2Biases[2]).toBe(layer2Biases[2]); }); it('copies weights correctly', () => { const net = new NeuralNetwork({ hiddenLayers: [3] }); net.verifyIsInitialized([ { input: Float32Array.from([1, 2]), output: Float32Array.from([1, 2, 3]), }, ]); const json = net.toJSON(); const layer2Weights = net.weights[2]; const jsonLayer2Weights = json.layers[2].weights; expect(jsonLayer2Weights.length).toBe(layer2Weights.length); expect(jsonLayer2Weights[0]).toEqual(Array.from(layer2Weights[0])); expect(jsonLayer2Weights[1]).toEqual(Array.from(layer2Weights[1])); expect(jsonLayer2Weights[2]).toEqual(Array.from(layer2Weights[2])); }); }); describe('.options', () => { describe('.inputSize', () => { it('exports default correctly', () => { const net = new NeuralNetwork(); net.train([{ input: [1], output: [1] }], { iterations: 1 }); expect(net.toJSON().options.inputSize).toEqual(defaults().inputSize); }); it('exports non-default correctly', () => { const net = new NeuralNetwork({ inputSize: 7, outputSize: 1 }); expect(net.toJSON().options.inputSize).toEqual(7); }); }); describe('.outputSize', () => { it('exports default correctly', () => { const net = new NeuralNetwork(); net.train([ { input: [1], output: [1], }, ]); expect(net.toJSON().options.outputSize).toEqual( defaults().outputSize ); }); it('exports non-default correctly', () => { const net = new NeuralNetwork({ outputSize: 7 }); net.train( [ { input: [1], output: [1], // note: different size that that of option on purpose }, ], { iterations: 1 } ); expect(net.toJSON().options.outputSize).toEqual(7); }); }); describe('.binaryThresh', () => { it('exports default correctly', () => { const net = new NeuralNetwork(); net.train( [ { input: [1], output: [1], }, ], { iterations: 1 } ); expect(net.toJSON().options.binaryThresh).toEqual( defaults().binaryThresh ); }); it('exports non-default correctly', () => { const net = new NeuralNetwork({ binaryThresh: 7 }); net.train( [ { input: [1], output: [1], }, ], { iterations: 1 } ); expect(net.toJSON().options.binaryThresh).toEqual(7); }); }); describe('.hiddenLayers', () => { it('exports default correctly', () => { const net = new NeuralNetwork(); net.train([{ input: [1], output: [1] }], { iterations: 1 }); expect(net.toJSON().options.hiddenLayers).toEqual( defaults().hiddenLayers ); }); it('exports non-default correctly', () => { const net = new NeuralNetwork({ inputSize: 1, hiddenLayers: [7, 8, 9], outputSize: 1, }); expect(net.toJSON().options.hiddenLayers).toEqual([7, 8, 9]); }); }); }); describe('.trainOpts', () => { describe('.activation', () => { it('exports default correctly', () => { const net = new NeuralNetwork(); net.sizes = [1]; expect(net.trainOpts.activation).toBe(trainDefaults().activation); const json = net.toJSON(); expect(json.trainOpts.activation).toBe(trainDefaults().activation); }); it('exports non-default correctly', () => { const net = new NeuralNetwork({ activation: 'leaky-relu' }); net.sizes = [1]; expect(net.trainOpts.activation).toBe('leaky-relu'); const json = net.toJSON(); expect(json.trainOpts.activation).toBe('leaky-relu'); }); }); describe('.iterations', () => { it('copies default value when no value used', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); net.trainingTick = () => false; net.train([{ input: [], output: [] }]); const json = net.toJSON(); expect(json.trainOpts.iterations).toBe(trainDefaults().iterations); }); it('copies custom value when defined', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); net.trainingTick = () => false; net.train([{ input: [], output: [] }], { iterations: 3 }); const json = net.toJSON(); expect(json.trainOpts.iterations).toBe(3); }); }); describe('.errorThresh', () => { it('copies default value when no value used', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); net.trainingTick = () => false; net.train([{ input: [], output: [] }]); const json = net.toJSON(); expect(json.trainOpts.errorThresh).toBe(trainDefaults().errorThresh); }); it('copies custom value when defined', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); net.trainingTick = () => false; net.train([{ input: [], output: [] }], { errorThresh: 0.05 }); const json = net.toJSON(); expect(json.trainOpts.errorThresh).toBe(0.05); }); }); describe('.log', () => { it('copies default value when no value used', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); net.trainingTick = () => false; net.train([{ input: [], output: [] }]); const json = net.toJSON(); expect(json.trainOpts.log).toEqual(trainDefaults().log); }); it('copies custom value when defined as boolean', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); const log = true; net.trainingTick = () => false; net.train([{ input: [], output: [] }], { log }); const json = net.toJSON(); expect(json.trainOpts.log).toBe(log); }); it('uses `true` when used with a custom function', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); const log = () => {}; net.trainingTick = () => false; net.train([{ input: [], output: [] }], { log }); const json = net.toJSON(); expect(json.trainOpts.log).toBe(true); }); }); describe('.logPeriod', () => { it('copies default value when no value used', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); net.trainingTick = () => false; net.train([{ input: [], output: [] }]); const json = net.toJSON(); expect(json.trainOpts.logPeriod).toBe(trainDefaults().logPeriod); }); it('copies custom value when defined', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); net.trainingTick = () => false; net.train([{ input: [], output: [] }], { logPeriod: 4 }); const json = net.toJSON(); expect(json.trainOpts.logPeriod).toBe(4); }); }); describe('.learningRate', () => { it('copies default value when no value used', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); net.trainingTick = () => false; net.train([{ input: [], output: [] }]); const json = net.toJSON(); expect(json.trainOpts.learningRate).toBe( trainDefaults().learningRate ); }); it('copies custom value when defined', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); net.trainingTick = () => false; net.train([{ input: [], output: [] }], { learningRate: 0.72 }); const json = net.toJSON(); expect(json.trainOpts.learningRate).toBe(0.72); }); }); describe('.momentum', () => { it('copies default value when no value used', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); net.trainingTick = () => false; net.train([{ input: [], output: [] }]); const json = net.toJSON(); expect(json.trainOpts.momentum).toBe(trainDefaults().momentum); }); it('copies custom value when defined', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); net.trainingTick = () => false; net.train([{ input: [], output: [] }], { momentum: 0.313 }); const json = net.toJSON(); expect(json.trainOpts.momentum).toBe(0.313); }); }); describe('.callbackPeriod', () => { it('copies default value when no value used', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); net.trainingTick = () => false; net.train([{ input: [], output: [] }]); const json = net.toJSON(); expect(json.trainOpts.callbackPeriod).toBe( trainDefaults().callbackPeriod ); }); it('copies custom value when defined', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); net.trainingTick = () => false; net.train([{ input: [], output: [] }], { callbackPeriod: 50 }); const json = net.toJSON(); expect(json.trainOpts.callbackPeriod).toBe(50); }); }); describe('.timeout', () => { it('uses undefined in place of Infinity when no value used for default value', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); net.trainingTick = () => false; net.train([{ input: [], output: [] }]); const json = net.toJSON(); expect(trainDefaults().timeout).toBe(Infinity); expect(json.trainOpts.timeout).toBe('Infinity'); }); it('copies custom value when defined', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); net.trainingTick = () => false; net.train([{ input: [], output: [] }], { timeout: 50 }); const json = net.toJSON(); expect(json.trainOpts.timeout).toBe(50); }); }); }); }); describe('.fromJSON() deserialization', () => { describe('json.sizes', () => { it('copies json.sizes correctly [1,2,3]', () => { const net = new NeuralNetwork(); net.sizes = [1, 2, 3]; net.initialize(); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); expect(newNet.sizes).toEqual([1, 2, 3]); }); it('copies json.sizes correctly [3,2,1]', () => { const net = new NeuralNetwork(); net.sizes = [3, 2, 1]; net.initialize(); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); expect(newNet.sizes).toEqual([3, 2, 1]); }); }); describe('hidden layers', () => { it('copies biases correctly', () => { const net = new NeuralNetwork({ hiddenLayers: [3] }); net.verifyIsInitialized([ { input: Float32Array.from([1, 2]), output: Float32Array.from([1, 2, 3]), }, ]); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); const biases = net.biases; const newNetBiases = newNet.biases; expect(newNetBiases[1].length).toBe(biases[1].length); expect(newNetBiases[1][0]).toBe(biases[1][0]); expect(newNetBiases[1][1]).toBe(biases[1][1]); expect(newNetBiases[1][2]).toBe(biases[1][2]); }); it('copies weights correctly', () => { const net = new NeuralNetwork({ hiddenLayers: [3] }); net.verifyIsInitialized([ { input: Float32Array.from([1, 2]), output: Float32Array.from([1, 2, 3]), }, ]); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON( JSON.parse(JSON.stringify(json)) ); const layer2Weights = net.weights[2]; const newNetLayer2Weights = newNet.weights[2]; expect(layer2Weights).toEqual(newNetLayer2Weights); }); }); describe('output layer', () => { it('copies biases correctly', () => { const net = new NeuralNetwork({ hiddenLayers: [3] }); net.verifyIsInitialized([ { input: Float32Array.from([1, 2]), output: Float32Array.from([1, 2, 3]), }, ]); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); const biases = net.biases; const newNetBiases = newNet.biases; expect(newNetBiases[2].length).toBe(biases[2].length); expect(newNetBiases[2][0]).toBe(biases[2][0]); expect(newNetBiases[2][1]).toBe(biases[2][1]); expect(newNetBiases[2][2]).toBe(biases[2][2]); }); it('copies weights correctly', () => { const net = new NeuralNetwork({ hiddenLayers: [3] }); net.verifyIsInitialized([ { input: Float32Array.from([1, 2]), output: Float32Array.from([1, 2, 3]), }, ]); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON( JSON.parse(JSON.stringify(json)) ); const layer2Weights = net.weights[2]; const newNetLayer2Weights = newNet.weights[2]; expect(newNetLayer2Weights).toEqual(layer2Weights); }); }); describe('.options', () => { describe('.inputSize', () => { it('exports default correctly', () => { const net = new NeuralNetwork(); net.sizes = [1]; expect(net.options.inputSize).toBe(defaults().inputSize); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); expect(newNet.options.inputSize).toBe(defaults().inputSize); }); it('exports non-default correctly', () => { const net = new NeuralNetwork({ inputSize: 4 }); net.sizes = [1]; expect(net.options.inputSize).toBe(4); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); expect(newNet.options.inputSize).toBe(4); }); }); describe('.outputSize', () => { it('exports default correctly', () => { const net = new NeuralNetwork(); net.sizes = [1]; expect(net.options.outputSize).toBe(defaults().outputSize); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); expect(newNet.options.outputSize).toBe(defaults().outputSize); }); it('exports non-default correctly', () => { const net = new NeuralNetwork({ outputSize: 4 }); net.sizes = [1]; expect(net.options.outputSize).toBe(4); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); expect(newNet.options.outputSize).toBe(4); }); }); describe('.binaryThresh', () => { it('exports default correctly', () => { const net = new NeuralNetwork(); net.sizes = [1]; expect(net.options.binaryThresh).toBe(defaults().binaryThresh); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); expect(newNet.options.binaryThresh).toBe(defaults().binaryThresh); }); it('exports non-default correctly', () => { const net = new NeuralNetwork({ binaryThresh: 4 }); net.sizes = [1]; expect(net.options.binaryThresh).toBe(4); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); expect(newNet.options.binaryThresh).toBe(4); }); }); describe('.hiddenLayers', () => { it('exports default correctly', () => { const net = new NeuralNetwork(); net.sizes = [1]; expect(net.options.hiddenLayers).toEqual(defaults().hiddenLayers); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); expect(newNet.options.hiddenLayers).toEqual(defaults().hiddenLayers); }); it('exports non-default correctly', () => { const net = new NeuralNetwork({ hiddenLayers: [4, 5, 6] }); net.sizes = [1]; expect(net.options.hiddenLayers).toEqual([4, 5, 6]); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); expect(newNet.options.hiddenLayers).toEqual([4, 5, 6]); }); }); }); describe('.trainOpts', () => { describe('.activation', () => { it('exports default correctly', () => { const net = new NeuralNetwork(); net.sizes = [1]; expect(net.trainOpts.activation).toBe(trainDefaults().activation); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); expect(newNet.trainOpts.activation).toBe(trainDefaults().activation); }); it('exports non-default correctly', () => { const net = new NeuralNetwork({ activation: 'leaky-relu' }); net.sizes = [1]; expect(net.trainOpts.activation).toBe('leaky-relu'); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); expect(newNet.trainOpts.activation).toBe('leaky-relu'); }); }); describe('.iterations', () => { it('copies default value when no value used', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); net.trainingTick = () => false; net.train([{ input: [], output: [] }]); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); expect(newNet.trainOpts.iterations).toBe(trainDefaults().iterations); }); it('copies custom value when defined', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); net.trainingTick = () => false; net.train([{ input: [], output: [] }], { iterations: 3 }); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); expect(newNet.trainOpts.iterations).toBe(3); }); }); describe('.errorThresh', () => { it('copies default value when no value used', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); net.trainingTick = () => false; net.train([{ input: [], output: [] }]); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); expect(newNet.trainOpts.errorThresh).toBe( trainDefaults().errorThresh ); }); it('copies custom value when defined', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); net.trainingTick = () => false; net.train([{ input: [], output: [] }], { errorThresh: 0.05 }); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); expect(newNet.trainOpts.errorThresh).toBe(0.05); }); }); describe('.log', () => { it('copies default value when no value used', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); net.trainingTick = () => false; net.train([{ input: [], output: [] }]); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); expect(newNet.trainOpts.log).toBe(trainDefaults().log); }); it('uses net.logTrainingStatus for `true`', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); const log = true; net.trainingTick = () => false; net.train([{ input: [], output: [] }], { log }); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); expect(newNet.trainOpts.log).toBe(net.logTrainingStatus); }); it('reverts to net.logTrainingStatus when used with custom function', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); const log = () => {}; net.trainingTick = () => false; net.train([{ input: [], output: [] }], { log }); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); expect(newNet.trainOpts.log).toBe(net.logTrainingStatus); }); }); describe('.logPeriod', () => { it('copies default value when no value used', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); net.trainingTick = () => false; net.train([{ input: [], output: [] }]); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); expect(newNet.trainOpts.logPeriod).toBe(trainDefaults().logPeriod); }); it('copies custom value when defined', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); net.trainingTick = () => false; net.train([{ input: [], output: [] }], { logPeriod: 4 }); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); expect(newNet.trainOpts.logPeriod).toBe(4); }); }); describe('.learningRate', () => { it('copies default value when no value used', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); net.trainingTick = () => false; net.train([{ input: [], output: [] }]); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); expect(newNet.trainOpts.learningRate).toBe( trainDefaults().learningRate ); }); it('copies custom value when defined', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); net.trainingTick = () => false; net.train([{ input: [], output: [] }], { learningRate: 0.72 }); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); expect(newNet.trainOpts.learningRate).toBe(0.72); }); }); describe('.momentum', () => { it('copies default value when no value used', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); net.trainingTick = () => false; net.train([{ input: [], output: [] }]); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); expect(newNet.trainOpts.momentum).toBe(trainDefaults().momentum); }); it('copies custom value when defined', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); net.trainingTick = () => false; net.train([{ input: [], output: [] }], { momentum: 0.313 }); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); expect(newNet.trainOpts.momentum).toBe(0.313); }); }); describe('.callback', () => { it('does not copy', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); net.trainingTick = () => false; net.train([{ input: [], output: [] }], { iterations: 1 }); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); expect(newNet.trainOpts.callback).toBe(undefined); }); it('does not copy when used with custom value', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); const callback = () => {}; net.trainingTick = () => false; net.train([{ input: [], output: [] }], { callback }); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); expect(newNet.trainOpts.callback).toBe(undefined); }); }); describe('.callbackPeriod', () => { it('copies default value when no value used', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); net.trainingTick = () => false; net.train([{ input: [], output: [] }]); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); expect(newNet.trainOpts.callbackPeriod).toBe( trainDefaults().callbackPeriod ); }); it('copies custom value when defined', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); net.trainingTick = () => false; net.train([{ input: [], output: [] }], { callbackPeriod: 50 }); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); expect(newNet.trainOpts.callbackPeriod).toBe(50); }); }); describe('.timeout', () => { it('uses undefined in place of Infinity when no value used for default value', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); net.trainingTick = () => false; net.train([{ input: [], output: [] }]); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); expect(trainDefaults().timeout).toBe(Infinity); expect(newNet.trainOpts.timeout).toBe(Infinity); }); it('copies custom value when defined', () => { const net = new NeuralNetwork({ hiddenLayers: [2] }); net.trainingTick = () => false; net.train([{ input: [], output: [] }], { timeout: 50 }); const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); expect(newNet.trainOpts.timeout).toBe(50); }); }); }); it('can run originalNet, and serializedNet, with same output', () => { const net = new NeuralNetwork({ hiddenLayers: [3] }); net.train([{ input: [1, 1, 1], output: [1, 1, 1] }], { iterations: 3, }); const input = [1, 1, 1]; const json = net.toJSON(); const newNet = new NeuralNetwork().fromJSON(json); const output1 = net.run(input); const output2 = newNet.run(input); expect(output2).toEqual(output1); }); it('if json.trainOpts is not set, ._updateTrainingOptions() is not called abd activation defaults to sigmoid', () => { const net = new NeuralNetwork(); // @ts-expect-error Property '_updateTrainingOptions' does not exist on type 'NeuralNetwork' // This is a private function net._updateTrainingOptions = () => { throw new Error('_updateTrainingOptions was called'); }; // net.fromJSON({ sizes: [], layers: [] }); // expect(net.activation === 'sigmoid').toBeTruthy(); }); }); }); describe('default net json', () => { const activation = 'leaky-relu'; const originalNet = new NeuralNetwork({ activation }); originalNet.train( [ { input: { 0: Math.random(), b: Math.random() }, output: { c: Math.random(), 0: Math.random() }, }, { input: { 0: Math.random(), b: Math.random() }, output: { c: Math.random(), 0: Math.random() }, }, ], { timeout: 4 } ); const serialized = originalNet.toJSON(); const serializedNet = new NeuralNetwork().fromJSON( JSON.parse(JSON.stringify(serialized)) ); const input = { 0: Math.random(), b: Math.random() }; const originalNetTrainOpts = originalNet.trainOpts; const serializedNetTrainOpts = serializedNet.trainOpts; describe('.options', () => { it('option inputSize', () => { expect(originalNet.options.inputSize).toEqual( serialized.options.inputSize ); }); it('option hiddenLayers', () => { expect(originalNet.options.hiddenLayers).toEqual( serialized.options.hiddenLayers ); }); it('option outputSize', () => { expect(originalNet.options.outputSize).toEqual( serialized.options.outputSize ); }); it('option binaryThresh', () => { expect(originalNet.options.binaryThresh).toEqual( serialized.options.binaryThresh ); }); }); describe('.trainOpts', () => { it('training options activation', () => { expect(originalNetTrainOpts.activation).toBe( serializedNetTrainOpts.activation ); }); it('training options iterations', () => { expect(originalNetTrainOpts.iterations).toBe( serializedNetTrainOpts.iterations ); }); it('training options errorThresh', () => { expect(originalNetTrainOpts.errorThresh).toBe( serializedNetTrainOpts.errorThresh ); }); it('training options log', () => { expect(originalNetTrainOpts.log).toBe(serializedNetTrainOpts.log); }); it('training options logPeriod', () => { expect(originalNetTrainOpts.logPeriod).toBe( serializedNetTrainOpts.logPeriod ); }); it('training options learningRate', () => { expect(originalNetTrainOpts.learningRate).toBe( serializedNetTrainOpts.learningRate ); }); it('training options momentum', () => { expect(originalNetTrainOpts.momentum).toBe( serializedNetTrainOpts.momentum ); }); it('training options callback', () => { expect(originalNetTrainOpts.callback).toBe( serializedNetTrainOpts.callback ); }); it('training options callbackPeriod', () => { expect(originalNetTrainOpts.callbackPeriod).toBe( serializedNetTrainOpts.callbackPeriod ); }); it('training options timeout', () => { expect(originalNetTrainOpts.timeout).toBe(serializedNetTrainOpts.timeout); }); it('training options praxis', () => { expect(originalNetTrainOpts.praxis).toBe(serializedNetTrainOpts.praxis); }); it('training options beta1', () => { expect(originalNetTrainOpts.beta1).toBe(serializedNetTrainOpts.beta1); }); it('training options beta2', () => { expect(originalNetTrainOpts.beta2).toBe(serializedNetTrainOpts.beta2); }); it('training options epsilon', () => { expect(originalNetTrainOpts.epsilon).toBe(serializedNetTrainOpts.epsilon); }); }); describe('.sizes', () => { it('sizes are same', () => { expect(serialized.sizes).toEqual(originalNet.sizes); }); }); it('can run originalNet, and serializedNet, with same output', () => { const output1 = originalNet.run(input); const output2 = serializedNet.run(input); expect(output2).toEqual(output1); }); }); ================================================ FILE: src/neural-network.options.test.ts ================================================ import { NeuralNetwork } from './neural-network'; describe('NeuralNetwork', () => { describe('options', () => { it('hiddenLayers', () => { const net = new NeuralNetwork({ hiddenLayers: [8, 7] }); net.train([ { input: [0, 0], output: [0] }, { input: [0, 1], output: [1] }, { input: [1, 0], output: [1] }, { input: [1, 1], output: [0] }, ]); const json = net.toJSON(); expect(json.layers.length).toBe(4); expect(json.layers[1].weights.length).toBe(8); expect(json.layers[2].weights.length).toBe(7); }); it('hiddenLayers default expand to input size', () => { const net = new NeuralNetwork(); net.train([ { input: [0, 0, 1, 1, 1, 1, 1, 1, 1], output: [0] }, { input: [0, 1, 1, 1, 1, 1, 1, 1, 1], output: [1] }, { input: [1, 0, 1, 1, 1, 1, 1, 1, 1], output: [1] }, { input: [1, 1, 1, 1, 1, 1, 1, 1, 1], output: [0] }, ]); const json = net.toJSON(); expect(json.layers.length).toBe(3); expect(json.layers[1].weights.length).toBe(4); }); }); describe('after initialization', () => { describe('trainOpts', () => { it('iterations should be settable in the constructor', () => { const options = { iterations: 5 }; const net = new NeuralNetwork(options); expect(options.iterations).toBe(net.trainOpts.iterations); }); it('errorThresh should be settable in the constructor', () => { const options = { errorThresh: 0.1 }; const net = new NeuralNetwork(options); expect(options.errorThresh).toBe(net.trainOpts.errorThresh); }); it('log should allow setting the training options to the constructor', () => { const log = function () {}; const options = { log: log }; const net = new NeuralNetwork(options); expect(net.trainOpts.log).toBe(log); }); it('logPeriod should be settable in the constructor', () => { const options = { logPeriod: 5 }; const net = new NeuralNetwork(options); expect(options.logPeriod).toBe(net.trainOpts.logPeriod); }); it('learningRate should be settable in the constructor', () => { const options = { learningRate: 0.5 }; const net = new NeuralNetwork(options); expect(options.learningRate).toBe(net.trainOpts.learningRate); }); it('momentum should be settable in the constructor', () => { const options = { momentum: 0.2 }; const net = new NeuralNetwork(options); expect(options.momentum).toBe(net.trainOpts.momentum); }); it('callback should be settable in the constructor', () => { const cb = function () {}; const options = { callback: cb }; const net = new NeuralNetwork(options); expect(net.trainOpts.callback).toBe(cb); }); it('callbackPeriod should be settable in the constructor', () => { const options = { callbackPeriod: 2 }; const net = new NeuralNetwork(options); expect(options.callbackPeriod).toBe(net.trainOpts.callbackPeriod); }); it('timeout should be settable in the constructor', () => { const options = { timeout: 1500 }; const net = new NeuralNetwork(options); expect(options.timeout).toBe(net.trainOpts.timeout); }); it('binaryThresh should be settable in the constructor', () => { const options = { binaryThresh: 0.2 }; const net = new NeuralNetwork(options); expect(options.binaryThresh).toBe(net.options.binaryThresh); }); it('hiddenLayers should be settable in the constructor', () => { const options = { hiddenLayers: [2, 3, 4] }; const net = new NeuralNetwork(options); expect(options.hiddenLayers).toEqual(net.options.hiddenLayers); }); it('activation should be settable in the constructor', () => { const options = { activation: 'relu' }; const net = new NeuralNetwork(options); expect(options.activation).toBe(net.trainOpts.activation); }); it('leakyReluAlpha should be settable in the constructor', () => { const options = { leakyReluAlpha: 0.1337 }; const net = new NeuralNetwork(options); expect(options.leakyReluAlpha).toBe(net.trainOpts.leakyReluAlpha); }); }); }); }); ================================================ FILE: src/neural-network.test-method.test.ts ================================================ import { NeuralNetwork } from './neural-network'; import { INeuralNetworkBinaryTestResult } from './neural-network-types'; describe('NeuralNetwork.test()', () => { describe('using binary data', () => { const trainingData = [ { input: [0, 0], output: [0] }, { input: [0, 1], output: [1] }, { input: [1, 0], output: [1] }, { input: [1, 1], output: [0] }, ]; const net = new NeuralNetwork(); net.train(trainingData); it('can test XOR data', () => { const test1 = net.test([ trainingData[0], ]) as INeuralNetworkBinaryTestResult; expect(test1.error < 0.05).toBeTruthy(); expect(test1.misclasses.length).toBe(0); expect(test1.trueNeg).toBe(1); expect(test1.truePos).toBe(0); expect(test1.falseNeg).toBe(0); expect(test1.falsePos).toBe(0); expect(test1.total).toBe(1); expect(test1.precision).toBe(0); expect(test1.recall).toBe(0); expect(test1.accuracy).toBe(1); const test2 = net.test([ trainingData[1], ]) as INeuralNetworkBinaryTestResult; expect(Object.keys(test2).length).toBe(10); expect(test2.error < 0.05).toBeTruthy(); expect(test2.misclasses.length).toBe(0); expect(test2.trueNeg).toBe(0); expect(test2.truePos).toBe(1); expect(test2.falseNeg).toBe(0); expect(test2.falsePos).toBe(0); expect(test2.total).toBe(1); expect(test2.precision).toBe(1); expect(test2.recall).toBe(1); expect(test2.accuracy).toBe(1); const test3 = net.test([ trainingData[2], ]) as INeuralNetworkBinaryTestResult; expect(Object.keys(test3).length).toBe(10); expect(test3.error < 0.05).toBeTruthy(); expect(test3.misclasses.length).toBe(0); expect(test3.trueNeg).toBe(0); expect(test3.truePos).toBe(1); expect(test3.falseNeg).toBe(0); expect(test3.falsePos).toBe(0); expect(test3.total).toBe(1); expect(test3.precision).toBe(1); expect(test3.recall).toBe(1); expect(test3.accuracy).toBe(1); const test4 = net.test([ trainingData[3], ]) as INeuralNetworkBinaryTestResult; expect(Object.keys(test4).length).toBe(10); expect(test4.error < 0.05).toBeTruthy(); expect(test4.misclasses.length).toBe(0); expect(test4.trueNeg).toBe(1); expect(test4.truePos).toBe(0); expect(test4.falseNeg).toBe(0); expect(test4.falsePos).toBe(0); expect(test4.total).toBe(1); expect(test4.precision).toBe(0); expect(test4.recall).toBe(0); expect(test4.accuracy).toBe(1); }); }); describe('using simple math float data', () => { const trainingData = [ { input: { one: 1, two: 1, three: 0, four: 0 }, output: { three: 1, four: 0, five: 0, six: 0 }, }, { input: { one: 1, two: 0, three: 1, four: 0 }, output: { three: 0, four: 1, five: 0, six: 0 }, }, { input: { one: 0, two: 1, three: 1, four: 0 }, output: { three: 0, four: 0, five: 1, six: 0 }, }, { input: { one: 0, two: 1, three: 0, four: 1 }, output: { three: 0, four: 0, five: 0, six: 1 }, }, ]; const net = new NeuralNetwork(); net.train(trainingData); it('can test simple math data', () => { const test1 = net.test([trainingData[0]]); expect(Object.keys(test1).length).toBe(3); expect(test1.total).toBe(1); expect(test1.error < 0.05).toBeTruthy(); expect(test1.misclasses.length).toBe(0); const test2 = net.test([trainingData[1]]); expect(Object.keys(test2).length).toBe(3); expect(test2.total).toBe(1); expect(test2.error < 0.05).toBeTruthy(); expect(test2.misclasses.length).toBe(0); const test3 = net.test([trainingData[2]]); expect(Object.keys(test3).length).toBe(3); expect(test3.total).toBe(1); expect(test3.error < 0.05).toBeTruthy(); expect(test3.misclasses.length).toBe(0); const test4 = net.test([trainingData[3]]); expect(Object.keys(test4).length).toBe(3); expect(test4.total).toBe(1); expect(test4.error < 0.05).toBeTruthy(); expect(test4.misclasses.length).toBe(0); }); }); }); ================================================ FILE: src/neural-network.to-function.test.ts ================================================ import { NeuralNetwork } from './neural-network'; import { INumberHash } from './lookup'; import { xorTrainingData } from './test-utils'; describe('.toFunction()', () => { it.each(['sigmoid', 'relu', 'relu', 'leaky-relu'])( 'having %p activation, runs same as original network', (activation) => { const originalNet = new NeuralNetwork({ activation, }); originalNet.train(xorTrainingData); const xor = originalNet.toFunction(); expect(xor([0, 0])[0]).toBeCloseTo(originalNet.run([0, 0])[0]); expect(xor([0, 1])[0]).toBeCloseTo(originalNet.run([0, 1])[0]); expect(xor([1, 0])[0]).toBeCloseTo(originalNet.run([1, 0])[0]); expect(xor([1, 1])[0]).toBeCloseTo(originalNet.run([1, 1])[0]); } ); it('can work with partial input objects', () => { const trainingData = [ { input: { 'I am super happy!': 1 }, output: { happy: 1 } }, { input: { 'What a pill!': 1 }, output: { sarcastic: 1 } }, { input: { 'I am super unhappy!': 1 }, output: { sad: 1 } }, { input: { 'Are we there yet?': 1 }, output: { excited: 1 } }, ]; interface IInput extends INumberHash { 'I am super happy!': number; 'What a pill!': number; 'I am super unhappy!': number; 'Are we there yet?': number; } interface IOutput extends INumberHash { happy: number; sarcastic: number; sad: number; excited: number; } const net = new NeuralNetwork({ hiddenLayers: [3], }); net.train(trainingData, { iterations: 1000, errorThresh: 0.01, }); const happyOutput = net.run({ 'I am super happy!': 1 }); expect(happyOutput.happy).toBeGreaterThan(0.5); expect(happyOutput.sarcastic).toBeLessThan(0.5); expect(happyOutput.sad).toBeLessThan(0.5); expect(happyOutput.excited).toBeLessThan(0.5); const sarcasticOutput = net.run({ 'What a pill!': 1 }); expect(sarcasticOutput.happy).toBeLessThan(0.5); expect(sarcasticOutput.sarcastic).toBeGreaterThan(0.5); expect(sarcasticOutput.sad).toBeLessThan(0.5); expect(sarcasticOutput.excited).toBeLessThan(0.5); const sadOutput = net.run({ 'I am super unhappy!': 1 }); expect(sadOutput.happy).toBeLessThan(0.5); expect(sadOutput.sarcastic).toBeLessThan(0.5); expect(sadOutput.sad).toBeGreaterThan(0.5); expect(sadOutput.excited).toBeLessThan(0.5); const excitedOutput = net.run({ 'Are we there yet?': 1 }); expect(excitedOutput.happy).toBeLessThan(0.5); expect(excitedOutput.sarcastic).toBeLessThan(0.5); expect(excitedOutput.sad).toBeLessThan(0.5); expect(excitedOutput.excited).toBeGreaterThan(0.5); const run = net.toFunction(); const runHappyOutput = run({ 'I am super happy!': 1 }); expect(runHappyOutput.happy).toBeCloseTo(happyOutput.happy); expect(runHappyOutput.sarcastic).toBeCloseTo(happyOutput.sarcastic); expect(runHappyOutput.sad).toBeCloseTo(happyOutput.sad); expect(runHappyOutput.excited).toBeCloseTo(happyOutput.excited); const runSarcasticOutput = run({ 'What a pill!': 1 }); expect(runSarcasticOutput.happy).toBeCloseTo(sarcasticOutput.happy); expect(runSarcasticOutput.sarcastic).toBeCloseTo(sarcasticOutput.sarcastic); expect(runSarcasticOutput.sad).toBeCloseTo(sarcasticOutput.sad); expect(runSarcasticOutput.excited).toBeCloseTo(sarcasticOutput.excited); const runSadOutput = run({ 'I am super unhappy!': 1 }); expect(runSadOutput.happy).toBeCloseTo(sadOutput.happy); expect(runSadOutput.sarcastic).toBeCloseTo(sadOutput.sarcastic); expect(runSadOutput.sad).toBeCloseTo(sadOutput.sad); expect(runSadOutput.excited).toBeCloseTo(sadOutput.excited); const runExcitedOutput = run({ 'Are we there yet?': 1 }); expect(runExcitedOutput.happy).toBeCloseTo(excitedOutput.happy); expect(runExcitedOutput.sarcastic).toBeCloseTo(excitedOutput.sarcastic); expect(runExcitedOutput.sad).toBeCloseTo(excitedOutput.sad); expect(runExcitedOutput.excited).toBeCloseTo(excitedOutput.excited); }); }); ================================================ FILE: src/neural-network.trainopts.test.ts ================================================ import { NeuralNetwork } from './neural-network'; const data = [ { input: [0, 0], output: [0] }, { input: [0, 1], output: [1] }, { input: [1, 0], output: [1] }, { input: [1, 1], output: [1] }, ]; describe('NeuralNetwork.train()', () => { describe('train() options', () => { it('train until error threshold reached', () => { const net = new NeuralNetwork(); const res = net.train(data, { errorThresh: 0.2 }); expect(res.error < 0.2).toBeTruthy(); }); it('train until max iterations reached', () => { const net = new NeuralNetwork(); const res = net.train(data, { iterations: 25 }); expect(res.iterations).toBe(25); }); it('training callback called with training stats', () => { const iters = 100; const period = 20; const target = iters / period; let calls = 0; const net = new NeuralNetwork(); net.train(data, { iterations: iters, callbackPeriod: period, callback: (res) => { expect(res.iterations % period === 0).toBeTruthy(); calls++; }, }); expect(target === calls).toBeTruthy(); }); it('learningRate - higher learning rate should train faster', () => { const data = [ { input: [0, 0], output: [0] }, { input: [0, 1], output: [1] }, { input: [1, 0], output: [1] }, { input: [1, 1], output: [1] }, ]; const net = new NeuralNetwork(); const res = net.train(data, { learningRate: 0.5 }); const net2 = new NeuralNetwork(); const res2 = net2.train(data, { learningRate: 0.8 }); expect(res.iterations > res2.iterations * 1.1).toBeTruthy(); }); it('momentum - higher momentum should train faster', () => { const data = [ { input: [0, 0], output: [0] }, { input: [0, 1], output: [1] }, { input: [1, 0], output: [1] }, { input: [1, 1], output: [1] }, ]; const net = new NeuralNetwork({ momentum: 0.1 }); const res = net.train(data); const net2 = new NeuralNetwork({ momentum: 0.5 }); const res2 = net2.train(data); expect(Math.abs(res.iterations - res2.iterations)).toBeLessThan(500); }); }); describe('train() and trainAsync()', () => { let prepTrainingSpy: jest.SpyInstance; let trainingTickSpy: jest.SpyInstance; beforeEach(() => { prepTrainingSpy = jest.spyOn(NeuralNetwork.prototype, 'prepTraining'); trainingTickSpy = jest.spyOn(NeuralNetwork.prototype, 'trainingTick'); }); afterEach(() => { prepTrainingSpy.mockRestore(); trainingTickSpy.mockRestore(); }); test('both call this.prepTraining()', async () => { const syncNet = new NeuralNetwork(); const options = { iterations: 1 }; syncNet.train(data, options); expect(prepTrainingSpy).toHaveBeenCalledWith(data, options); prepTrainingSpy.mockReset(); const asyncNet = new NeuralNetwork(); asyncNet.trainAsync(data, options).then(console.log).catch(console.error); expect(prepTrainingSpy).toHaveBeenCalledWith(data, options); }); test('both call this.trainingTick()', async () => { const syncNet = new NeuralNetwork(); const options = { iterations: 1 }; const dataFormatted = data.map((datum) => { return { input: Float32Array.from(datum.input), output: Float32Array.from(datum.output), }; }); syncNet.train(data, options); expect(trainingTickSpy.mock.calls[0][0]).toEqual(dataFormatted); expect(trainingTickSpy.mock.calls[0][1].error).toBeLessThan(2); expect(trainingTickSpy.mock.calls[0][1].iterations).toEqual(1); expect(trainingTickSpy.mock.calls[0][2]).toEqual(Infinity); trainingTickSpy.mockReset(); const asyncNet = new NeuralNetwork(); asyncNet.trainAsync(data, options).then(console.log).catch(console.error); expect(trainingTickSpy.mock.calls[0][0]).toEqual(dataFormatted); expect(trainingTickSpy.mock.calls[0][1].error).toBeLessThan(2); expect(trainingTickSpy.mock.calls[0][1].iterations).toEqual(0); expect(trainingTickSpy.mock.calls[0][2]).toEqual(Infinity); }); }); describe('training options validation', () => { it('iterations validation', () => { const net = new NeuralNetwork(); expect(() => { // @ts-expect-error iterations is expected as number net.updateTrainingOptions({ iterations: 'should be a string' }); }).toThrow(); expect(() => { // @ts-expect-error iterations is expected as number net.updateTrainingOptions({ iterations: () => {} }); }).toThrow(); expect(() => { // @ts-expect-error iterations is expected as number net.updateTrainingOptions({ iterations: false }); }).toThrow(); expect(() => { net.updateTrainingOptions({ iterations: -1 }); }).toThrow(); expect(() => { net.updateTrainingOptions({ iterations: 5000 }); }).not.toThrow(); }); it('errorThresh validation', () => { const net = new NeuralNetwork(); expect(() => { // @ts-expect-error errorThresh is expected as number net.updateTrainingOptions({ errorThresh: 'no strings' }); }).toThrow(); expect(() => { // @ts-expect-error errorThresh is expected as number net.updateTrainingOptions({ errorThresh: () => {} }); }).toThrow(); expect(() => { net.updateTrainingOptions({ errorThresh: 5 }); }).toThrow(); expect(() => { net.updateTrainingOptions({ errorThresh: -1 }); }).toThrow(); expect(() => { // @ts-expect-error errorThresh is expected as number net.updateTrainingOptions({ errorThresh: false }); }).toThrow(); expect(() => { net.updateTrainingOptions({ errorThresh: 0.008 }); }).not.toThrow(); }); it('log validation', () => { const net = new NeuralNetwork(); expect(() => { // @ts-expect-error log should be boolean or function net.updateTrainingOptions({ log: 'no strings' }); }).toThrow(); expect(() => { // @ts-expect-error log should be boolean or function net.updateTrainingOptions({ log: 4 }); }).toThrow(); expect(() => { net.updateTrainingOptions({ log: false }); }).not.toThrow(); expect(() => { net.updateTrainingOptions({ log: () => {} }); }).not.toThrow(); }); it('logPeriod validation', () => { const net = new NeuralNetwork(); expect(() => { // @ts-expect-error logPeriod should be positive number net.updateTrainingOptions({ logPeriod: 'no strings' }); }).toThrow(); expect(() => { net.updateTrainingOptions({ logPeriod: -50 }); }).toThrow(); expect(() => { // @ts-expect-error logPeriod should be positive number net.updateTrainingOptions({ logPeriod: () => {} }); }).toThrow(); expect(() => { // @ts-expect-error logPeriod should be positive number net.updateTrainingOptions({ logPeriod: false }); }).toThrow(); expect(() => { net.updateTrainingOptions({ logPeriod: 40 }); }).not.toThrow(); }); it('learningRate validation', () => { const net = new NeuralNetwork(); expect(() => { // @ts-expect-error learningRate should be positive number net.updateTrainingOptions({ learningRate: 'no strings' }); }).toThrow(); expect(() => { net.updateTrainingOptions({ learningRate: -50 }); }).toThrow(); expect(() => { net.updateTrainingOptions({ learningRate: 50 }); }).toThrow(); expect(() => { // @ts-expect-error learningRate should be positive number net.updateTrainingOptions({ learningRate: () => {} }); }).toThrow(); expect(() => { // @ts-expect-error learningRate should be positive number net.updateTrainingOptions({ learningRate: false }); }).toThrow(); expect(() => { net.updateTrainingOptions({ learningRate: 0.5 }); }).not.toThrow(); }); it('momentum validation', () => { const net = new NeuralNetwork(); expect(() => { // @ts-expect-error momentum should be positive number net.updateTrainingOptions({ momentum: 'no strings' }); }).toThrow(); expect(() => { net.updateTrainingOptions({ momentum: -50 }); }).toThrow(); expect(() => { net.updateTrainingOptions({ momentum: 50 }); }).toThrow(); expect(() => { // @ts-expect-error momentum should be positive number net.updateTrainingOptions({ momentum: () => {} }); }).toThrow(); expect(() => { // @ts-expect-error momentum should be positive number net.updateTrainingOptions({ momentum: false }); }).toThrow(); expect(() => { net.updateTrainingOptions({ momentum: 0.8 }); }).not.toThrow(); }); it('callback validation', () => { const net = new NeuralNetwork(); expect(() => { // @ts-expect-error callback should be a function net.updateTrainingOptions({ callback: 'no strings' }); }).toThrow(); expect(() => { // @ts-expect-error callback should be a function net.updateTrainingOptions({ callback: 4 }); }).toThrow(); expect(() => { // @ts-expect-error callback should be a function net.updateTrainingOptions({ callback: false }); }).toThrow(); expect(() => { net.updateTrainingOptions({ callback: undefined }); }).not.toThrow(); expect(() => { net.updateTrainingOptions({ callback: () => {} }); }).not.toThrow(); }); it('callbackPeriod validation', () => { const net = new NeuralNetwork(); expect(() => { // @ts-expect-error callbackPeriod should be a number net.updateTrainingOptions({ callbackPeriod: 'no strings' }); }).toThrow(); expect(() => { net.updateTrainingOptions({ callbackPeriod: -50 }); }).toThrow(); expect(() => { // @ts-expect-error callbackPeriod should be a number net.updateTrainingOptions({ callbackPeriod: () => {} }); }).toThrow(); expect(() => { // @ts-expect-error callbackPeriod should be a number net.updateTrainingOptions({ callbackPeriod: false }); }).toThrow(); expect(() => { net.updateTrainingOptions({ callbackPeriod: 40 }); }).not.toThrow(); }); it('timeout validation', () => { const net = new NeuralNetwork(); expect(() => { // @ts-expect-error timeout should be a number net.updateTrainingOptions({ timeout: 'no strings' }); }).toThrow(); expect(() => { net.updateTrainingOptions({ timeout: -50 }); }).toThrow(); expect(() => { // @ts-expect-error timeout should be a number net.updateTrainingOptions({ timeout: () => {} }); }).toThrow(); expect(() => { // @ts-expect-error timeout should be a number net.updateTrainingOptions({ timeout: false }); }).toThrow(); expect(() => { net.updateTrainingOptions({ timeout: 40 }); }).not.toThrow(); }); it('should retain the options from instantiation as defaults', () => { const config = { iterations: 1, errorThresh: 0.0001, binaryThresh: 0.05, hiddenLayers: [1], activation: 'sigmoid', }; const net = new NeuralNetwork(config); const trainData = [ { input: [0, 0], output: [0] }, { input: [0, 1], output: [1] }, { input: [1, 0], output: [1] }, { input: [1, 1], output: [0] }, ]; const trainResult = net.train(trainData); expect(trainResult.iterations).toBe(1); }); }); }); ================================================ FILE: src/neural-network.ts ================================================ import { Thaw } from 'thaw.js'; import { ITrainingStatus } from './feed-forward'; import { INumberHash, lookup } from './lookup'; import { INeuralNetworkBinaryTestResult, INeuralNetworkState, INeuralNetworkTestResult, } from './neural-network-types'; import { arrayToFloat32Array } from './utilities/cast'; import { LookupTable } from './utilities/lookup-table'; import { max } from './utilities/max'; import { mse } from './utilities/mse'; import { randos } from './utilities/randos'; import { zeros } from './utilities/zeros'; type NeuralNetworkFormatter = | ((v: INumberHash) => Float32Array) | ((v: number[]) => Float32Array); export function getTypedArrayFn( value: INeuralNetworkData, table: INumberHash | null ): null | NeuralNetworkFormatter { if ((value as Float32Array).buffer instanceof ArrayBuffer) { return null; } if (Array.isArray(value)) { return arrayToFloat32Array; } if (!table) throw new Error('table is not Object'); const { length } = Object.keys(table); return (v: INumberHash): Float32Array => { const array = new Float32Array(length); for (const p in table) { if (!table.hasOwnProperty(p)) continue; if (typeof v[p] !== 'number') continue; array[table[p]] = v[p] || 0; } return array; }; } export type NeuralNetworkActivation = | 'sigmoid' | 'relu' | 'leaky-relu' | 'tanh'; export interface IJSONLayer { biases: number[]; weights: number[][]; } export interface INeuralNetworkJSON { type: string; sizes: number[]; layers: IJSONLayer[]; inputLookup: INumberHash | null; inputLookupLength: number; outputLookup: INumberHash | null; outputLookupLength: number; options: INeuralNetworkOptions; trainOpts: INeuralNetworkTrainOptionsJSON; } export interface INeuralNetworkOptions { inputSize: number; outputSize: number; binaryThresh: number; hiddenLayers?: number[]; } export function defaults(): INeuralNetworkOptions { return { inputSize: 0, outputSize: 0, binaryThresh: 0.5, }; } export interface INeuralNetworkTrainOptionsJSON { activation: NeuralNetworkActivation | string; iterations: number; errorThresh: number; log: boolean; logPeriod: number; leakyReluAlpha: number; learningRate: number; momentum: number; callbackPeriod: number; timeout: number | 'Infinity'; praxis?: 'adam'; beta1: number; beta2: number; epsilon: number; } export interface INeuralNetworkPreppedTrainingData { status: ITrainingStatus; preparedData: Array>; endTime: number; } export interface INeuralNetworkTrainOptions { activation: NeuralNetworkActivation | string; iterations: number; errorThresh: number; log: boolean | ((status: INeuralNetworkState) => void); logPeriod: number; leakyReluAlpha: number; learningRate: number; momentum: number; callback?: (status: { iterations: number; error: number }) => void; callbackPeriod: number; timeout: number; praxis?: 'adam'; beta1: number; beta2: number; epsilon: number; } export function trainDefaults(): INeuralNetworkTrainOptions { return { activation: 'sigmoid', iterations: 20000, // the maximum times to iterate the training data errorThresh: 0.005, // the acceptable error percentage from training data log: false, // true to use console.log, when a function is supplied it is used logPeriod: 10, // iterations between logging out leakyReluAlpha: 0.01, learningRate: 0.3, // multiply's against the input and the delta then adds to momentum momentum: 0.1, // multiply's against the specified "change" then adds to learning rate for change callbackPeriod: 10, // the number of iterations through the training data between callback calls timeout: Infinity, // the max number of milliseconds to train for beta1: 0.9, beta2: 0.999, epsilon: 1e-8, }; } export type INeuralNetworkData = number[] | Float32Array | Partial; // TODO: should be replaced by ITrainingDatum export interface INeuralNetworkDatum { input: InputType; output: OutputType; } export interface INeuralNetworkDatumFormatted { input: T; output: T; } export class NeuralNetwork< InputType extends INeuralNetworkData, OutputType extends INeuralNetworkData > { options: INeuralNetworkOptions = defaults(); trainOpts: INeuralNetworkTrainOptions = trainDefaults(); sizes: number[] = []; outputLayer = -1; biases: Float32Array[] = []; weights: Float32Array[][] = []; // weights for bias nodes outputs: Float32Array[] = []; // state for training deltas: Float32Array[] = []; changes: Float32Array[][] = []; // for momentum errors: Float32Array[] = []; errorCheckInterval = 1; inputLookup: INumberHash | null = null; inputLookupLength = 0; outputLookup: INumberHash | null = null; outputLookupLength = 0; _formatInput: NeuralNetworkFormatter | null = null; _formatOutput: NeuralNetworkFormatter | null = null; runInput: (input: Float32Array) => Float32Array = (input: Float32Array) => { this.setActivation(); return this.runInput(input); }; calculateDeltas: (output: Float32Array) => void = ( output: Float32Array ): void => { this.setActivation(); return this.calculateDeltas(output); }; // adam biasChangesLow: Float32Array[] = []; biasChangesHigh: Float32Array[] = []; changesLow: Float32Array[][] = []; changesHigh: Float32Array[][] = []; iterations = 0; constructor( options: Partial = {} ) { this.options = { ...this.options, ...options }; this.updateTrainingOptions(options); const { inputSize, hiddenLayers, outputSize } = this.options; if (inputSize && outputSize) { this.sizes = [inputSize].concat(hiddenLayers ?? []).concat([outputSize]); } } /** * * Expects this.sizes to have been set */ initialize(): void { if (!this.sizes.length) { throw new Error('Sizes must be set before initializing'); } this.outputLayer = this.sizes.length - 1; this.biases = new Array(this.outputLayer); // weights for bias nodes this.weights = new Array(this.outputLayer); this.outputs = new Array(this.outputLayer); // state for training this.deltas = new Array(this.outputLayer); this.changes = new Array(this.outputLayer); // for momentum this.errors = new Array(this.outputLayer); for (let layerIndex = 0; layerIndex <= this.outputLayer; layerIndex++) { const size = this.sizes[layerIndex]; this.deltas[layerIndex] = zeros(size); this.errors[layerIndex] = zeros(size); this.outputs[layerIndex] = zeros(size); if (layerIndex > 0) { this.biases[layerIndex] = randos(size); this.weights[layerIndex] = new Array(size); this.changes[layerIndex] = new Array(size); for (let nodeIndex = 0; nodeIndex < size; nodeIndex++) { const prevSize = this.sizes[layerIndex - 1]; this.weights[layerIndex][nodeIndex] = randos(prevSize); this.changes[layerIndex][nodeIndex] = zeros(prevSize); } } } this.setActivation(); if (this.trainOpts.praxis === 'adam') { this._setupAdam(); } } setActivation(activation?: NeuralNetworkActivation): void { const value = activation ?? this.trainOpts.activation; switch (value) { case 'sigmoid': this.runInput = this._runInputSigmoid; this.calculateDeltas = this._calculateDeltasSigmoid; break; case 'relu': this.runInput = this._runInputRelu; this.calculateDeltas = this._calculateDeltasRelu; break; case 'leaky-relu': this.runInput = this._runInputLeakyRelu; this.calculateDeltas = this._calculateDeltasLeakyRelu; break; case 'tanh': this.runInput = this._runInputTanh; this.calculateDeltas = this._calculateDeltasTanh; break; default: throw new Error( `Unknown activation ${value}. Available activations are: 'sigmoid', 'relu', 'leaky-relu', 'tanh'` ); } } get isRunnable(): boolean { return this.sizes.length > 0; } run(input: Partial): OutputType { if (!this.isRunnable) { throw new Error('network not runnable'); } let formattedInput: Float32Array; if (this.inputLookup) { formattedInput = lookup.toArray( this.inputLookup, (input as unknown) as INumberHash, this.inputLookupLength ); } else { formattedInput = (input as unknown) as Float32Array; } this.validateInput(formattedInput); const output = this.runInput(formattedInput).slice(0); if (this.outputLookup) { return (lookup.toObject( this.outputLookup, output ) as unknown) as OutputType; } return (output as unknown) as OutputType; } _runInputSigmoid(input: Float32Array): Float32Array { this.outputs[0] = input; // set output state of input layer let output = null; for (let layer = 1; layer <= this.outputLayer; layer++) { const activeLayer = this.sizes[layer]; const activeWeights = this.weights[layer]; const activeBiases = this.biases[layer]; const activeOutputs = this.outputs[layer]; for (let node = 0; node < activeLayer; node++) { const weights = activeWeights[node]; let sum = activeBiases[node]; for (let k = 0; k < weights.length; k++) { sum += weights[k] * input[k]; } // sigmoid activeOutputs[node] = 1 / (1 + Math.exp(-sum)); } output = input = activeOutputs; } if (!output) { throw new Error('output was empty'); } return output; } _runInputRelu(input: Float32Array): Float32Array { this.outputs[0] = input; // set output state of input layer let output = null; for (let layer = 1; layer <= this.outputLayer; layer++) { const activeSize = this.sizes[layer]; const activeWeights = this.weights[layer]; const activeBiases = this.biases[layer]; const activeOutputs = this.outputs[layer]; for (let node = 0; node < activeSize; node++) { const weights = activeWeights[node]; let sum = activeBiases[node]; for (let k = 0; k < weights.length; k++) { sum += weights[k] * input[k]; } // relu activeOutputs[node] = sum < 0 ? 0 : sum; } output = input = activeOutputs; } if (!output) { throw new Error('output was empty'); } return output; } _runInputLeakyRelu(input: Float32Array): Float32Array { this.outputs[0] = input; // set output state of input layer const { leakyReluAlpha } = this.trainOpts; let output = null; for (let layer = 1; layer <= this.outputLayer; layer++) { const activeSize = this.sizes[layer]; const activeWeights = this.weights[layer]; const activeBiases = this.biases[layer]; const activeOutputs = this.outputs[layer]; for (let node = 0; node < activeSize; node++) { const weights = activeWeights[node]; let sum = activeBiases[node]; for (let k = 0; k < weights.length; k++) { sum += weights[k] * input[k]; } // leaky relu activeOutputs[node] = Math.max(sum, leakyReluAlpha * sum); } output = input = activeOutputs; } if (!output) { throw new Error('output was empty'); } return output; } _runInputTanh(input: Float32Array): Float32Array { this.outputs[0] = input; // set output state of input layer let output = null; for (let layer = 1; layer <= this.outputLayer; layer++) { const activeSize = this.sizes[layer]; const activeWeights = this.weights[layer]; const activeBiases = this.biases[layer]; const activeOutputs = this.outputs[layer]; for (let node = 0; node < activeSize; node++) { const weights = activeWeights[node]; let sum = activeBiases[node]; for (let k = 0; k < weights.length; k++) { sum += weights[k] * input[k]; } // tanh activeOutputs[node] = Math.tanh(sum); } output = input = activeOutputs; } if (!output) { throw new Error('output was empty'); } return output; } /** * * Verifies network sizes are initialized * If they are not it will initialize them based off the data set. */ verifyIsInitialized( preparedData: Array> ): void { if (this.sizes.length && this.outputLayer > 0) return; this.sizes = []; this.sizes.push(preparedData[0].input.length); if (!this.options.hiddenLayers) { this.sizes.push( Math.max(3, Math.floor(preparedData[0].input.length / 2)) ); } else { this.options.hiddenLayers.forEach((size) => { this.sizes.push(size); }); } this.sizes.push(preparedData[0].output.length); this.initialize(); } updateTrainingOptions(trainOpts: Partial): void { const merged = { ...this.trainOpts, ...trainOpts }; this.validateTrainingOptions(merged); this.trainOpts = merged; this.setLogMethod(this.trainOpts.log); } validateTrainingOptions(options: INeuralNetworkTrainOptions): void { const validations: { [fnName: string]: () => boolean } = { activation: () => { return ['sigmoid', 'relu', 'leaky-relu', 'tanh'].includes( options.activation ); }, iterations: () => { const val = options.iterations; return typeof val === 'number' && val > 0; }, errorThresh: () => { const val = options.errorThresh; return typeof val === 'number' && val > 0 && val < 1; }, log: () => { const val = options.log; return typeof val === 'function' || typeof val === 'boolean'; }, logPeriod: () => { const val = options.logPeriod; return typeof val === 'number' && val > 0; }, leakyReluAlpha: () => { const val = options.leakyReluAlpha; return typeof val === 'number' && val > 0 && val < 1; }, learningRate: () => { const val = options.learningRate; return typeof val === 'number' && val > 0 && val < 1; }, momentum: () => { const val = options.momentum; return typeof val === 'number' && val > 0 && val < 1; }, callback: () => { const val = options.callback; return typeof val === 'function' || val === undefined; }, callbackPeriod: () => { const val = options.callbackPeriod; return typeof val === 'number' && val > 0; }, timeout: () => { const val = options.timeout; return typeof val === 'number' && val > 0; }, praxis: () => { const val = options.praxis; return !val || val === 'adam'; }, beta1: () => { const val = options.beta1; return val > 0 && val < 1; }, beta2: () => { const val = options.beta2; return val > 0 && val < 1; }, epsilon: () => { const val = options.epsilon; return val > 0 && val < 1; }, }; for (const p in validations) { const v = (options as unknown) as { [v: string]: string }; if (!validations[p]()) { throw new Error( `[${p}, ${v[p]}] is out of normal training range, your network will probably not train.` ); } } } /** * * Gets JSON of trainOpts object * NOTE: Activation is stored directly on JSON object and not in the training options */ getTrainOptsJSON(): INeuralNetworkTrainOptionsJSON { const { activation, iterations, errorThresh, log, logPeriod, leakyReluAlpha, learningRate, momentum, callbackPeriod, timeout, praxis, beta1, beta2, epsilon, } = this.trainOpts; return { activation, iterations, errorThresh, log: typeof log === 'function' ? true : typeof log === 'boolean' ? log : false, logPeriod, leakyReluAlpha, learningRate, momentum, callbackPeriod, timeout: timeout === Infinity ? 'Infinity' : timeout, praxis, beta1, beta2, epsilon, }; } setLogMethod(log: boolean | ((state: INeuralNetworkState) => void)): void { if (typeof log === 'function') { this.trainOpts.log = log; } else if (log) { this.trainOpts.log = this.logTrainingStatus; } else { this.trainOpts.log = false; } } logTrainingStatus(status: INeuralNetworkState): void { console.log( `iterations: ${status.iterations}, training error: ${status.error}` ); } calculateTrainingError( data: Array> ): number { let sum = 0; for (let i = 0; i < data.length; ++i) { sum += this.trainPattern(data[i], true) as number; } return sum / data.length; } trainPatterns(data: Array>): void { for (let i = 0; i < data.length; ++i) { this.trainPattern(data[i]); } } trainingTick( data: Array>, status: INeuralNetworkState, endTime: number ): boolean { const { callback, callbackPeriod, errorThresh, iterations, log, logPeriod, } = this.trainOpts; if ( status.iterations >= iterations || status.error <= errorThresh || Date.now() >= endTime ) { return false; } status.iterations++; if (log && status.iterations % logPeriod === 0) { status.error = this.calculateTrainingError(data); (log as (state: INeuralNetworkState) => void)(status); } else if (status.iterations % this.errorCheckInterval === 0) { status.error = this.calculateTrainingError(data); } else { this.trainPatterns(data); } if (callback && status.iterations % callbackPeriod === 0) { callback({ iterations: status.iterations, error: status.error, }); } return true; } prepTraining( data: Array>, options: Partial = {} ): INeuralNetworkPreppedTrainingData { this.updateTrainingOptions(options); const preparedData = this.formatData(data); const endTime = Date.now() + this.trainOpts.timeout; const status = { error: 1, iterations: 0, }; this.verifyIsInitialized(preparedData); this.validateData(preparedData); return { preparedData, status, endTime, }; } train( data: Array, Partial>>, options: Partial = {} ): INeuralNetworkState { const { preparedData, status, endTime } = this.prepTraining( data as Array>, options ); while (true) { if (!this.trainingTick(preparedData, status, endTime)) { break; } } return status; } async trainAsync( data: Array>, options: Partial = {} ): Promise { const { preparedData, status, endTime } = this.prepTraining(data, options); return await new Promise((resolve, reject) => { try { const thawedTrain: Thaw = new Thaw( new Array(this.trainOpts.iterations), { delay: true, each: () => this.trainingTick(preparedData, status, endTime) || thawedTrain.stop(), done: () => resolve(status), } ); thawedTrain.tick(); } catch (trainError) { reject(trainError); } }); } trainPattern( value: INeuralNetworkDatumFormatted, logErrorRate?: boolean ): number | null { // forward propagate this.runInput(value.input); // back propagate this.calculateDeltas(value.output); this.adjustWeights(); if (logErrorRate) { return mse(this.errors[this.outputLayer]); } return null; } _calculateDeltasSigmoid(target: Float32Array): void { for (let layer = this.outputLayer; layer >= 0; layer--) { const activeSize = this.sizes[layer]; const activeOutput = this.outputs[layer]; const activeError = this.errors[layer]; const activeDeltas = this.deltas[layer]; const nextLayer = this.weights[layer + 1]; for (let node = 0; node < activeSize; node++) { const output = activeOutput[node]; let error = 0; if (layer === this.outputLayer) { error = target[node] - output; } else { const deltas = this.deltas[layer + 1]; for (let k = 0; k < deltas.length; k++) { error += deltas[k] * nextLayer[k][node]; } } activeError[node] = error; activeDeltas[node] = error * output * (1 - output); } } } _calculateDeltasRelu(target: Float32Array): void { for (let layer = this.outputLayer; layer >= 0; layer--) { const currentSize = this.sizes[layer]; const currentOutputs = this.outputs[layer]; const nextWeights = this.weights[layer + 1]; const nextDeltas = this.deltas[layer + 1]; const currentErrors = this.errors[layer]; const currentDeltas = this.deltas[layer]; for (let node = 0; node < currentSize; node++) { const output = currentOutputs[node]; let error = 0; if (layer === this.outputLayer) { error = target[node] - output; } else { for (let k = 0; k < nextDeltas.length; k++) { error += nextDeltas[k] * nextWeights[k][node]; } } currentErrors[node] = error; currentDeltas[node] = output > 0 ? error : 0; } } } _calculateDeltasLeakyRelu(target: Float32Array): void { const alpha = this.trainOpts.leakyReluAlpha; for (let layer = this.outputLayer; layer >= 0; layer--) { const currentSize = this.sizes[layer]; const currentOutputs = this.outputs[layer]; const nextDeltas = this.deltas[layer + 1]; const nextWeights = this.weights[layer + 1]; const currentErrors = this.errors[layer]; const currentDeltas = this.deltas[layer]; for (let node = 0; node < currentSize; node++) { const output = currentOutputs[node]; let error = 0; if (layer === this.outputLayer) { error = target[node] - output; } else { for (let k = 0; k < nextDeltas.length; k++) { error += nextDeltas[k] * nextWeights[k][node]; } } currentErrors[node] = error; currentDeltas[node] = output > 0 ? error : alpha * error; } } } _calculateDeltasTanh(target: Float32Array): void { for (let layer = this.outputLayer; layer >= 0; layer--) { const currentSize = this.sizes[layer]; const currentOutputs = this.outputs[layer]; const nextDeltas = this.deltas[layer + 1]; const nextWeights = this.weights[layer + 1]; const currentErrors = this.errors[layer]; const currentDeltas = this.deltas[layer]; for (let node = 0; node < currentSize; node++) { const output = currentOutputs[node]; let error = 0; if (layer === this.outputLayer) { error = target[node] - output; } else { for (let k = 0; k < nextDeltas.length; k++) { error += nextDeltas[k] * nextWeights[k][node]; } } currentErrors[node] = error; currentDeltas[node] = (1 - output * output) * error; } } } /** * * Changes weights of networks */ adjustWeights(): void { const { learningRate, momentum } = this.trainOpts; for (let layer = 1; layer <= this.outputLayer; layer++) { const incoming = this.outputs[layer - 1]; const activeSize = this.sizes[layer]; const activeDelta = this.deltas[layer]; const activeChanges = this.changes[layer]; const activeWeights = this.weights[layer]; const activeBiases = this.biases[layer]; for (let node = 0; node < activeSize; node++) { const delta = activeDelta[node]; for (let k = 0; k < incoming.length; k++) { let change = activeChanges[node][k]; change = learningRate * delta * incoming[k] + momentum * change; activeChanges[node][k] = change; activeWeights[node][k] += change; } activeBiases[node] += learningRate * delta; } } } _setupAdam(): void { this.biasChangesLow = []; this.biasChangesHigh = []; this.changesLow = []; this.changesHigh = []; this.iterations = 0; for (let layer = 0; layer <= this.outputLayer; layer++) { const size = this.sizes[layer]; if (layer > 0) { this.biasChangesLow[layer] = zeros(size); this.biasChangesHigh[layer] = zeros(size); this.changesLow[layer] = new Array(size); this.changesHigh[layer] = new Array(size); for (let node = 0; node < size; node++) { const prevSize = this.sizes[layer - 1]; this.changesLow[layer][node] = zeros(prevSize); this.changesHigh[layer][node] = zeros(prevSize); } } } this.adjustWeights = this._adjustWeightsAdam; } _adjustWeightsAdam(): void { this.iterations++; const { iterations } = this; const { beta1, beta2, epsilon, learningRate } = this.trainOpts; for (let layer = 1; layer <= this.outputLayer; layer++) { const incoming = this.outputs[layer - 1]; const currentSize = this.sizes[layer]; const currentDeltas = this.deltas[layer]; const currentChangesLow = this.changesLow[layer]; const currentChangesHigh = this.changesHigh[layer]; const currentWeights = this.weights[layer]; const currentBiases = this.biases[layer]; const currentBiasChangesLow = this.biasChangesLow[layer]; const currentBiasChangesHigh = this.biasChangesHigh[layer]; for (let node = 0; node < currentSize; node++) { const delta = currentDeltas[node]; for (let k = 0; k < incoming.length; k++) { const gradient = delta * incoming[k]; const changeLow = currentChangesLow[node][k] * beta1 + (1 - beta1) * gradient; const changeHigh = currentChangesHigh[node][k] * beta2 + (1 - beta2) * gradient * gradient; const momentumCorrection = changeLow / (1 - Math.pow(beta1, iterations)); const gradientCorrection = changeHigh / (1 - Math.pow(beta2, iterations)); currentChangesLow[node][k] = changeLow; currentChangesHigh[node][k] = changeHigh; currentWeights[node][k] += (learningRate * momentumCorrection) / (Math.sqrt(gradientCorrection) + epsilon); } const biasGradient = currentDeltas[node]; const biasChangeLow = currentBiasChangesLow[node] * beta1 + (1 - beta1) * biasGradient; const biasChangeHigh = currentBiasChangesHigh[node] * beta2 + (1 - beta2) * biasGradient * biasGradient; const biasMomentumCorrection = currentBiasChangesLow[node] / (1 - Math.pow(beta1, iterations)); const biasGradientCorrection = currentBiasChangesHigh[node] / (1 - Math.pow(beta2, iterations)); currentBiasChangesLow[node] = biasChangeLow; currentBiasChangesHigh[node] = biasChangeHigh; currentBiases[node] += (learningRate * biasMomentumCorrection) / (Math.sqrt(biasGradientCorrection) + epsilon); } } } validateData(data: Array>): void { const inputSize = this.sizes[0]; const outputSize = this.sizes[this.sizes.length - 1]; const { length } = data; for (let i = 0; i < length; i++) { const { input, output } = data[i]; if (input.length !== inputSize) { throw new Error( `input at index ${i} length ${input.length} must be ${inputSize}` ); } if (data[i].output.length !== outputSize) { throw new Error( `output at index ${i} length ${output.length} must be ${outputSize}` ); } } } validateInput(formattedInput: Float32Array): void { const inputSize = this.sizes[0]; if (formattedInput.length !== inputSize) { throw new Error( `input length ${formattedInput.length} must match options.inputSize of ${inputSize}` ); } } formatData( data: Array> ): Array> { if (!Array.isArray(data[0].input)) { if (this.inputLookup) { this.inputLookupLength = Object.keys(this.inputLookup).length; } else { const inputLookup = new LookupTable(data, 'input'); this.inputLookup = inputLookup.table; this.inputLookupLength = inputLookup.length; } } if (!Array.isArray(data[0].output)) { if (this.outputLookup) { this.outputLookupLength = Object.keys(this.outputLookup).length; } else { const lookup = new LookupTable(data, 'output'); this.outputLookup = lookup.table; this.outputLookupLength = lookup.length; } } if (!this._formatInput) { this._formatInput = getTypedArrayFn(data[0].input, this.inputLookup); } if (!this._formatOutput) { this._formatOutput = getTypedArrayFn(data[0].output, this.outputLookup); } // turn sparse hash input into arrays with 0s as filler if (this._formatInput && this._formatOutput) { const result: Array> = []; for (let i = 0; i < data.length; i++) { result.push({ input: (this._formatInput as (v: INumberHash) => Float32Array)( (data[i].input as unknown) as INumberHash ), output: (this._formatOutput as (v: INumberHash) => Float32Array)( (data[i].output as unknown) as INumberHash ), }); } return result; } if (this._formatInput) { const result: Array> = []; for (let i = 0; i < data.length; i++) { result.push({ input: (this._formatInput as (v: INumberHash) => Float32Array)( (data[i].input as unknown) as INumberHash ), output: (data[i].output as unknown) as Float32Array, }); } return result; } if (this._formatOutput) { const result: Array> = []; for (let i = 0; i < data.length; i++) { result.push({ input: (data[i].input as unknown) as Float32Array, output: (this._formatOutput as (v: INumberHash) => Float32Array)( (data[i].output as unknown) as INumberHash ), }); } return result; } return (data as unknown) as Array< INeuralNetworkDatumFormatted >; } addFormat(data: INeuralNetworkDatum): void { if (!Array.isArray(data.input) || typeof data.input[0] !== 'number') { this.inputLookup = lookup.addKeys( (data.input as unknown) as INumberHash, this.inputLookup ?? {} ); if (this.inputLookup) { this.inputLookupLength = Object.keys(this.inputLookup).length; } } if (!Array.isArray(data.output) || typeof data.output[0] !== 'number') { this.outputLookup = lookup.addKeys( (data.output as unknown) as INumberHash, this.outputLookup ?? {} ); if (this.outputLookup) { this.outputLookupLength = Object.keys(this.outputLookup).length; } } } test( data: Array, Partial>> ): INeuralNetworkTestResult | INeuralNetworkBinaryTestResult { const { preparedData } = this.prepTraining( data as Array> ); // for binary classification problems with one output node const isBinary = preparedData[0].output.length === 1; // for classification problems const misclasses = []; // run each pattern through the trained network and collect // error and misclassification statistics let errorSum = 0; if (isBinary) { let falsePos = 0; let falseNeg = 0; let truePos = 0; let trueNeg = 0; for (let i = 0; i < preparedData.length; i++) { const output = this.runInput(preparedData[i].input); const target = preparedData[i].output; const actual = output[0] > this.options.binaryThresh ? 1 : 0; const expected = target[0]; if (actual !== expected) { const misclass = preparedData[i]; misclasses.push({ input: misclass.input, output: misclass.output, actual, expected, }); } if (actual === 0 && expected === 0) { trueNeg++; } else if (actual === 1 && expected === 1) { truePos++; } else if (actual === 0 && expected === 1) { falseNeg++; } else if (actual === 1 && expected === 0) { falsePos++; } errorSum += mse( output.map((value, i) => { return target[i] - value; }) ); } return { error: errorSum / preparedData.length, misclasses, total: preparedData.length, trueNeg, truePos, falseNeg, falsePos, precision: truePos > 0 ? truePos / (truePos + falsePos) : 0, recall: truePos > 0 ? truePos / (truePos + falseNeg) : 0, accuracy: (trueNeg + truePos) / preparedData.length, }; } for (let i = 0; i < preparedData.length; i++) { const output = this.runInput(preparedData[i].input); const target = preparedData[i].output; const actual = output.indexOf(max(output)); const expected = target.indexOf(max(target)); if (actual !== expected) { const misclass = preparedData[i]; misclasses.push({ input: misclass.input, output: misclass.output, actual, expected, }); } errorSum += mse( output.map((value, i) => { return target[i] - value; }) ); } return { error: errorSum / preparedData.length, misclasses, total: preparedData.length, }; } toJSON(): INeuralNetworkJSON { if (!this.isRunnable) { this.initialize(); } // use Array.from, keeping json small const jsonLayerWeights = this.weights.map((layerWeights) => { return layerWeights.map((layerWeights) => Array.from(layerWeights)); }); const jsonLayerBiases = this.biases.map((layerBiases) => Array.from(layerBiases) ); const jsonLayers: IJSONLayer[] = []; const outputLength = this.sizes.length - 1; for (let i = 0; i <= outputLength; i++) { jsonLayers.push({ weights: jsonLayerWeights[i] ?? [], biases: jsonLayerBiases[i] ?? [], }); } return { type: 'NeuralNetwork', sizes: [...this.sizes], layers: jsonLayers, inputLookup: this.inputLookup ? { ...this.inputLookup } : null, inputLookupLength: this.inputLookupLength, outputLookup: this.outputLookup ? { ...this.outputLookup } : null, outputLookupLength: this.outputLookupLength, options: { ...this.options }, trainOpts: this.getTrainOptsJSON(), }; } fromJSON(json: INeuralNetworkJSON): this { this.options = { ...defaults(), ...json.options }; if (json.hasOwnProperty('trainOpts')) { const trainOpts = { ...json.trainOpts, timeout: json.trainOpts.timeout === 'Infinity' ? Infinity : json.trainOpts.timeout, }; this.updateTrainingOptions(trainOpts); } this.sizes = json.sizes; this.initialize(); this.inputLookup = json.inputLookup ? { ...json.inputLookup } : null; this.inputLookupLength = json.inputLookupLength; this.outputLookup = json.outputLookup ? { ...json.outputLookup } : null; this.outputLookupLength = json.outputLookupLength; const jsonLayers = json.layers; const layerWeights = this.weights.map((layerWeights, layerIndex) => { return jsonLayers[layerIndex].weights.map((layerWeights) => Float32Array.from(layerWeights) ); }); const layerBiases = this.biases.map((layerBiases, layerIndex) => Float32Array.from(jsonLayers[layerIndex].biases) ); for (let i = 0; i <= this.outputLayer; i++) { this.weights[i] = layerWeights[i] || []; this.biases[i] = layerBiases[i] || []; } return this; } toFunction( cb?: (source: string) => string ): (input: Partial) => OutputType { const { activation, leakyReluAlpha } = this.trainOpts; let needsVar = false; const nodeHandle = (layerIndex: number, nodeIndex: number): string => { if (layerIndex === 0) { return `(input[${nodeIndex}]||0)`; } const weights: Float32Array = this.weights[layerIndex][nodeIndex]; const bias: number = this.biases[layerIndex][nodeIndex]; if (!weights) { throw new Error( `weights at layerIndex ${layerIndex} & nodeIndex ${nodeIndex} not found` ); } if (!bias) { throw new Error( `bias as layerIndex ${layerIndex} & nodeIndex ${nodeIndex} not found` ); } const weightsArray: string[] = []; weights.forEach((weight: number, subNodeIndex: number): void => { if (weight < 0) { weightsArray.push( `${weight}*${nodeHandle(layerIndex - 1, subNodeIndex)}` ); } else { weightsArray.push( `+${weight}*${nodeHandle(layerIndex - 1, subNodeIndex)}` ); } }); const result = `(${bias.toString()}${weightsArray.join('')})`; switch (activation) { case 'sigmoid': return `1/(1+1/Math.exp(${result}))`; case 'relu': { needsVar = true; return `((v=${result})<0?0:v)`; } case 'leaky-relu': { needsVar = true; return `Math.max((v=${result}),${leakyReluAlpha}*v)`; } case 'tanh': return `Math.tanh(${result})`; default: throw new Error( `Unknown activation ${activation}. Available activations are: 'sigmoid', 'relu', 'leaky-relu', 'tanh'` ); } }; function checkKeys(keys: string[]): void { if (keys.find((v) => v.includes('"'))) { throw new Error(`key contains '"', which is not compatible`); } } const layersAsMath: string[] = []; let result: string; let inputLookup = ''; if (this.inputLookup) { const keys = Object.keys(this.inputLookup); checkKeys(keys); inputLookup = `input = new Float32Array([${Object.keys(this.inputLookup) .map((key) => `input["${key}"]`) .join(',')}]);`; } if (this.sizes.length < 1) throw new Error('No layers'); for ( let nodeIndex = 0; nodeIndex < this.sizes[this.outputLayer]; nodeIndex++ ) { layersAsMath.push(nodeHandle(this.outputLayer, nodeIndex)); } if (this.outputLookup) { const keys = Object.keys(this.outputLookup); checkKeys(keys); const values = keys .map((key, i) => `"${key}":${layersAsMath[i]}`) .join(','); result = `{${values}}`; } else { result = `[${layersAsMath.join(',')}]`; } const source = `${inputLookup}${needsVar ? 'var v;' : ''}return ${result};`; // eslint-disable-next-line @typescript-eslint/no-implied-eval,no-new-func return new Function('input', cb ? cb(source) : source) as ( input: Partial ) => OutputType; } } ================================================ FILE: src/neural-network.unit.test.ts ================================================ import { getTypedArrayFn, NeuralNetwork } from './neural-network'; describe('NeuralNetwork', () => { describe('validateData', () => { describe('when an input is not same as options.inputSize', () => { it('throws', () => { expect(() => { new NeuralNetwork({ inputSize: 1, outputSize: 1 }).validateData([ { input: new Float32Array([1, 1]), output: new Float32Array([1]), }, ]); }).toThrow(); }); }); describe('when an output is not same as options.outputSize', () => { it('throws', () => { expect(() => { new NeuralNetwork({ inputSize: 1, outputSize: 1 }).validateData([ { input: new Float32Array([1]), output: new Float32Array([1, 1]), }, ]); }).toThrow(); }); }); describe('when an input is same as options.inputSize', () => { it('does not throw', () => { expect(() => { new NeuralNetwork({ inputSize: 1, outputSize: 1 }).validateData([ { input: new Float32Array([1]), output: new Float32Array([1]), }, ]); }).not.toThrow(); }); }); describe('when an output is same as options.outputSize', () => { it('does not throw', () => { expect(() => { new NeuralNetwork({ inputSize: 1, outputSize: 1 }).validateData([ { input: new Float32Array([1]), output: new Float32Array([1]), }, ]); }).not.toThrow(); }); }); }); describe('run', () => { describe('when input is not same as options.inputSize', () => { it('throws', () => { const net = new NeuralNetwork({ inputSize: 1, hiddenLayers: [1], outputSize: 1, }); net.train([{ input: [1], output: [1] }], { iterations: 1 }); expect(() => { net.run([1, 1]); }).toThrow(); }); }); describe('when input is same as options.inputSize', () => { it('throws', () => { const net = new NeuralNetwork({ inputSize: 1, hiddenLayers: [1], outputSize: 1, }); net.train([{ input: [1], output: [1] }], { iterations: 1 }); expect(() => { net.run([1]); }).not.toThrow(); }); }); }); describe('getTypedArrayFn', () => { it('can handle constructor', () => { // https://github.com/BrainJS/brain.js/issues/681 const fn = getTypedArrayFn({ constructor: 0 }, { constructor: 0 }); // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error expect(fn({ anything: 1 })[0]).toBe(0); }); }); }); ================================================ FILE: src/praxis/README.md ================================================ # [Praxis](https://en.wikipedia.org/wiki/Praxis_(process)) Models to assist in helping neural networks improve their abilities. ## Why the name? "Efficiency" is what is trying to be obtained, we could effectively call them "heuristic"s (probably the more technical name), but that'd be no fun to type. Too if we are targeting simplicity the very model, should not its name reflect that? with Here is a list of other projects and what they call their "heuristic" models: | Project Name | Praxis Synonym | Url | |--------------|---------------------|-----| | Caffe | Solvers | https://github.com/BVLC/caffe/tree/master/src/caffe/solvers | | Tensor | Estimator/Optimizer | https://github.com/tensorflow/tensorflow/tree/master/tensorflow/python/estimator | | torch | Optim | https://github.com/torch/optim | | Synaptic | Trainer | https://github.com/cazala/synaptic/blob/master/src/Trainer.js | | mlpack | Optimizer | https://github.com/mlpack/mlpack/tree/master/src/mlpack/core/optimizers | | Shogun | Optimization | https://github.com/shogun-toolbox/shogun/tree/develop/src/shogun/optimization | | Accord.net | Models | https://github.com/accord-net/framework/tree/master/Sources/Accord.Statistics/Models | | Brain.js | Praxis | | A praxis can be used on a layer as its means of learning like this: ```js import { Pool } from 'brain.js/layer'; import { MRmsProp } from 'brain.js/praxis'; new Pool({ praxis: (layer) => new MRmsProp(layer, { /* optional settings*/ }) }); ``` For layer and praxis shorthand helpers you can do: ```js import { pool } from 'brain.js/layer'; import { mRmsProp } from 'brain.js/praxis'; pool({ praxis: mRmsProp }); ``` A praxis can also be used with the `FeedForward` and planned `Recurrent` classes like this, which will cause all layers to inherit praxis: ```js import { input, pool, relu, output } from 'brain.js/layer'; import { mRmsProp } from 'brain.js/praxis'; new FeedForward({ praxis: mRmsProp, // defines for all layers, their praxis input: () => input(), hiddenLayers: [ (input) => pool({ praxis: mRmsProp }, input), // overrides network praxis (input) => relu(input) ], output: () => output() }) ``` ================================================ FILE: src/praxis/arthur-deviation-biases.end-to-end.test.ts ================================================ import { GPU } from 'gpu.js'; import { ArthurDeviationBiases } from './arthur-deviation-biases'; import { random } from '../layer'; import { NeuralNetwork } from '../neural-network'; import { setup, teardown } from '../utilities/kernel'; import { mockLayer, xorTrainingData } from '../test-utils'; describe('ArthurDeviationBiases', () => { beforeEach(() => { setup( new GPU({ mode: 'cpu', }) ); }); afterEach(() => { teardown(); }); describe('.run()', () => { test('correctly runs values', () => { const layer = mockLayer({ weights: [[1]], deltas: [[1]], width: 1, height: 1, }); const praxis = new ArthurDeviationBiases(layer); praxis.setupKernels(); const result = praxis.run(layer) as number[][]; expect(result[0][0].toFixed(5)).toEqual((1.3).toFixed(5).toString()); }); test('matches NeuralNetwork.adjustWeights output', () => { const net = new NeuralNetwork(); net.train(xorTrainingData, { iterations: 1, }); const layer1 = random({ id: 'biases', height: 3 }); const praxis = new ArthurDeviationBiases(layer1, { learningRate: net.trainOpts.learningRate, }); expect(praxis.settings.learningRate).toBe(net.trainOpts.learningRate); net.deltas[0][0] = 1; net.deltas[0][1] = 2; (layer1.deltas as number[][])[0][0] = net.deltas[1][0] = 3; (layer1.deltas as number[][])[1][0] = net.deltas[1][1] = 4; (layer1.deltas as number[][])[2][0] = net.deltas[1][2] = 5; net.deltas[2][0] = 6; (layer1.weights as number[][])[0][0] = net.biases[1][0] = 7; (layer1.weights as number[][])[1][0] = net.biases[1][1] = 8; (layer1.weights as number[][])[2][0] = net.biases[1][2] = 9; net.biases[2][0] = 10; net.adjustWeights(); praxis.setupKernels(); const result = praxis.run(layer1) as number[][]; expect(result[0][0]).not.toBe(0); expect(result[0][0]).toBe(net.biases[1][0]); expect(result[1][0]).not.toBe(0); expect(result[1][0]).toBe(net.biases[1][1]); expect(result[2][0]).not.toBe(0); expect(result[2][0]).toBe(net.biases[1][2]); }); }); }); ================================================ FILE: src/praxis/arthur-deviation-biases.ts ================================================ import { makeKernel } from '../utilities/kernel'; import { BasePraxis, IPraxisSettings } from './base-praxis'; import { ILayer } from '../layer/base-layer'; import { IKernelFunctionThis, IKernelRunShortcut, KernelOutput } from 'gpu.js'; export interface IUpdateThis extends IKernelFunctionThis { constants: { learningRate: number; }; } export function update( this: IUpdateThis, weights: number[][], deltas: number[][] ): number { return ( weights[this.thread.y][this.thread.x] + this.constants.learningRate * deltas[this.thread.y][this.thread.x] ); } export interface IArthurDeviationBiasesSettings extends IPraxisSettings { learningRate?: number; } export const defaultSettings = { learningRate: 0.3, }; export class ArthurDeviationBiases extends BasePraxis { settings: IArthurDeviationBiasesSettings; kernel: IKernelRunShortcut | null; constructor(layer: ILayer, settings?: IArthurDeviationBiasesSettings) { super(layer); this.settings = { ...defaultSettings, ...settings }; this.kernel = null; } run(layer: ILayer): KernelOutput { return (this.kernel as IKernelRunShortcut)(layer.weights, layer.deltas); } setupKernels(): void { this.kernel = makeKernel(update, { output: [this.width, this.height], constants: { learningRate: this.settings.learningRate ?? 0.01, }, }); } } export function arthurDeviationBiases( layer: ILayer, settings?: Partial ): ArthurDeviationBiases { return new ArthurDeviationBiases(layer, settings); } ================================================ FILE: src/praxis/arthur-deviation-biases.unit.test.ts ================================================ import { gpuMock } from 'gpu-mock.js'; import { GPU, IKernelRunShortcut } from 'gpu.js'; import { ArthurDeviationBiases, arthurDeviationBiases, update, } from '../praxis/arthur-deviation-biases'; import { setup, teardown } from '../utilities/kernel'; import { mockLayer, shave } from '../test-utils'; describe('ArthurDeviationBiases Class: Unit', () => { beforeEach(() => { setup( new GPU({ mode: 'cpu', }) ); }); afterEach(() => { teardown(); }); describe('update()', () => { it('performs math correctly', () => { const weights = [[1, 2, 3]]; const deltas = [[0.5, 0.4, 0.3]]; const width = 3; const height = 1; const kernel = gpuMock(update, { output: [width, height], constants: { learningRate: 0.5, momentum: 0.2, }, }); const result = kernel(weights, deltas) as Float32Array[]; const value = new Float32Array([1.25, 2.20000005, 3.1500001]); expect(shave(result[0])).toEqual(shave(value)); }); }); describe('.setupKernels()', () => { test('instantiates .kernel', () => { const layer = mockLayer({ width: 2, height: 2 }); const p = new ArthurDeviationBiases(layer); p.setupKernels(); expect(p.kernel).not.toBe(null); }); }); describe('.run()', () => { it('calls this.kernel() returns kernel output', () => { const mockWeights = 1; const mockDeltas = 2; const layer = mockLayer({ width: 2, height: 2, deltas: mockDeltas, weights: mockWeights, }); const p = new ArthurDeviationBiases(layer); interface I { kernel: IKernelRunShortcut; } p.setupKernels(); const kernelSpy = jest.spyOn(p as I, 'kernel'); const mockResult = [[1]]; kernelSpy.mockReturnValue(mockResult); const result = p.run(layer); expect(result).toBe(mockResult); expect(p.kernel).toHaveBeenCalledWith(mockWeights, mockDeltas); }); }); describe('arthurDeviationBiases lambda', () => { it('creates a new instance of ArthurDeviationBiases', () => { const layer = mockLayer({ width: 1, height: 1 }); const p = arthurDeviationBiases(layer); expect(p.layerTemplate).toBe(layer); }); }); }); ================================================ FILE: src/praxis/arthur-deviation-weights.end-to-end.test.ts ================================================ import { GPU } from 'gpu.js'; import { ArthurDeviationWeights } from './'; import { random } from '../layer'; import { NeuralNetwork } from '../neural-network'; import { setup, teardown } from '../utilities/kernel'; import { xorTrainingData } from '../test-utils'; describe('ArthurDeviationWeights Class: End to End', () => { beforeEach(() => { setup( new GPU({ mode: 'cpu', }) ); }); afterEach(() => { teardown(); }); describe('.run()', () => { test('correctly runs values', () => { const layer = { weights: [[1]], deltas: [[1]], width: 1, height: 1 }; // @ts-expect-error missing properties in layer const praxis = new ArthurDeviationWeights(layer, { weightsLayer: { weights: [[1]], deltas: [[1]], }, incomingLayer: { weights: [[1]], deltas: [[1]], }, deltaLayer: { weights: [[1]], deltas: [[1]], }, }); praxis.setupKernels(); const result = praxis.run() as number[][]; expect(result[0][0].toFixed(5)).toEqual((1.3).toFixed(5).toString()); }); }); test('matches NeuralNetwork.adjustWeights output', () => { const net = new NeuralNetwork(); net.train(xorTrainingData, { iterations: 1, }); const inputs = random({ id: 'input', height: 2 }); const weights = random({ id: 'weights', height: 3, width: 2 }); const biases = random({ id: 'biases', height: 3 }); const trainOpts = net.trainOpts; const praxis = new ArthurDeviationWeights(weights, { weightsLayer: weights, incomingLayer: inputs, deltaLayer: biases, learningRate: trainOpts.learningRate, }); expect(praxis.learningRate).toBe(trainOpts.learningRate); const inputWeights = inputs.weights as number[][]; const netOutputs = net.outputs; const netChanges = net.changes; const netWeights = net.weights; const netDeltas = net.deltas; const weightsWeights = weights.weights as number[][]; const biasesWeights = biases.weights as number[][]; const biasesDeltas = biases.deltas as number[][]; let praxisChanges = praxis.changes as number[][]; inputWeights[0][0] = netOutputs[0][0] = 11; inputWeights[1][0] = netOutputs[0][1] = 22; praxisChanges[0][0] = netChanges[1][0][0] = 1; praxisChanges[0][1] = netChanges[1][0][1] = 2; praxisChanges[1][0] = netChanges[1][1][0] = 3; praxisChanges[1][1] = netChanges[1][1][1] = 4; praxisChanges[2][0] = netChanges[1][2][0] = 5; praxisChanges[2][1] = netChanges[1][2][1] = 6; netChanges[2][0][0] = 7; netChanges[2][0][2] = 8; netChanges[2][0][3] = 9; weightsWeights[0][0] = netWeights[1][0][0] = 1; weightsWeights[0][1] = netWeights[1][0][1] = 2; weightsWeights[1][0] = netWeights[1][1][0] = 3; weightsWeights[1][1] = netWeights[1][1][1] = 4; weightsWeights[2][0] = netWeights[1][2][0] = 5; weightsWeights[2][1] = netWeights[1][2][1] = 6; biasesWeights[0][0] = netWeights[2][0][0] = 7; biasesWeights[1][0] = netWeights[2][0][1] = 8; biasesWeights[2][0] = netWeights[2][0][2] = 9; netDeltas[0][0] = 1; netDeltas[0][1] = 2; biasesDeltas[0][0] = netDeltas[1][0] = 3; biasesDeltas[1][0] = netDeltas[1][1] = 4; biasesDeltas[2][0] = netDeltas[1][2] = 5; netDeltas[2][0] = 6; net.adjustWeights(); praxis.setupKernels(); const result = praxis.run() as number[][]; praxisChanges = praxis.changes as number[][]; expect(praxisChanges[0][0]).toBe(netChanges[1][0][0]); expect(praxisChanges[0][1]).toBe(netChanges[1][0][1]); expect(praxisChanges[1][0]).toBe(netChanges[1][1][0]); expect(praxisChanges[1][1]).toBe(netChanges[1][1][1]); expect(praxisChanges[2][0]).toBe(netChanges[1][2][0]); expect(praxisChanges[2][1]).toBe(netChanges[1][2][1]); expect(result[0][0]).toBe(netWeights[1][0][0]); expect(result[0][1]).toBe(netWeights[1][0][1]); expect(result[1][0]).toBe(netWeights[1][1][0]); expect(result[1][1]).toBe(netWeights[1][1][1]); expect(result[2][0]).toBe(netWeights[1][2][0]); expect(result[2][1]).toBe(netWeights[1][2][1]); }); }); ================================================ FILE: src/praxis/arthur-deviation-weights.ts ================================================ import { makeKernelMap } from '../utilities/kernel'; import { zeros2D } from '../utilities/zeros-2d'; import { BasePraxis, IPraxisSettings } from './base-praxis'; import { ILayer } from '../layer/base-layer'; import { IConstantsThis, IKernelFunctionThis, IKernelMapRunShortcut, ISubKernelObject, ISubKernelsResults, KernelOutput, } from 'gpu.js'; export function updateChange(value: number): number { return value; } export interface IUpdateConstants extends IConstantsThis { learningRate: number; momentum: number; } export function update( this: IKernelFunctionThis, changes: number[][], weights: number[][], incomingWeights: number[][], inputDeltas: number[][] ): number { const lastChange: number = changes[this.thread.y][this.thread.x]; const inputDelta: number = inputDeltas[this.thread.y][0]; const weight: number = weights[this.thread.y][this.thread.x]; const incoming: number = incomingWeights[this.thread.x][0]; const change = this.constants.learningRate * inputDelta * incoming + this.constants.momentum * lastChange; updateChange(change); return weight + change; } export interface IArthurDeviationWeightsSettings extends IPraxisSettings { learningRate?: number; momentum?: number; weightsLayer?: ILayer | null; incomingLayer?: ILayer | null; deltaLayer?: ILayer | null; } export interface IKernelMapResults extends ISubKernelsResults { changes: KernelOutput; } export const defaultSettings: IArthurDeviationWeightsSettings = { learningRate: 0.3, momentum: 0.1, weightsLayer: null, incomingLayer: null, deltaLayer: null, }; export class ArthurDeviationWeights extends BasePraxis { changes: KernelOutput; kernelMap: IKernelMapRunShortcut | null = null; settings: IArthurDeviationWeightsSettings; get learningRate(): number { return this.settings.learningRate as number; } get momentum(): number { return this.settings.momentum as number; } get weightsLayer(): ILayer { return this.settings.weightsLayer as ILayer; } set weightsLayer(layer: ILayer) { this.settings.weightsLayer = layer; } get deltaLayer(): ILayer { return this.settings.deltaLayer as ILayer; } set deltaLayer(layer: ILayer) { this.settings.deltaLayer = layer; } get incomingLayer(): ILayer { return this.settings.incomingLayer as ILayer; } set incomingLayer(layer: ILayer) { this.settings.incomingLayer = layer; } constructor(layer: ILayer, settings?: IArthurDeviationWeightsSettings) { super(layer); this.settings = { ...defaultSettings, ...settings }; this.changes = zeros2D(layer.width, layer.height); } run(): KernelOutput { const output = (this.kernelMap as IKernelMapRunShortcut)( this.changes, this.weightsLayer.weights, this.incomingLayer.weights, this.deltaLayer.deltas ); this.changes = output.changes; return output.result; } setupKernels(): void { this.kernelMap = makeKernelMap, IUpdateConstants>( { changes: updateChange, }, update, { output: [this.width, this.height], constants: { learningRate: this.learningRate, momentum: this.momentum, }, } ); } } export function arthurDeviationWeights( layer: ILayer, settings?: Partial ): ArthurDeviationWeights { return new ArthurDeviationWeights(layer, settings); } ================================================ FILE: src/praxis/arthur-deviation-weights.unit.test.ts ================================================ import { GPU, IKernelRunShortcut } from 'gpu.js'; import { gpuMock } from 'gpu-mock.js'; import { ArthurDeviationWeights, arthurDeviationWeights, IArthurDeviationWeightsSettings, update, updateChange, } from './arthur-deviation-weights'; import { setup, teardown } from '../utilities/kernel'; import { mockLayer, shave, shave2D } from '../test-utils'; describe('ArthurDeviationWeights Class: Unit', () => { beforeEach(() => { setup( new GPU({ mode: 'cpu', }) ); }); afterEach(() => { teardown(); }); describe('update()', () => { it('performs math correctly', () => { const changes = [[1, 2, 3]]; const weights = [[1, 2, 3]]; const incomingWeights = [[1], [2], [3]]; const inputDeltas = [[1]]; const width = 3; const height = 1; const kernel: IKernelRunShortcut = gpuMock(update, { output: [width, height], constants: { learningRate: 0.5, momentum: 0.2, }, }); const result = kernel( changes, weights, incomingWeights, inputDeltas ) as Float32Array[]; // Corrected this array, need to recheck const value: Float32Array = new Float32Array([ 1.70000005, 3.4000001, 5.0999999, ]); expect(shave2D(result)).toEqual([shave(value)]); }); }); describe('updateChange()', () => { it('returns the value it is given', () => { const mockValue = 0; expect(updateChange(mockValue)).toBe(mockValue); }); }); describe('.constructor()', () => { describe('.changes', () => { it('gets dimensions from inputLayer', () => { const width = 2; const height = 3; const layer = mockLayer({ width, height }); const p = new ArthurDeviationWeights(layer); const changes = p.changes as number[][]; expect(changes.length).toBe(height); expect(changes[0].length).toBe(width); }); }); describe('.weightsLayer', () => { const layer = mockLayer({ width: 1, height: 1 }); const weightsLayer = mockLayer({}); const p = new ArthurDeviationWeights(layer, { weightsLayer, }); expect(p.weightsLayer).toBe(weightsLayer); }); describe('.incomingLayer', () => { const layer = mockLayer({ width: 1, height: 1 }); const incomingLayer = mockLayer({ width: 1, height: 1 }); const p = new ArthurDeviationWeights(layer, { incomingLayer, }); expect(p.incomingLayer).toBe(incomingLayer); }); describe('.deltaLayer', () => { const layer = mockLayer({ width: 1, height: 1 }); const deltaLayer = mockLayer({ width: 1, height: 1 }); const p = new ArthurDeviationWeights(layer, { deltaLayer, }); expect(p.deltaLayer).toBe(deltaLayer); }); }); describe('.run()', () => { it('calls this.kernel(), sets this.changes, and returns kernel output.results', () => { const layer = mockLayer({ width: 1, height: 1, weights: [[1]] }); const weightsLayer = mockLayer({ width: 1, height: 1, weights: [[2]] }); const incomingLayer = mockLayer({ width: 1, height: 1, weights: [[3]] }); const deltaLayer = mockLayer({ width: 1, height: 1, deltas: [[4]] }); const p: ArthurDeviationWeights = new ArthurDeviationWeights(layer, { weightsLayer, incomingLayer, deltaLayer, }); p.setupKernels(); const oldChanges = p.changes; const result = p.run(); expect(result).toBeInstanceOf(Array); expect(p.changes).not.toBe(oldChanges); }); }); describe('arthurDeviationWeights lambda', () => { it('creates a new instance of ArthurDeviationWeights', () => { const layer = mockLayer({ height: 1, width: 1 }); const weightsLayer = mockLayer({ height: 1, width: 1 }); const deltaLayer = mockLayer({ height: 1, width: 1 }); const incomingLayer = mockLayer({ height: 1, width: 1 }); const settings: Partial = { weightsLayer, deltaLayer, incomingLayer, }; const p = arthurDeviationWeights(layer, settings); expect(p.weightsLayer).toBe(weightsLayer); expect(p.deltaLayer).toBe(deltaLayer); // deltasLayer did not exist, so changed to deltaLayer expect(p.incomingLayer).toBe(incomingLayer); expect(p.layerTemplate).toBe(layer); }); }); }); ================================================ FILE: src/praxis/base-praxis.ts ================================================ import { ILayer } from '../layer'; import { IKernelRunShortcut, KernelOutput } from 'gpu.js'; export interface ILayerTemplate { width: number; height: number; depth: number; } export interface IPraxisJSON { width: number; height: number; depth: number; } export interface IPraxisSettings { width?: number; height?: number; depth?: number; kernel?: IKernelRunShortcut | null; } export interface IPraxis { layerTemplate: ILayerTemplate | null; kernel: IKernelRunShortcut | null; settings: Partial; setupKernels: () => void; width: number; height: number; depth: number; run: | ((layer: ILayer, learningRate: number) => KernelOutput) | ((layer: ILayer, learningRate?: number) => KernelOutput); toJSON: () => Partial; } export abstract class BasePraxis implements IPraxis { layerTemplate: ILayerTemplate; kernel: IKernelRunShortcut | null; settings: Partial; get width(): number { return this.layerTemplate.width; } get height(): number { return this.layerTemplate.height; } get depth(): number { return this.layerTemplate.depth; } constructor( layerTemplate: ILayerTemplate, settings: Partial = {} ) { this.layerTemplate = layerTemplate; this.settings = { ...settings }; this.kernel = null; } setupKernels(): void {} reuseKernels(praxis: IPraxis): void { if (praxis.width !== this.width) { throw new Error( `${this.constructor.name} kernel width mismatch ${praxis.width} is not ${this.width}` ); } if (praxis.height !== this.height) { throw new Error( `${this.constructor.name} kernel width mismatch ${praxis.height} is not ${this.height}` ); } if (praxis.hasOwnProperty('kernel')) { this.kernel = praxis.kernel; } } abstract run(layer: ILayer, learningRate?: number): KernelOutput; toJSON(): Partial { return { ...this.settings }; } } ================================================ FILE: src/praxis/index.ts ================================================ export { ArthurDeviationBiases, arthurDeviationBiases, } from './arthur-deviation-biases'; export { ArthurDeviationWeights, arthurDeviationWeights, } from './arthur-deviation-weights'; export { MomentumRootMeanSquaredPropagation, momentumRootMeanSquaredPropagation, MRmsProp, mRmsProp, } from './momentum-root-mean-squared-propagation'; ================================================ FILE: src/praxis/momentum-root-mean-squared-propagation.test.ts ================================================ import { GPU } from 'gpu.js'; import { MomentumRootMeanSquaredPropagation } from './momentum-root-mean-squared-propagation'; import { setup, teardown } from '../utilities/kernel'; import { mockLayer } from '../test-utils'; describe('MomentumRootMeanSquaredPropagation', () => { beforeEach(() => { setup( new GPU({ mode: 'cpu', }) ); }); afterEach(() => { teardown(); }); describe('.run()', () => { test('correctly runs values', () => { const layer = mockLayer({ weights: [[1]], deltas: [[1]], width: 1, height: 1, }); const praxis = new MomentumRootMeanSquaredPropagation(layer, { decayRate: 0.999, clipValue: 5, learningRate: 0.01, regularizationStrength: 0.000001, smoothEps: 1e-8, }); praxis.setupKernels(); const result = praxis.run(layer); expect((result as number[][])[0][0].toFixed(5)).toEqual( (0.68377).toString() ); }); test('correctly adjusts decayRate', () => { const layer = mockLayer({ weights: [[1]], deltas: [[1]], width: 1, height: 1, }); const praxis = new MomentumRootMeanSquaredPropagation(layer, { decayRate: 0.299, clipValue: 5, learningRate: 0.01, regularizationStrength: 0.000001, smoothEps: 1e-8, }); praxis.setupKernels(); const result = praxis.run(layer); expect((result as number[][])[0][0].toFixed(5)).toEqual( (0.98806).toString() ); }); }); }); ================================================ FILE: src/praxis/momentum-root-mean-squared-propagation.ts ================================================ import { BasePraxis, ILayerTemplate, IPraxisSettings } from './base-praxis'; import { makeKernelMap, release } from '../utilities/kernel'; import { zeros2D } from '../utilities/zeros-2d'; import { IConstantsThis, IKernelFunctionThis, IKernelMapRunShortcut, ISubKernelObject, KernelOutput, } from 'gpu.js'; import { ILayer } from '../layer'; export function getMomentum( delta: number, decay: number, previousMomentum: number ): number { return previousMomentum * decay + (1 - decay) * delta * delta; } export function clipByValue(value: number, max: number, min: number): number { if (value > max) { return max; } if (value < min) { return min; } return value; } interface IUpdate extends IConstantsThis { clipValue: number; decayRate: number; smoothEps: number; regularizationStrength: number; } /** * @description Momentum Root Mean Square Propagation Function */ export function update( this: IKernelFunctionThis, weights: number[][], deltas: number[][], previousMomenta: number[][] ): number { const delta = deltas[this.thread.y][this.thread.x]; const clippedDelta = clipByValue( delta, this.constants.clipValue, -this.constants.clipValue ); const weight = weights[this.thread.y][this.thread.x]; const previousMomentum = previousMomenta[this.thread.y][this.thread.x]; const momentum = getMomentum( delta, this.constants.decayRate, previousMomentum ); return ( weight + (-this.constants.learningRate * clippedDelta) / Math.sqrt(momentum + this.constants.smoothEps) - this.constants.regularizationStrength * weight ); } export function isClippedByValue( value: number, max: number, min: number ): number { if (value > max) { return 1; } if (value < min) { return 1; } return 0; } export interface IMomentumRootMeanSquaredPropagationSettings extends IPraxisSettings { decayRate?: number; regularizationStrength?: number; learningRate?: number; smoothEps: number; clipValue: number; } export const defaults: IMomentumRootMeanSquaredPropagationSettings = { decayRate: 0.999, regularizationStrength: 0.000001, learningRate: 0.01, smoothEps: 1e-8, clipValue: 5, }; export class MomentumRootMeanSquaredPropagation extends BasePraxis { momenta: KernelOutput; kernelMap: IKernelMapRunShortcut | null = null; settings: Partial; get clipValue(): number { return this.settings.clipValue as number; } get decayRate(): number { return this.settings.decayRate as number; } get learningRate(): number { return this.settings.learningRate as number; } get regularizationStrength(): number { return this.settings.regularizationStrength as number; } get smoothEps(): number { return this.settings.smoothEps as number; } constructor( layerTemplate: ILayerTemplate, settings: Partial = {} ) { super(layerTemplate); this.settings = { ...defaults, ...settings }; this.momenta = zeros2D(layerTemplate.width, layerTemplate.height); } run(layer: ILayer): KernelOutput { const { momenta, result } = (this.kernelMap as IKernelMapRunShortcut< ISubKernelObject >)(layer.weights, layer.deltas, this.momenta); release(this.momenta); this.momenta = momenta; return result; } setupKernels(): void { this.kernelMap = makeKernelMap( { momenta: getMomentum, }, update, { output: [this.width, this.height], constants: { clipValue: this.clipValue, decayRate: this.decayRate, learningRate: this.learningRate, regularizationStrength: this.regularizationStrength, smoothEps: this.smoothEps, }, functions: [clipByValue], immutable: true, } ); } } export function momentumRootMeanSquaredPropagation( layer: ILayer, settings: Partial ): MomentumRootMeanSquaredPropagation { return new MomentumRootMeanSquaredPropagation(layer, settings); } /** * @description Mathematician friendly name of MomentumRootMeanSquaredPropagation class. For those that are not mere mortals */ export const MRmsProp = MomentumRootMeanSquaredPropagation; export const mRmsProp = momentumRootMeanSquaredPropagation; ================================================ FILE: src/recurrent/gru-time-step.test.ts ================================================ import { RNNTimeStep } from './rnn-time-step'; import { GRUTimeStep } from './gru-time-step'; describe('GRUTimeStep', () => { describe('.getHiddenLayer()', () => { test('overrides RNNTimeStep', () => { expect(typeof GRUTimeStep.prototype.getHiddenLayer).toEqual('function'); expect(GRUTimeStep.prototype.getHiddenLayer).not.toEqual( RNNTimeStep.prototype.getHiddenLayer ); }); }); describe('.getEquation()', () => { test('overrides RNNTimeStep', () => { expect(typeof GRUTimeStep.prototype.getEquation).toEqual('function'); expect(GRUTimeStep.prototype.getEquation).not.toEqual( RNNTimeStep.prototype.getEquation ); }); }); }); ================================================ FILE: src/recurrent/gru-time-step.ts ================================================ import { getGRUHiddenLayer, getGRUEquation, IGRUHiddenLayer } from './gru'; import { Matrix } from './matrix'; import { Equation } from './matrix/equation'; import { RNNTimeStep } from './rnn-time-step'; import { IRNNHiddenLayer } from './rnn'; export class GRUTimeStep extends RNNTimeStep { getHiddenLayer(hiddenSize: number, prevSize: number): IRNNHiddenLayer { return getGRUHiddenLayer(hiddenSize, prevSize); } getEquation( equation: Equation, inputMatrix: Matrix, previousResult: Matrix, hiddenLayer: IRNNHiddenLayer ): Matrix { return getGRUEquation( equation, inputMatrix, previousResult, hiddenLayer as IGRUHiddenLayer ); } } ================================================ FILE: src/recurrent/gru.test.ts ================================================ import { GRU } from './gru'; import { IMatrixJSON } from './matrix'; import { RNN } from './rnn'; import { DataFormatter } from '../utilities/data-formatter'; describe('GRU', () => { describe('.getHiddenLayer()', () => { test('overrides RNN', () => { expect(typeof GRU.prototype.getHiddenLayer).toEqual('function'); expect(GRU.prototype.getHiddenLayer).not.toEqual( RNN.prototype.getHiddenLayer ); }); }); describe('.getEquation()', () => { test('overrides RNN', () => { expect(typeof GRU.prototype.getEquation).toEqual('function'); expect(GRU.prototype.getEquation).not.toEqual(RNN.prototype.getEquation); }); }); describe('math', () => { jest.retryTimes(5); it('can predict math', () => { const net = new GRU(); const items = new Set([]); for (let i = 0; i < 10; i++) { for (let j = 0; j < 10; j++) { items.add(`${i}+${j}=${i + j}`); items.add(`${j}+${i}=${i + j}`); } } net.train(Array.from(items), { iterations: 60, errorThresh: 0.03 }); for (let i = 0; i < 10; i++) { const output = net.run(`${i}+`); expect(/^[0-9]+[=][0-9]+$/.test(output)).toBe(true); } }); }); describe('printable characters', () => { jest.retryTimes(5); it('can learn a phrase', (done) => { const net = new GRU(); net.train( [ { input: 'hello world', output: 'comment', }, ], { iterations: 200 } ); const result = net.run('hello world'); expect(result).toBe('comment'); done(); }); it('can predict a phrase when given the first letter', (done) => { const phrase = 'bob'; const dataFormatter = new DataFormatter(['b', 'o']); const net = new GRU({ inputSize: 3, inputRange: dataFormatter.characters.length, outputSize: 3, dataFormatter, }); net.initialize(); for (let i = 0; i < 200; i++) { net.trainPattern(dataFormatter.toIndexes(phrase)); // if (i % 10 === 0) { // console.log(dataFormatter.toCharacters(net.run()).join('')); // } } const result = net.run(dataFormatter.toIndexes('b')); expect(result).toBe('ob'); done(); }); it('can learn a phrase, export it to a function, and it still runs', (done) => { const phrase = 'hello world;|something I comment about'; const dataFormatter = DataFormatter.fromString(phrase); const phraseAsIndices = dataFormatter.toIndexes(phrase); const net = new GRU({ inputSize: 40, inputRange: dataFormatter.characters.length, outputSize: 40, dataFormatter, }); net.initialize(); for (let i = 0; i < 200; i++) { net.trainPattern(phraseAsIndices); // if (i % 10 === 0) { // console.log(dataFormatter.toCharacters(net.run()).join('')); // } } expect(net.run()).toBe(phrase); done(); }); }); describe('json', () => { describe('.toJSON', () => { it('can export model as json', () => { const net = new GRU({ inputSize: 6, inputRange: 12, outputSize: 6, }); const json = net.toJSON(); function compare(left: IMatrixJSON, right: IMatrixJSON) { left.weights.forEach((value, i) => { expect(value).toBe(right.weights[i]); }); expect(left.rows).toBe(right.rows); expect(left.columns).toBe(right.columns); } compare(json.input, net.model.input.toJSON()); net.model.hiddenLayers.forEach((layer, i) => { for (const p in layer) { if (!layer.hasOwnProperty(p)) continue; compare(json.hiddenLayers[i][p], layer[p].toJSON()); } }); compare(json.output, net.model.output.toJSON()); compare(json.outputConnector, net.model.outputConnector.toJSON()); }); }); describe('.fromJSON', () => { it('can import model from json', () => { const dataFormatter = new DataFormatter('abcdef'.split('')); const jsonString = JSON.stringify( new GRU({ inputSize: 6, // <- length inputRange: dataFormatter.characters.length, outputSize: dataFormatter.characters.length, // <- length }).toJSON() ); const clone = new GRU(); clone.fromJSON(JSON.parse(jsonString)); expect(jsonString).toEqual(JSON.stringify(clone.toJSON())); expect(clone.options.inputSize).toEqual(6); expect(clone.options.inputRange).toEqual( dataFormatter.characters.length ); expect(clone.options.outputSize).toEqual( dataFormatter.characters.length ); }); it('can import model from json and train again', () => { const dataFormatter = new DataFormatter('abcdef'.split('')); const jsonString = JSON.stringify( new GRU({ inputSize: dataFormatter.characters.length, // <- length inputRange: dataFormatter.characters.length, outputSize: dataFormatter.characters.length, // <- length dataFormatter, }).toJSON() ); const clone = new GRU(); clone.fromJSON(JSON.parse(jsonString)); clone.trainPattern([0, 1, 2, 3, 4, 5]); expect(jsonString).not.toEqual(JSON.stringify(clone.toJSON())); expect(clone.options.inputSize).toEqual(7); // 6 + also unrecognized expect(clone.options.inputRange).toEqual( dataFormatter.characters.length ); expect(clone.options.outputSize).toEqual( dataFormatter.characters.length ); }); }); }); describe('.toFunction', () => { it('can output same as run method', () => { const dataFormatter = new DataFormatter(['h', 'i', ' ', 'm', 'o', '!']); const net = new GRU({ inputSize: 6, inputRange: dataFormatter.characters.length, outputSize: 6, dataFormatter, }); net.initialize(); for (let i = 0; i < 200; i++) { net.trainPattern(dataFormatter.toIndexes('hi mom!')); // if (i % 10) { // console.log(dataFormatter.toCharacters(net.run()).join('')); // } } const lastOutput = net.run(); const fn = net.toFunction(); expect(fn()).toBe(lastOutput); }); it('can include the DataFormatter', () => { const net = new GRU(); net.train(['hi mom!'], { iterations: 200 }); const expected = net.run('hi '); const newNet = net.toFunction(); const output = newNet('hi '); expect(output).toBe(expected); }); }); }); ================================================ FILE: src/recurrent/gru.ts ================================================ import { Matrix } from './matrix'; import { Equation } from './matrix/equation'; import { RandomMatrix } from './matrix/random-matrix'; import { IRNNHiddenLayer, RNN } from './rnn'; export interface IGRUHiddenLayer extends IRNNHiddenLayer { updateGateInputMatrix: RandomMatrix; updateGateHiddenMatrix: RandomMatrix; updateGateBias: Matrix; resetGateInputMatrix: RandomMatrix; resetGateHiddenMatrix: RandomMatrix; resetGateBias: Matrix; cellWriteInputMatrix: RandomMatrix; cellWriteHiddenMatrix: RandomMatrix; cellWriteBias: Matrix; } export class GRU extends RNN { getHiddenLayer(hiddenSize: number, prevSize: number): IRNNHiddenLayer { return getGRUHiddenLayer(hiddenSize, prevSize); } getEquation( equation: Equation, inputMatrix: Matrix, previousResult: Matrix, hiddenLayer: IRNNHiddenLayer ): Matrix { return getGRUEquation( equation, inputMatrix, previousResult, hiddenLayer as IGRUHiddenLayer ); } } export function getGRUHiddenLayer( hiddenSize: number, prevSize: number ): IGRUHiddenLayer { return { // update Gate // wzxh updateGateInputMatrix: new RandomMatrix(hiddenSize, prevSize, 0.08), // wzhh updateGateHiddenMatrix: new RandomMatrix(hiddenSize, hiddenSize, 0.08), // bz updateGateBias: new Matrix(hiddenSize, 1), // reset Gate // wrxh resetGateInputMatrix: new RandomMatrix(hiddenSize, prevSize, 0.08), // wrhh resetGateHiddenMatrix: new RandomMatrix(hiddenSize, hiddenSize, 0.08), // br resetGateBias: new Matrix(hiddenSize, 1), // cell write parameters // wcxh cellWriteInputMatrix: new RandomMatrix(hiddenSize, prevSize, 0.08), // wchh cellWriteHiddenMatrix: new RandomMatrix(hiddenSize, hiddenSize, 0.08), // bc cellWriteBias: new Matrix(hiddenSize, 1), }; } export function getGRUEquation( equation: Equation, inputMatrix: Matrix, previousResult: Matrix, hiddenLayer: IGRUHiddenLayer ): Matrix { if ( !hiddenLayer.updateGateInputMatrix || !hiddenLayer.updateGateHiddenMatrix || !hiddenLayer.updateGateBias || !hiddenLayer.resetGateInputMatrix || !hiddenLayer.resetGateHiddenMatrix || !hiddenLayer.resetGateBias || !hiddenLayer.cellWriteInputMatrix || !hiddenLayer.cellWriteHiddenMatrix || !hiddenLayer.cellWriteBias ) { throw new Error('hiddenLayer does not have expected properties'); } const sigmoid = equation.sigmoid.bind(equation); const add = equation.add.bind(equation); const multiply = equation.multiply.bind(equation); const multiplyElement = equation.multiplyElement.bind(equation); const tanh = equation.tanh.bind(equation); const allOnes = equation.allOnes.bind(equation); const cloneNegative = equation.cloneNegative.bind(equation); // update gate const updateGate = sigmoid( add( add( multiply(hiddenLayer.updateGateInputMatrix, inputMatrix), multiply(hiddenLayer.updateGateHiddenMatrix, previousResult) ), hiddenLayer.updateGateBias ) ); // reset gate const resetGate = sigmoid( add( add( multiply(hiddenLayer.resetGateInputMatrix, inputMatrix), multiply(hiddenLayer.resetGateHiddenMatrix, previousResult) ), hiddenLayer.resetGateBias ) ); // cell const cell = tanh( add( add( multiply(hiddenLayer.cellWriteInputMatrix, inputMatrix), multiply( hiddenLayer.cellWriteHiddenMatrix, multiplyElement(resetGate, previousResult) ) ), hiddenLayer.cellWriteBias ) ); // compute hidden state as gated, saturated cell activations // negate updateGate return add( multiplyElement( add( allOnes(updateGate.rows, updateGate.columns), cloneNegative(updateGate) ), cell ), multiplyElement(previousResult, updateGate) ); } ================================================ FILE: src/recurrent/lstm-time-step.end-to-end.test.ts ================================================ import { LSTMTimeStep } from './lstm-time-step'; jest.retryTimes(5); describe('LSTMTimeStep', () => { it('can learn xor', () => { const net = new LSTMTimeStep({ inputSize: 1, hiddenLayers: [10], outputSize: 1, }); const xorNetValues = [ [[0.001], [0.001], [0.001]], [[0.001], [1], [1]], [[1], [0.001], [1]], [[1], [1], [0.001]], ]; const errorThresh = 0.03; const iterations = 5000; const status = net.train(xorNetValues, { iterations, errorThresh }); expect( status.error <= errorThresh || status.iterations <= iterations ).toBeTruthy(); expect(net.run([[0.001], [0.001]])[0]).toBeLessThan(0.1); expect(net.run([[0.001], [1]])[0]).toBeGreaterThan(0.9); expect(net.run([[1], [0.001]])[0]).toBeGreaterThan(0.9); expect(net.run([[1], [1]])[0]).toBeLessThan(0.1); }); }); ================================================ FILE: src/recurrent/lstm-time-step.test.ts ================================================ import { RNNTimeStep } from './rnn-time-step'; import { LSTMTimeStep } from './lstm-time-step'; import { getHiddenLSTMLayer, getLSTMEquation, ILSTMHiddenLayer } from './lstm'; import { Matrix } from './matrix'; import { Equation } from './matrix/equation'; jest.mock('./matrix/random-matrix', () => { class MockRandomMatrix { get rows(): number { return this.realMatrix.rows; } get columns(): number { return this.realMatrix.columns; } get weights(): Float32Array { return this.realMatrix.weights; } set weights(weights: Float32Array) { this.realMatrix.weights = weights; } get deltas(): Float32Array { return this.realMatrix.weights; } set deltas(deltas: Float32Array) { this.realMatrix.deltas = deltas; } get setWeight(): (row: number, column: number, value: number) => void { return this.realMatrix.setWeight; } get getWeight(): (row: number, column: number) => number { return this.realMatrix.getWeight; } get setDelta(): (row: number, column: number, value: number) => void { return this.realMatrix.setDelta; } get getDelta(): (row: number, column: number) => number { return this.realMatrix.getDelta; } realMatrix: Matrix; constructor(rows: number, columns: number, std: number) { this.realMatrix = new Matrix(rows, columns); let value = 1; this.realMatrix.iterate({ column: (rowIndex, columnIndex) => { this.setWeight(rowIndex, columnIndex, value++); }, }); } } return { RandomMatrix: MockRandomMatrix, }; }); describe('LSTMTimeStep', () => { describe('.getHiddenLayer()', () => { test('overrides RNNTimeStep', () => { expect(typeof LSTMTimeStep.prototype.getHiddenLayer).toEqual('function'); expect(LSTMTimeStep.prototype.getHiddenLayer).not.toEqual( RNNTimeStep.prototype.getHiddenLayer ); }); }); describe('.getEquation()', () => { test('overrides RNNTimeStep', () => { expect(typeof LSTMTimeStep.prototype.getEquation).toEqual('function'); expect(LSTMTimeStep.prototype.getEquation).not.toEqual( RNNTimeStep.prototype.getEquation ); }); }); describe('.getLSTMEquation()', () => { let hiddenLayer: ILSTMHiddenLayer; beforeEach(() => { hiddenLayer = getHiddenLSTMLayer(3, 3); }); it('correctly computes a hidden state', () => { const equation = new Equation(); const inputMatrix = new Matrix(3, 1); inputMatrix.setWeight(0, 0, 1); inputMatrix.setWeight(1, 0, 2); inputMatrix.setWeight(2, 0, 3); const previousResult = new Matrix(3, 1); previousResult.setWeight(0, 0, 1); previousResult.setWeight(1, 0, 2); previousResult.setWeight(2, 0, 3); equation.input(new Matrix(3, 1)); const lstmEquation = getLSTMEquation( equation, inputMatrix, previousResult, hiddenLayer ); expect(lstmEquation).toBeInstanceOf(Matrix); const result = equation.runInput(new Float32Array([0, 0, 0])); expect(result.getWeight(0, 0)).toBeCloseTo(0.9640275835990906); expect(result.getWeight(1, 0)).toBeCloseTo(0.9950547814369202); expect(result.getWeight(2, 0)).toBeCloseTo(0.9993293285369873); expect(equation.states.length).toBe(26); expect(equation.states[0].forwardFn.name).toBe('forwardFn'); // input gate expect(equation.states[1].forwardFn.name).toBe('multiply'); expect(equation.states[2].forwardFn.name).toBe('multiply'); expect(equation.states[3].forwardFn.name).toBe('add'); expect(equation.states[4].forwardFn.name).toBe('add'); expect(equation.states[5].forwardFn.name).toBe('sigmoid'); // forget gate expect(equation.states[6].forwardFn.name).toBe('multiply'); expect(equation.states[7].forwardFn.name).toBe('multiply'); expect(equation.states[8].forwardFn.name).toBe('add'); expect(equation.states[9].forwardFn.name).toBe('add'); expect(equation.states[10].forwardFn.name).toBe('sigmoid'); // output gate expect(equation.states[11].forwardFn.name).toBe('multiply'); expect(equation.states[12].forwardFn.name).toBe('multiply'); expect(equation.states[13].forwardFn.name).toBe('add'); expect(equation.states[14].forwardFn.name).toBe('add'); expect(equation.states[15].forwardFn.name).toBe('sigmoid'); // cell write expect(equation.states[16].forwardFn.name).toBe('multiply'); expect(equation.states[17].forwardFn.name).toBe('multiply'); expect(equation.states[18].forwardFn.name).toBe('add'); expect(equation.states[19].forwardFn.name).toBe('add'); expect(equation.states[20].forwardFn.name).toBe('tanh'); // new activation expect(equation.states[21].forwardFn.name).toBe('multiplyElement'); expect(equation.states[22].forwardFn.name).toBe('multiplyElement'); expect(equation.states[23].forwardFn.name).toBe('add'); expect(equation.states[24].forwardFn.name).toBe('tanh'); expect(equation.states[25].forwardFn.name).toBe('multiplyElement'); }); }); }); ================================================ FILE: src/recurrent/lstm-time-step.ts ================================================ import { getHiddenLSTMLayer, getLSTMEquation, ILSTMHiddenLayer } from './lstm'; import { Matrix } from './matrix'; import { Equation } from './matrix/equation'; import { RNNTimeStep } from './rnn-time-step'; import { IRNNHiddenLayer } from './rnn'; export class LSTMTimeStep extends RNNTimeStep { getHiddenLayer(hiddenSize: number, prevSize: number): IRNNHiddenLayer { return getHiddenLSTMLayer(hiddenSize, prevSize); } getEquation( equation: Equation, inputMatrix: Matrix, previousResult: Matrix, hiddenLayer: IRNNHiddenLayer ): Matrix { return getLSTMEquation( equation, inputMatrix, previousResult, hiddenLayer as ILSTMHiddenLayer ); } } ================================================ FILE: src/recurrent/lstm.test.ts ================================================ import { LSTM } from './lstm'; import { IMatrixJSON, Matrix } from './matrix'; import { RNN } from './rnn'; import { DataFormatter } from '../utilities/data-formatter'; describe('LSTM', () => { describe('.getHiddenLayer()', () => { test('overrides RNN', () => { expect(typeof LSTM.prototype.getHiddenLayer).toEqual('function'); expect(LSTM.prototype.getHiddenLayer).not.toEqual( RNN.prototype.getHiddenLayer ); }); }); describe('getEquation', () => { test('overrides RNN', () => { expect(typeof LSTM.prototype.getEquation).toEqual('function'); expect(LSTM.prototype.getEquation).not.toEqual(RNN.prototype.getEquation); }); }); describe('math', () => { it('can predict math', () => { const net = new LSTM(); const items = new Set([]); for (let i = 0; i < 10; i++) { for (let j = 0; j < 10; j++) { items.add(`${i}+${j}=${i + j}`); items.add(`${j}+${i}=${i + j}`); } } net.train(Array.from(items), { iterations: 60, errorThresh: 0.03 }); for (let i = 0; i < 10; i++) { const output = net.run(`${i}+`); expect(/^[0-9]+[=][0-9]+$/.test(output)).toBe(true); } }); }); describe('json', () => { describe('.toJSON', () => { it('can export model as json', () => { const net = new LSTM({ inputSize: 6, inputRange: 12, outputSize: 6, }); const json = net.toJSON(); function compare(left: IMatrixJSON, right: Matrix) { left.weights.forEach((value, i) => { expect(value).toBe(right.weights[i]); }); expect(left.rows).toBe(right.rows); expect(left.columns).toBe(right.columns); } compare(json.input, net.model.input); net.model.hiddenLayers.forEach((layer, i) => { for (const p in layer) { compare(json.hiddenLayers[i][p], layer[p]); } }); compare(json.output, net.model.output); compare(json.outputConnector, net.model.outputConnector); }); }); describe('.fromJSON', () => { it('can import model from json', () => { const dataFormatter = new DataFormatter('abcdef'.split('')); const json = new LSTM({ inputSize: 6, // <- length inputRange: dataFormatter.characters.length, outputSize: dataFormatter.characters.length, // <- length }).toJSON(); const clone = new LSTM(); clone.fromJSON(JSON.parse(JSON.stringify(json))); expect(json).toEqual(clone.toJSON()); expect(clone.options.inputSize).toBe(6); expect(clone.options.inputRange).toBe(dataFormatter.characters.length); expect(clone.options.outputSize).toBe(dataFormatter.characters.length); }); it('can train imported model from json', () => { const dataFormatter = new DataFormatter('abcdef'.split('')); const json = new LSTM({ inputSize: 6, // <- length inputRange: dataFormatter.characters.length, outputSize: dataFormatter.characters.length, // <- length }).toJSON(); const clone = new LSTM(); clone.fromJSON(JSON.parse(JSON.stringify(json))); clone.trainPattern([0, 1, 2, 3, 4, 5]); expect(json).not.toEqual(clone.toJSON()); expect(clone.options.inputSize).toBe(6); expect(clone.options.inputRange).toBe(dataFormatter.characters.length); expect(clone.options.outputSize).toBe(dataFormatter.characters.length); }); }); }); describe('.toFunction', () => { it('can output same as run method', () => { const dataFormatter = new DataFormatter(['h', 'i', ' ', 'm', 'o', '!']); const net = new LSTM({ inputSize: 1, inputRange: dataFormatter.characters.length, outputSize: 1, dataFormatter, }); net.initialize(); for (let i = 0; i < 100; i++) { net.trainPattern(dataFormatter.toIndexes('hi mom!')); } const lastOutput = net.run(); expect(lastOutput).toBe('hi mom!'); }); it('can include the DataFormatter', () => { const net = new LSTM(); net.train(['hi mom!'], { iterations: 100 }); const expected = net.run('hi '); const newNet = net.toFunction(); const output = newNet('hi '); expect(output).toBe(expected); }); }); describe('.run', () => { jest.retryTimes(5); it('can predict greetings in 200 trainings', () => { const net = new LSTM(); const trainingData = [ { input: 'hi', output: 'mom', }, { input: 'howdy', output: 'dad', }, { input: 'hello', output: 'sis', }, { input: 'yo', output: 'bro', }, ]; net.train(trainingData, { iterations: 200 }); expect(net.run('hi')).toBe('mom'); expect(net.run('howdy')).toBe('dad'); expect(net.run('hello')).toBe('sis'); expect(net.run('yo')).toBe('bro'); }); it('can predict a string from index in 200 trainings', () => { const net = new LSTM(); const transactionTypes = { credit: '0', debit: '1', personalCard: '2', other: '3', }; const trainingData = [ { input: transactionTypes.credit, output: 'credit', }, { input: transactionTypes.debit, output: 'debit', }, { input: transactionTypes.personalCard, output: 'personal card', }, { input: transactionTypes.other, output: 'other', }, ]; net.train(trainingData, { iterations: 200 }); expect(net.run([transactionTypes.credit])).toBe('credit'); expect(net.run([transactionTypes.debit])).toBe('debit'); expect(net.run([transactionTypes.personalCard])).toBe('personal card'); expect(net.run([transactionTypes.other])).toBe('other'); }); }); }); ================================================ FILE: src/recurrent/lstm.ts ================================================ import { Matrix } from './matrix'; import { Equation } from './matrix/equation'; import { RandomMatrix } from './matrix/random-matrix'; import { IRNNHiddenLayer, RNN } from './rnn'; export interface ILSTMHiddenLayer extends IRNNHiddenLayer { inputMatrix: Matrix; inputHidden: Matrix; inputBias: Matrix; forgetMatrix: Matrix; forgetHidden: Matrix; forgetBias: Matrix; outputMatrix: Matrix; outputHidden: Matrix; outputBias: Matrix; cellActivationMatrix: Matrix; cellActivationHidden: Matrix; cellActivationBias: Matrix; } export class LSTM extends RNN { getHiddenLayer(hiddenSize: number, prevSize: number): IRNNHiddenLayer { return getHiddenLSTMLayer(hiddenSize, prevSize); } getEquation( equation: Equation, inputMatrix: Matrix, previousResult: Matrix, hiddenLayer: IRNNHiddenLayer ): Matrix { return getLSTMEquation( equation, inputMatrix, previousResult, hiddenLayer as ILSTMHiddenLayer ); } } export function getHiddenLSTMLayer( hiddenSize: number, prevSize: number ): ILSTMHiddenLayer { return { // gates parameters // wix inputMatrix: new RandomMatrix(hiddenSize, prevSize, 0.08), // wih inputHidden: new RandomMatrix(hiddenSize, hiddenSize, 0.08), // bi inputBias: new Matrix(hiddenSize, 1), // wfx forgetMatrix: new RandomMatrix(hiddenSize, prevSize, 0.08), // wfh forgetHidden: new RandomMatrix(hiddenSize, hiddenSize, 0.08), // bf forgetBias: new Matrix(hiddenSize, 1), // wox outputMatrix: new RandomMatrix(hiddenSize, prevSize, 0.08), // woh outputHidden: new RandomMatrix(hiddenSize, hiddenSize, 0.08), // bo outputBias: new Matrix(hiddenSize, 1), // cell write params // wcx cellActivationMatrix: new RandomMatrix(hiddenSize, prevSize, 0.08), // wch cellActivationHidden: new RandomMatrix(hiddenSize, hiddenSize, 0.08), // bc cellActivationBias: new Matrix(hiddenSize, 1), }; } export function getLSTMEquation( equation: Equation, inputMatrix: Matrix, previousResult: Matrix, hiddenLayer: ILSTMHiddenLayer ): Matrix { if ( !hiddenLayer.inputMatrix || !hiddenLayer.inputHidden || !hiddenLayer.inputBias || !hiddenLayer.forgetMatrix || !hiddenLayer.forgetHidden || !hiddenLayer.forgetBias || !hiddenLayer.outputMatrix || !hiddenLayer.outputHidden || !hiddenLayer.outputBias || !hiddenLayer.cellActivationMatrix || !hiddenLayer.cellActivationHidden || !hiddenLayer.cellActivationBias ) { throw new Error('hiddenLayer does not have expected properties'); } const sigmoid = equation.sigmoid.bind(equation); const add = equation.add.bind(equation); const multiply = equation.multiply.bind(equation); const multiplyElement = equation.multiplyElement.bind(equation); const tanh = equation.tanh.bind(equation); const inputGate = sigmoid( add( add( multiply(hiddenLayer.inputMatrix, inputMatrix), multiply(hiddenLayer.inputHidden, previousResult) ), hiddenLayer.inputBias ) ); const forgetGate = sigmoid( add( add( multiply(hiddenLayer.forgetMatrix, inputMatrix), multiply(hiddenLayer.forgetHidden, previousResult) ), hiddenLayer.forgetBias ) ); // output gate const outputGate = sigmoid( add( add( multiply(hiddenLayer.outputMatrix, inputMatrix), multiply(hiddenLayer.outputHidden, previousResult) ), hiddenLayer.outputBias ) ); // write operation on cells const cellWrite = tanh( add( add( multiply(hiddenLayer.cellActivationMatrix, inputMatrix), multiply(hiddenLayer.cellActivationHidden, previousResult) ), hiddenLayer.cellActivationBias ) ); // compute new cell activation const retainCell = multiplyElement(forgetGate, previousResult); // what do we keep from cell const writeCell = multiplyElement(inputGate, cellWrite); // what do we write to cell const cell = add(retainCell, writeCell); // new cell contents // compute hidden state as gated, saturated cell activations return multiplyElement(outputGate, tanh(cell)); } ================================================ FILE: src/recurrent/matrix/add-b.ts ================================================ import { Matrix } from '.'; /** * adds {from} deltas to {left} and {right} deltas */ export function addB(product: Matrix, left: Matrix, right: Matrix): void { for (let i = 0; i < product.deltas.length; i++) { left.deltas[i] = product.deltas[i]; right.deltas[i] = product.deltas[i]; } } ================================================ FILE: src/recurrent/matrix/add.ts ================================================ import { Matrix } from '.'; /** * add {left} and {right} matrix weights into {into} */ export function add(product: Matrix, left: Matrix, right: Matrix): void { for (let i = 0; i < left.weights.length; i++) { product.weights[i] = left.weights[i] + right.weights[i]; product.deltas[i] = 0; } } ================================================ FILE: src/recurrent/matrix/all-ones.ts ================================================ import { Matrix } from '.'; /** * makes matrix weights and deltas all ones */ export function allOnes(product: Matrix): void { for (let i = 0; i < product.weights.length; i++) { product.weights[i] = 1; product.deltas[i] = 0; } } ================================================ FILE: src/recurrent/matrix/clone-negative.ts ================================================ import { Matrix } from '.'; export function cloneNegative(product: Matrix, left: Matrix): void { product.rows = left.rows; product.columns = left.columns; product.weights = left.weights.slice(0); product.deltas = left.deltas.slice(0); for (let i = 0; i < left.weights.length; i++) { product.weights[i] = -left.weights[i]; product.deltas[i] = 0; } } ================================================ FILE: src/recurrent/matrix/clone.ts ================================================ import { Matrix } from '.'; function clone(product: Matrix): Matrix { const cloned = new Matrix(); cloned.rows = product.rows; cloned.columns = product.columns; cloned.weights = product.weights.slice(0); cloned.deltas = product.deltas.slice(0); return cloned; } export default clone; ================================================ FILE: src/recurrent/matrix/copy.ts ================================================ import { Matrix } from '.'; export function copy(product: Matrix, left: Matrix): void { product.rows = left.rows; product.columns = left.columns; product.weights = left.weights.slice(0); product.deltas = left.deltas.slice(0); } ================================================ FILE: src/recurrent/matrix/equation.test.ts ================================================ import { Matrix } from './'; import { OnesMatrix } from './ones-matrix'; import { Equation, IState } from './equation'; function fourSquareMatrix(value: number): Matrix { const result = new Matrix(4, 4); result.weights.forEach((_, i) => { result.weights[i] = value; }); return result; } function getState(props: Partial): IState { return { ...props, name: 'test-state', product: new Matrix(1, 1), left: new Matrix(1, 1), right: new Matrix(1, 1), forwardFn: jest.fn( (left: Matrix, right: Matrix, index: number): void => {} ), backpropagationFn: jest.fn( (left: Matrix, right: Matrix, index: number): void => {} ), }; } describe('equation', () => { describe('run', () => { it('calls all forwardFn properties', () => { const equation = new Equation(); for (let i = 0; i < 10; i++) { equation.states.push(getState({ name: `test-state-${i}` })); } equation.runIndex(); equation.states.forEach((state) => { expect(state.forwardFn).toBeCalled(); }); }); }); describe('runBack', () => { it('calls all forwardFn properties', () => { const equation = new Equation(); for (let i = 0; i < 10; i++) { equation.states.push(getState({ name: `test-state-${i}` })); } equation.backpropagate(); equation.states.forEach((state) => { expect(state.backpropagationFn).toBeCalled(); }); }); }); describe('add', () => { it('calls forwardFn', () => { const equation = new Equation(); const input = fourSquareMatrix(1); equation.add(input, fourSquareMatrix(1)); expect(equation.states.length).toBe(1); jest.spyOn(equation.states[0], 'forwardFn'); equation.runIndex(); expect(equation.states[0].forwardFn).toBeCalled(); }); }); describe('multiply', () => { it('calls forwardFn', () => { const equation = new Equation(); const input = fourSquareMatrix(1); equation.multiply(input, fourSquareMatrix(1)); expect(equation.states.length).toBe(1); jest.spyOn(equation.states[0], 'forwardFn'); equation.runIndex(); expect(equation.states[0].forwardFn).toBeCalled(); }); }); describe('multiplyElement', () => { it('calls forwardFn', () => { const equation = new Equation(); const input = fourSquareMatrix(1); equation.add(input, fourSquareMatrix(1)); expect(equation.states.length).toBe(1); jest.spyOn(equation.states[0], 'forwardFn'); equation.runIndex(); expect(equation.states[0].forwardFn).toBeCalled(); }); }); describe('relu', () => { it('calls forwardFn', () => { const equation = new Equation(); const input = fourSquareMatrix(1); equation.add(input, fourSquareMatrix(1)); expect(equation.states.length).toBe(1); jest.spyOn(equation.states[0], 'forwardFn'); equation.runIndex(); expect(equation.states[0].forwardFn).toBeCalled(); }); }); describe('inputMatrixToRow', () => { it('calls forwardFn', () => { const equation = new Equation(); const input = fourSquareMatrix(1); equation.add(input, fourSquareMatrix(1)); expect(equation.states.length).toBe(1); jest.spyOn(equation.states[0], 'forwardFn'); equation.runIndex(); expect(equation.states[0].forwardFn).toBeCalled(); }); }); describe('sigmoid', () => { it('calls forwardFn', () => { const equation = new Equation(); const input = fourSquareMatrix(1); equation.add(input, fourSquareMatrix(1)); expect(equation.states.length).toBe(1); jest.spyOn(equation.states[0], 'forwardFn'); equation.runIndex(); expect(equation.states[0].forwardFn).toBeCalled(); }); }); describe('tanh', () => { it('calls forwardFn', () => { const equation = new Equation(); const input = fourSquareMatrix(1); equation.add(input, fourSquareMatrix(1)); expect(equation.states.length).toBe(1); jest.spyOn(equation.states[0], 'forwardFn'); equation.runIndex(); expect(equation.states[0].forwardFn).toBeCalled(); }); }); describe('nesting', () => { it('can nest 3 deep and run forward', () => { const equation = new Equation(); const input = fourSquareMatrix(2); equation.multiply( equation.multiply( equation.multiply(input, fourSquareMatrix(2)), fourSquareMatrix(2) ), fourSquareMatrix(2) ); expect(equation.states.length).toBe(3); jest.spyOn(equation.states[0], 'forwardFn'); jest.spyOn(equation.states[1], 'forwardFn'); jest.spyOn(equation.states[2], 'forwardFn'); equation.runIndex(); equation.states.forEach((state) => { expect(state.forwardFn).toBeCalled(); }); }); it('can nest 3 deep and run backward', () => { const equation = new Equation(); const input = fourSquareMatrix(2); equation.tanh( equation.multiply( equation.add(input, fourSquareMatrix(2)), fourSquareMatrix(2) ) ); expect(equation.states.length).toBe(3); jest.spyOn(equation.states[0], 'backpropagationFn'); jest.spyOn(equation.states[1], 'backpropagationFn'); jest.spyOn(equation.states[2], 'backpropagationFn'); equation.backpropagate(); equation.states.forEach((state) => { expect(state.backpropagationFn).toBeCalled(); }); }); }); describe('inputMatrixToRow', () => { describe('runIndex', () => { it('can properly split up a matrix', () => { const input = new Matrix(2, 2); /** * Matrix like: * 1 1 * 2 2 */ input.weights.forEach((w, i) => { if (i < 2) { input.weights[i] = 1; } else { input.weights[i] = 2; } }); const equation = new Equation(); equation.add(new OnesMatrix(1, 2), equation.inputMatrixToRow(input)); let output = equation.runIndex(); expect(output.weights.length).toBe(2); expect(output.weights[0]).toBe(2); expect(output.weights[1]).toBe(2); output = equation.runIndex(1); expect(output.weights.length).toBe(2); expect(output.weights[0]).toBe(3); expect(output.weights[1]).toBe(3); }); }); describe('.backpropagate()', () => { it('can properly split up a matrix', () => { const input = new Matrix(2, 2); /** * Matrix like: * 1 1 * 2 2 */ input.weights.forEach((w, i) => { if (i < 2) { input.weights[i] = 1; } else { input.weights[i] = 2; } }); const equation = new Equation(); equation.add(new OnesMatrix(1, 2), equation.inputMatrixToRow(input)); let output = equation.runIndex(); expect(output.weights.length).toBe(2); output = equation.runIndex(1); expect(output.weights.length).toBe(2); }); }); }); }); ================================================ FILE: src/recurrent/matrix/equation.ts ================================================ import { Matrix } from '.'; import { add } from './add'; import { addB } from './add-b'; import { allOnes } from './all-ones'; import { cloneNegative } from './clone-negative'; import { multiply } from './multiply'; import { multiplyB } from './multiply-b'; import { multiplyElement } from './multiply-element'; import { multiplyElementB } from './multiply-element-b'; import { relu } from './relu'; import { reluB } from './relu-b'; import { rowPluck } from './row-pluck'; import { rowPluckB } from './row-pluck-b'; import { sigmoid } from './sigmoid'; import { sigmoidB } from './sigmoid-b'; import { softmax } from './softmax'; import { tanh } from './tanh'; import { tanhB } from './tanh-b'; type PropagateIndex = (product: Matrix, left: Matrix, index: number) => void; type PropagateProduct = (product: Matrix) => void; type PropagateProductFromLeft = (product: Matrix, left: Matrix) => void; type PropagateProductFromLeftRight = ( product: Matrix, left: Matrix, right: Matrix ) => void; type PropagateFunction = | PropagateIndex | PropagateProduct | PropagateProductFromLeft | PropagateProductFromLeftRight; export interface IState { name: string; product: Matrix; left?: Matrix; right?: Matrix; forwardFn: PropagateFunction; backpropagationFn: PropagateFunction; } export class Equation { states: IState[] = []; inputValue?: Float32Array; inputRow = 0; add(left: Matrix, right: Matrix): Matrix { if (left.weights.length !== right.weights.length) { throw new Error('misaligned matrices'); } const product = new Matrix(left.rows, left.columns); this.states.push({ name: 'add', product, left, right, forwardFn: add, backpropagationFn: addB, }); return product; } allOnes(rows: number, columns: number): Matrix { const product = new Matrix(rows, columns); this.states.push({ name: 'allOnes', product, left: product, forwardFn: allOnes, backpropagationFn: () => {}, }); return product; } cloneNegative(matrix: Matrix): Matrix { const product = new Matrix(matrix.rows, matrix.columns); this.states.push({ name: 'cloneNegative', product, left: matrix, forwardFn: cloneNegative, backpropagationFn: () => {}, }); return product; } /** * connects two matrices together by subtract */ subtract(left: Matrix, right: Matrix): Matrix { if (left.weights.length !== right.weights.length) { throw new Error('misaligned matrices'); } return this.add( this.add(this.allOnes(left.rows, left.columns), this.cloneNegative(left)), right ); } /** * connects two matrices together by multiply */ multiply(left: Matrix, right: Matrix): Matrix { if (left.columns !== right.rows) { throw new Error('misaligned matrices'); } const product = new Matrix(left.rows, right.columns); this.states.push({ name: 'multiply', product, left, right, forwardFn: multiply, backpropagationFn: multiplyB, }); return product; } /** * connects two matrices together by multiplyElement */ multiplyElement(left: Matrix, right: Matrix): Matrix { if (left.weights.length !== right.weights.length) { throw new Error('misaligned matrices'); } const product = new Matrix(left.rows, left.columns); this.states.push({ name: 'multiplyElement', product, left, right, forwardFn: multiplyElement, backpropagationFn: multiplyElementB, }); return product; } /** * connects a matrix to relu */ relu(matrix: Matrix): Matrix { const product = new Matrix(matrix.rows, matrix.columns); this.states.push({ name: 'relu', product, left: matrix, forwardFn: relu, backpropagationFn: reluB, }); return product; } /** * input a matrix */ input(input: Matrix): Matrix { this.states.push({ name: 'input', product: input, forwardFn: (product: Matrix) => { if (!this.inputValue) return; if (this.inputValue.length !== product.weights.length) { throw new Error('this.inputValue is of wrong dimensions'); } product.weights = input.weights = this.inputValue; }, backpropagationFn: () => {}, }); return input; } /** * connects a matrix via a row */ inputMatrixToRow(matrix: Matrix): Matrix { // eslint-disable-next-line @typescript-eslint/no-this-alias const self = this; const product = new Matrix(matrix.columns, 1); this.states.push({ name: 'inputMatrixToRow', product, left: matrix, get right() { return (self.inputRow as unknown) as Matrix; }, forwardFn: rowPluck, backpropagationFn: rowPluckB, }); return product; } /** * connects a matrix to sigmoid */ sigmoid(matrix: Matrix): Matrix { const product = new Matrix(matrix.rows, matrix.columns); this.states.push({ name: 'sigmoid', product, left: matrix, forwardFn: sigmoid, backpropagationFn: sigmoidB, }); return product; } /** * connects a matrix to tanh */ tanh(matrix: Matrix): Matrix { const product = new Matrix(matrix.rows, matrix.columns); this.states.push({ name: 'tanh', product, left: matrix, forwardFn: tanh, backpropagationFn: tanhB, }); return product; } /** * * Observe a matrix for debugging */ observe(matrix: Matrix): Matrix { this.states.push({ name: 'observe', product: new Matrix(), forwardFn: () => {}, backpropagationFn: () => {}, }); return matrix; } /** * Run index through equations via forward propagation */ runIndex(rowIndex = 0): Matrix { this.inputRow = rowIndex; let state = this.states[0]; for (let i = 0, max = this.states.length; i < max; i++) { state = this.states[i]; if (!state.hasOwnProperty('forwardFn')) continue; (state.forwardFn as PropagateProductFromLeftRight)( state.product, state.left as Matrix, state.right as Matrix ); } return state.product; } /** * Run value through equations via forward propagation */ runInput(inputValue: Float32Array): Matrix { this.inputValue = inputValue; let state = this.states[0]; for (let i = 0, max = this.states.length; i < max; i++) { state = this.states[i]; if (!state.hasOwnProperty('forwardFn')) continue; (state.forwardFn as PropagateProductFromLeftRight)( state.product, state.left as Matrix, state.right as Matrix ); } return state.product; } /** * Run value through equations via back propagation */ backpropagate(): Matrix { let i = this.states.length; let state = this.states[0]; while (i-- > 0) { state = this.states[i]; if (!state.hasOwnProperty('backpropagationFn')) continue; (state.backpropagationFn as PropagateProductFromLeftRight)( state.product, state.left as Matrix, state.right as Matrix ); } return state.product; } /** * Run index through equations via back propagation */ backpropagateIndex(rowIndex = 0): Matrix { this.inputRow = rowIndex; let i = this.states.length; let state = this.states[0]; while (i-- > 0) { state = this.states[i]; if (!state.hasOwnProperty('backpropagationFn')) continue; (state.backpropagationFn as PropagateProductFromLeftRight)( state.product, state.left as Matrix, state.right as Matrix ); } return state.product; } /** * Predict a target value from equation */ predictTarget(input: Float32Array, target: Float32Array): number { let errorSum = 0; const output = this.runInput(input); for (let i = 0; i < output.weights.length; i++) { const error = output.weights[i] - target[i]; // set gradients into log probabilities errorSum += Math.abs(error); // write gradients into log probabilities output.deltas[i] = error; } return errorSum; } /** * Predict a target index from equation */ predictTargetIndex(input: number, target: number): number { const output = this.runIndex(input); // set gradients into log probabilities const logProbabilities = output; // interpret output as log probabilities const probabilities = softmax(output); // compute the softmax probabilities // write gradients into log probabilities logProbabilities.deltas = probabilities.weights.slice(0); logProbabilities.deltas[target] -= 1; // accumulate base 2 log prob and do smoothing return -Math.log2(probabilities.weights[target]); } } ================================================ FILE: src/recurrent/matrix/index.test.ts ================================================ import { Matrix } from './'; describe('Matrix', () => { describe('.constructor()', () => { describe('.rows', () => { it('leaves rows as 0 if falsey', () => { expect(new Matrix(undefined, 2).rows).toBe(0); }); it('sets rows if truthy', () => { expect(new Matrix(1, 2).rows).toBe(1); }); }); describe('.columns', () => { it('leaves columns as 0 if falsey', () => { expect(new Matrix(1, undefined).columns).toBe(0); }); it('sets columns if truthy', () => { expect(new Matrix(1, 2).columns).toBe(2); }); }); describe('.weights', () => { it('sets .weights length from rows * columns size', () => { expect(new Matrix(33, 55).weights.length).toBe(33 * 55); }); it('sets .weights value from rows * columns of zeros', () => { expect(new Matrix(3, 3).weights).toEqual(new Float32Array(3 * 3)); }); }); describe('.deltas', () => { it('sets .deltas from rows * columns size', () => { expect(new Matrix(33, 55).deltas.length).toBe(33 * 55); }); it('sets .weights from rows * columns size', () => { expect(new Matrix(3, 3).deltas).toEqual(new Float32Array(3 * 3)); }); }); }); describe('.getWeight()', () => { it('throws if index greater than Matrix', () => { const matrix = new Matrix(2, 2); expect(() => { matrix.getWeight(2, 2); }).toThrow('get accessor is skewed'); }); it('throws if index less than Matrix', () => { const matrix = new Matrix(2, 2); expect(() => { matrix.getWeight(-2, -2); }).toThrow('get accessor is skewed'); }); it('returns individual weight', () => { const matrix = new Matrix(2, 2); matrix.weights[0] = 1; matrix.weights[1] = 2; matrix.weights[2] = 3; matrix.weights[3] = 4; expect(matrix.getWeight(0, 0)).toBe(1); expect(matrix.getWeight(0, 1)).toBe(2); expect(matrix.getWeight(1, 0)).toBe(3); expect(matrix.getWeight(1, 1)).toBe(4); }); }); describe('.setWeight()', () => { it('throws if index greater than Matrix', () => { const matrix = new Matrix(2, 2); expect(() => { matrix.setWeight(2, 2, 0); }).toThrow('set accessor is skewed'); }); it('throws if index less than Matrix', () => { const matrix = new Matrix(2, 2); expect(() => { matrix.setWeight(-2, -2, 0); }).toThrow('set accessor is skewed'); }); it('sets individual weight', () => { const matrix = new Matrix(2, 2); matrix.setWeight(0, 0, 1); matrix.setWeight(0, 1, 2); matrix.setWeight(1, 0, 3); matrix.setWeight(1, 1, 4); expect(matrix.weights[0]).toBe(1); expect(matrix.weights[1]).toBe(2); expect(matrix.weights[2]).toBe(3); expect(matrix.weights[3]).toBe(4); }); it('returns itself', () => { const matrix = new Matrix(1, 1); expect(matrix.setWeight(0, 0, 0)).toBe(matrix); }); }); describe('.getDelta()', () => { it('throws if index greater than Matrix', () => { const matrix = new Matrix(2, 2); expect(() => { matrix.getDelta(2, 2); }).toThrow('get accessor is skewed'); }); it('throws if index less than Matrix', () => { const matrix = new Matrix(2, 2); expect(() => { matrix.getDelta(-2, -2); }).toThrow('get accessor is skewed'); }); it('returns individual weight', () => { const matrix = new Matrix(2, 2); matrix.deltas[0] = 1; matrix.deltas[1] = 2; matrix.deltas[2] = 3; matrix.deltas[3] = 4; expect(matrix.getDelta(0, 0)).toBe(1); expect(matrix.getDelta(0, 1)).toBe(2); expect(matrix.getDelta(1, 0)).toBe(3); expect(matrix.getDelta(1, 1)).toBe(4); }); }); describe('.setDelta()', () => { it('throws if index greater than Matrix', () => { const matrix = new Matrix(2, 2); expect(() => { matrix.setDelta(2, 2, 0); }).toThrow('set accessor is skewed'); }); it('throws if index less than Matrix', () => { const matrix = new Matrix(2, 2); expect(() => { matrix.setDelta(-2, -2, 0); }).toThrow('set accessor is skewed'); }); it('sets individual weight', () => { const matrix = new Matrix(2, 2); matrix.setDelta(0, 0, 1); matrix.setDelta(0, 1, 2); matrix.setDelta(1, 0, 3); matrix.setDelta(1, 1, 4); expect(matrix.deltas[0]).toBe(1); expect(matrix.deltas[1]).toBe(2); expect(matrix.deltas[2]).toBe(3); expect(matrix.deltas[3]).toBe(4); }); it('returns itself', () => { const matrix = new Matrix(1, 1); expect(matrix.setDelta(0, 0, 0)).toBe(matrix); }); }); describe('.toJSON()', () => { it('serializes with rows, columns, and weights', () => { const matrix = new Matrix(3, 3); let value = 1; for (let row = 0; row < 3; row++) { for (let column = 0; column < 3; column++) { matrix.setWeight(row, column, value++); } } expect(matrix.toJSON()).toEqual({ rows: 3, columns: 3, weights: [1, 2, 3, 4, 5, 6, 7, 8, 9], }); }); }); describe('.fromJSON()', () => { it('deserializes to Matrix from json', () => { const json = { rows: 3, columns: 3, weights: [1, 2, 3, 4, 5, 6, 7, 8, 9], }; const matrix = Matrix.fromJSON(json); expect(matrix.rows).toBe(json.rows); expect(matrix.columns).toBe(json.columns); expect(matrix.weights).toEqual( new Float32Array([1, 2, 3, 4, 5, 6, 7, 8, 9]) ); }); }); describe('static .fromArray()', () => { it('deserializes to Matrix from nested array, populating weights', () => { const nestedArray = [ [1, 2, 3], [4, 5, 6], ]; const matrix = Matrix.fromArray(nestedArray); expect(matrix.rows).toBe(2); expect(matrix.columns).toBe(3); expect(matrix.weights).toEqual(Float32Array.from([1, 2, 3, 4, 5, 6])); }); }); describe('.toArray()', () => { it('deserializes to Matrix from nested array, populating weights by default', () => { const matrix = new Matrix(2, 3); matrix.setWeight(0, 0, 1); matrix.setWeight(0, 1, 2); matrix.setWeight(0, 2, 3); matrix.setWeight(1, 0, 4); matrix.setWeight(1, 1, 5); matrix.setWeight(1, 2, 6); expect(matrix.toArray()).toEqual([ [1, 2, 3], [4, 5, 6], ]); }); }); describe('.deltasToArray()', () => { it('return deltas', () => { const matrix = new Matrix(1, 1); matrix.deltas[0] = 999; expect(matrix.deltasToArray()).toEqual([[999]]); }); }); describe('.weightsToArray()', () => { it('return deltas', () => { const matrix = new Matrix(1, 1); matrix.weights[0] = 999; expect(matrix.weightsToArray()).toEqual([[999]]); }); }); describe('.fromArray()', () => { it('deserializes to Matrix from nested array, populating weights by default', () => { const nestedArray = [ [1, 2, 3], [4, 5, 6], ]; const matrix = new Matrix(2, 3).fromArray(nestedArray); expect(matrix.rows).toBe(2); expect(matrix.columns).toBe(3); expect(matrix.weights).toEqual(Float32Array.from([1, 2, 3, 4, 5, 6])); }); it('deserializes to Matrix from nested array, populating deltas when defined', () => { const nestedArray = [ [1, 2, 3], [4, 5, 6], ]; const matrix = new Matrix(2, 3).fromArray(nestedArray, 'deltas'); expect(matrix.rows).toBe(2); expect(matrix.columns).toBe(3); expect(matrix.weights).toEqual(Float32Array.from([0, 0, 0, 0, 0, 0])); expect(matrix.deltas).toEqual(Float32Array.from([1, 2, 3, 4, 5, 6])); }); }); describe('.iterate()', () => { it('iterates and calls for .column()', () => { const column = jest.fn(); new Matrix(1, 2).iterate({ column }); expect(column).toHaveBeenCalledWith(0, 0); expect(column).toHaveBeenCalledWith(0, 1); }); it('iterates and calls for .row()', () => { const row = jest.fn(); new Matrix(2, 1).iterate({ row }); expect(row).toHaveBeenCalledWith(0); expect(row).toHaveBeenCalledWith(1); }); it('returns this', () => { const matrix = new Matrix(2, 1); expect(matrix.iterate({ row: () => {} })).toBe(matrix); }); }); }); ================================================ FILE: src/recurrent/matrix/index.ts ================================================ import { zeros } from '../../utilities/zeros'; export interface IMatrixJSON { rows: number; columns: number; weights: number[]; } /** * A matrix */ export class Matrix { rows = 0; columns = 0; weights: Float32Array; deltas: Float32Array; constructor(rows?: number, columns?: number) { if (rows) this.rows = rows; if (columns) this.columns = columns; this.weights = zeros(this.rows * this.columns); this.deltas = zeros(this.rows * this.columns); } getWeight(row: number, col: number): number { // slow but careful accessor function // we want row-major order const ix = this.columns * row + col; if (ix < 0 || ix >= this.weights.length) { throw new Error('get accessor is skewed'); } return this.weights[ix]; } setWeight(row: number, col: number, v: number): Matrix { // slow but careful accessor function const ix = this.columns * row + col; if (ix < 0 || ix >= this.weights.length) { throw new Error('set accessor is skewed'); } this.weights[ix] = v; return this; } getDelta(row: number, col: number): number { // slow but careful accessor function // we want row-major order const ix = this.columns * row + col; if (ix < 0 || ix >= this.deltas.length) { throw new Error('get accessor is skewed'); } return this.deltas[ix]; } setDelta(row: number, col: number, v: number): Matrix { // slow but careful accessor function const ix = this.columns * row + col; if (ix < 0 || ix >= this.weights.length) { throw new Error('set accessor is skewed'); } this.deltas[ix] = v; return this; } toJSON(): IMatrixJSON { return { rows: this.rows, columns: this.columns, weights: Array.from(this.weights.slice(0)), }; } static fromJSON(json: IMatrixJSON): Matrix { const matrix = new Matrix(json.rows, json.columns); for (let i = 0, max = json.rows * json.columns; i < max; i++) { matrix.weights[i] = json.weights[i]; // copy over weights } return matrix; } static fromArray(weights: Float32Array[] | number[][]): Matrix { const matrix = new Matrix(weights.length, weights[0].length); matrix.fromArray(weights); return matrix; } deltasToArray(): number[][] { return this.toArray('deltas'); } weightsToArray(): number[][] { return this.toArray('weights'); } toArray(prop: 'weights' | 'deltas' = 'weights'): number[][] { const result: number[][] = new Array(this.rows); this.iterate({ row: (rowIndex): void => { result[rowIndex] = new Array(this.columns); }, column: (rowIndex, columnIndex): void => { if (prop === 'weights') { result[rowIndex][columnIndex] = this.getWeight(rowIndex, columnIndex); } else if (prop === 'deltas') { result[rowIndex][columnIndex] = this.getDelta(rowIndex, columnIndex); } }, }); return result; } fromArray( array: number[][] | Float32Array[], prop: 'weights' | 'deltas' = 'weights' ): this { if (array.length !== this.rows) { throw new Error('rows do not match'); } if (array[0].length !== this.columns) { throw new Error('columns do not match'); } this.iterate({ column: (rowIndex, columnIndex): void => { const value = array[rowIndex][columnIndex]; if (typeof value !== 'number') { throw new Error('value not number'); } if (prop === 'weights') { this.setWeight(rowIndex, columnIndex, value); } else if (prop === 'deltas') { this.setDelta(rowIndex, columnIndex, value); } }, }); return this; } iterate(callbacks: { column?: (rowIndex: number, columnIndex: number) => void; row?: (rowIndex: number) => void; }): this { const rows = this.rows; const columns = this.columns; for (let rowIndex = 0; rowIndex < rows; rowIndex++) { if (callbacks.row) { callbacks.row(rowIndex); } for (let columnIndex = 0; columnIndex < columns; columnIndex++) { if (callbacks.column) { callbacks.column(rowIndex, columnIndex); } } } return this; } } ================================================ FILE: src/recurrent/matrix/max-i.ts ================================================ import { Matrix } from '.'; export function maxI(matrix: Matrix): number { // argmax of array w const { weights } = matrix; let maxv = weights[0]; let maxix = 0; for (let i = 1; i < weights.length; i++) { const v = weights[i]; if (v < maxv) continue; maxix = i; maxv = v; } return maxix; } ================================================ FILE: src/recurrent/matrix/multiply-b.ts ================================================ import { Matrix } from '.'; /** * multiplies {from} deltas to {left} and {right} */ export function multiplyB(product: Matrix, left: Matrix, right: Matrix): void { const leftRows = left.rows; const leftColumns = left.columns; const rightColumns = right.columns; // loop over rows of left for (let leftRowRoot = 0; leftRowRoot < leftRows; leftRowRoot++) { const leftRowBase = leftColumns * leftRowRoot; const rightRowBase = rightColumns * leftRowRoot; // loop over cols of right for (let rightColumn = 0; rightColumn < rightColumns; rightColumn++) { // loop over columns of left for (let leftColumn = 0; leftColumn < leftColumns; leftColumn++) { const rightColumnBase = rightColumns * leftColumn; const leftRow = leftRowBase + leftColumn; const rightRow = rightColumnBase + rightColumn; const backPropagateValue = product.deltas[rightRowBase + rightColumn]; left.deltas[leftRow] += right.weights[rightRow] * backPropagateValue; right.deltas[rightRow] += left.weights[leftRow] * backPropagateValue; } } } } ================================================ FILE: src/recurrent/matrix/multiply-element-b.ts ================================================ import { Matrix } from '.'; /** * multiplies {left} and {right} weight by {from} deltas into {left} and {right} deltas */ export function multiplyElementB( product: Matrix, left: Matrix, right: Matrix ): void { for (let i = 0; i < left.weights.length; i++) { left.deltas[i] = right.weights[i] * product.deltas[i]; right.deltas[i] = left.weights[i] * product.deltas[i]; } } ================================================ FILE: src/recurrent/matrix/multiply-element.ts ================================================ import { Matrix } from '.'; export function multiplyElement( product: Matrix, left: Matrix, right: Matrix ): void { const { weights } = left; for (let i = 0; i < weights.length; i++) { product.weights[i] = left.weights[i] * right.weights[i]; product.deltas[i] = 0; } } ================================================ FILE: src/recurrent/matrix/multiply.ts ================================================ import { Matrix } from '.'; /** * multiply {left} and {right} matrix weights to {into} */ export function multiply(product: Matrix, left: Matrix, right: Matrix): void { const leftRows = left.rows; const leftColumns = left.columns; const rightColumns = right.columns; // loop over rows of left for (let leftRow = 0; leftRow < leftRows; leftRow++) { const leftRowBase = leftColumns * leftRow; const rightRowBase = rightColumns * leftRow; // loop over cols of right for (let rightColumn = 0; rightColumn < rightColumns; rightColumn++) { // dot product loop let dot = 0; // loop over columns of left for (let leftColumn = 0; leftColumn < leftColumns; leftColumn++) { const rightColumnBase = rightColumns * leftColumn; const leftIndex = leftRowBase + leftColumn; const rightIndex = rightColumnBase + rightColumn; dot += left.weights[leftIndex] * right.weights[rightIndex]; left.deltas[leftIndex] = 0; right.deltas[rightIndex] = 0; } product.weights[rightRowBase + rightColumn] = dot; } } } ================================================ FILE: src/recurrent/matrix/ones-matrix.ts ================================================ import { Matrix } from '.'; import { ones } from '../../utilities/ones'; /** return Matrix of ones */ export class OnesMatrix extends Matrix { constructor(rows: number, columns: number) { super(rows, columns); this.rows = rows; this.columns = columns; this.weights = ones(rows * columns); this.deltas = ones(rows * columns); } } ================================================ FILE: src/recurrent/matrix/random-matrix.ts ================================================ import { Matrix } from '.'; import { randomFloat } from '../../utilities/random'; /** return Matrix but filled with random numbers from gaussian */ export class RandomMatrix extends Matrix { std: number; constructor(rows: number, columns: number, std: number) { super(rows, columns); this.std = std; for (let i = 0, max = this.weights.length; i < max; i++) { this.weights[i] = randomFloat(-std, std); } } } ================================================ FILE: src/recurrent/matrix/random-n-matrix.ts ================================================ import { Matrix } from '.'; import { randomN } from '../../utilities/random'; export class RandomNMatrix extends Matrix { std: number; mu: number; constructor(rows: number, columns: number, mu: number, std: number) { super(rows, columns); this.std = std; this.mu = mu; this.fillRandN(); } // fill matrix with random gaussian numbers fillRandN(): void { for (let i = 0, max = this.weights.length; i < max; i++) { this.weights[i] = randomN(this.mu, this.std); } } } ================================================ FILE: src/recurrent/matrix/relu-b.ts ================================================ import { Matrix } from '.'; /** * adds {from} deltas to {m} deltas when {m} weights are above other a threshold of 0 */ export function reluB(product: Matrix, left: Matrix): void { for (let i = 0; i < product.deltas.length; i++) { left.deltas[i] = left.weights[i] > 0 ? product.deltas[i] : 0; } } ================================================ FILE: src/recurrent/matrix/relu.ts ================================================ import { Matrix } from '.'; /** * * relu {m} weights to {into} weights */ export function relu(product: Matrix, left: Matrix): void { for (let i = 0; i < left.weights.length; i++) { product.weights[i] = Math.max(0, left.weights[i]); // relu product.deltas[i] = 0; } } ================================================ FILE: src/recurrent/matrix/row-pluck-b.ts ================================================ import { Matrix } from '.'; /** * adds {from} deltas into {m} deltas */ export function rowPluckB( product: Matrix, left: Matrix, rowIndex: number ): void { const { columns } = left; const rowBase = columns * rowIndex; for (let column = 0; column < columns; column++) { left.deltas[rowBase + column] = product.deltas[column]; } } ================================================ FILE: src/recurrent/matrix/row-pluck.ts ================================================ import { Matrix } from '.'; export function rowPluck( product: Matrix, left: Matrix, rowPluckIndex: number ): void { const { columns } = left; const rowBase = columns * rowPluckIndex; for (let column = 0; column < columns; column++) { product.weights[column] = left.weights[rowBase + column]; product.deltas[column] = 0; } } ================================================ FILE: src/recurrent/matrix/sample-i.ts ================================================ import { Matrix } from '.'; import { randomFloat } from '../../utilities/random'; export function sampleI(matrix: Matrix): number { // sample argmax from w, assuming w are // probabilities that sum to one const r = randomFloat(0, 1); const w = matrix.weights; let x = 0; let i = 0; while (true) { x += w[i]; if (x > r) { return i; } i++; } } ================================================ FILE: src/recurrent/matrix/sigmoid-b.ts ================================================ import { Matrix } from '.'; export function sigmoidB(product: Matrix, left: Matrix): void { for (let i = 0; i < product.deltas.length; i++) { const mwi = product.weights[i]; left.deltas[i] = mwi * (1 - mwi) * product.deltas[i]; } } ================================================ FILE: src/recurrent/matrix/sigmoid.ts ================================================ import { Matrix } from '.'; export function sigmoid(product: Matrix, left: Matrix): void { // sigmoid nonlinearity for (let i = 0; i < left.weights.length; i++) { product.weights[i] = 1 / (1 + Math.exp(-left.weights[i])); product.deltas[i] = 0; } } // function sig(x) { // // helper function for computing sigmoid // return 1 / (1 + Math.exp(-x)); // } ================================================ FILE: src/recurrent/matrix/softmax.ts ================================================ import { Matrix } from '.'; export function softmax(matrix: Matrix): Matrix { // probability volume const result = new Matrix(matrix.rows, matrix.columns); let maxVal = -999999; for (let i = 0; i < matrix.weights.length; i++) { if (matrix.weights[i] > maxVal) { maxVal = matrix.weights[i]; } } let s = 0; for (let i = 0; i < matrix.weights.length; i++) { result.weights[i] = Math.exp(matrix.weights[i] - maxVal); s += result.weights[i]; } for (let i = 0; i < matrix.weights.length; i++) { result.weights[i] /= s; } // no backward pass here needed // since we will use the computed probabilities outside // to set gradients directly on m return result; } ================================================ FILE: src/recurrent/matrix/tanh-b.ts ================================================ import { Matrix } from '.'; export function tanhB(product: Matrix, left: Matrix): void { for (let i = 0; i < product.deltas.length; i++) { // grad for z = tanh(x) is (1 - z^2) const mwi = product.weights[i]; left.deltas[i] = (1 - mwi * mwi) * product.deltas[i]; } } ================================================ FILE: src/recurrent/matrix/tanh.ts ================================================ import { Matrix } from '.'; export function tanh(product: Matrix, left: Matrix): void { // tanh nonlinearity for (let i = 0; i < left.weights.length; i++) { product.weights[i] = Math.tanh(left.weights[i]); product.deltas[i] = 0; } } ================================================ FILE: src/recurrent/rnn-data-types.ts ================================================ /** TODO: might need to be extended to include string[][] */ // type Values = string[] | number[] | string; export type Value = string | number | boolean | string[] | number[] | boolean[]; export interface IRNNDatum { input: Value; output: Value; } export interface ITimeStepObject { [key: string]: number | number[]; } export type TimeStepArray = number[]; export type TimeStepValue = Array< number[] | number[][] | ITimeStepObject | ITimeStepObject[] | TimeStepArray >; export interface ITimeStepRNNDatum { input: TimeStepValue; output: TimeStepValue; } ================================================ FILE: src/recurrent/rnn-time-step.test.ts ================================================ import { INumberHash } from '../lookup'; import { LSTMTimeStep } from './lstm-time-step'; import { Matrix } from './matrix'; import { Equation } from './matrix/equation'; import { IRNNStatus } from './rnn'; import { RNNTimeStep, IRNNTimeStepJSON } from './rnn-time-step'; // TODO: break out LSTMTimeStep into its own tests describe('RNNTimeStep', () => { describe('.constructor()', () => { describe('when using options.json', () => { let fromJSONSpy: jest.SpyInstance; beforeEach(() => { fromJSONSpy = jest.spyOn(RNNTimeStep.prototype, 'fromJSON'); }); afterEach(() => { fromJSONSpy.mockRestore(); }); it('calls this.fromJSON with this value', () => { const json: IRNNTimeStepJSON = { type: 'RNNTimeStep', options: { inputSize: 1, inputRange: 1, hiddenLayers: [1], outputSize: 1, decayRate: 1, smoothEps: 1, regc: 1, clipval: 1, maxPredictionLength: 1, }, hiddenLayers: [ { weight: { rows: 1, columns: 1, weights: [1] }, transition: { rows: 1, columns: 1, weights: [1], }, bias: { rows: 1, columns: 1, weights: [1] }, }, ], outputConnector: { rows: 1, columns: 1, weights: [1], }, output: { rows: 1, columns: 1, weights: [1] }, inputLookup: { a: 0 }, inputLookupLength: 1, outputLookup: { a: 0 }, outputLookupLength: 1, }; // eslint-disable-next-line no-new new RNNTimeStep({ json }); expect(fromJSONSpy).toHaveBeenCalledWith(json); }); }); }); describe('.createInputMatrix()', () => { it('throws', () => { expect(() => { new RNNTimeStep().createInputMatrix(); }).toThrow(); }); }); describe('.createOutputMatrix()', () => { it('creates the outputConnector and output for model', () => { const net = new RNNTimeStep({ inputSize: 2, hiddenLayers: [9, 11], outputSize: 5, }); const { outputConnector, output } = net.createOutputMatrices(); expect(outputConnector.rows).toBe(5); expect(outputConnector.columns).toBe(11); expect(output.rows).toBe(5); expect(output.columns).toBe(1); }); }); describe('.bindEquation()', () => { let getEquationSpy: jest.SpyInstance; beforeEach(() => { getEquationSpy = jest.spyOn(RNNTimeStep.prototype, 'getEquation'); }); afterEach(() => { getEquationSpy.mockRestore(); }); it('calls static getEquation method', () => { const net = new RNNTimeStep(); net.initialize(); net.bindEquation(); expect(getEquationSpy).toBeCalled(); }); it('adds equations as expected', () => { const net = new RNNTimeStep({ inputSize: 2, hiddenLayers: [9, 11], outputSize: 5, }); net.initialize(); net.mapModel(); expect(net.model.equations.length).toBe(0); net.bindEquation(); expect(net.model.equations.length).toBe(1); net.bindEquation(); expect(net.model.equations.length).toBe(2); net.bindEquation(); expect(net.model.equations.length).toBe(3); }); }); describe('.mapModel()', () => { describe('when .createHiddenLayers() does not provide model.hiddenLayers', () => { it('throws', () => { const net = new RNNTimeStep({ hiddenLayers: [] }); expect(() => { net.mapModel(); }).not.toThrow(); }); }); it('maps models to model.allMatrices', () => { const net = new RNNTimeStep(); const model = net.mapModel(); expect(model.allMatrices.length).toBe(5); }); }); describe('.backpropagate()', () => { let equationsBackpropagateSpy: jest.SpyInstance; beforeEach(() => { equationsBackpropagateSpy = jest.spyOn( Equation.prototype, 'backpropagate' ); }); afterEach(() => { equationsBackpropagateSpy.mockRestore(); }); it('steps through model.equations in reverse, calling model.equations[index].backpropagate', () => { const net = new RNNTimeStep(); for (let i = 0; i < 3; i++) { const equation = new Equation(); equation.add(new Matrix(1, 1), new Matrix(1, 1)); net.model.equations.push(equation); } net.backpropagate(); expect(equationsBackpropagateSpy).toHaveBeenCalledTimes(3); }); }); describe('.run()', () => { describe('when called with unknown data shape', () => { it('throws', () => { const net = new RNNTimeStep({ inputSize: 1, outputSize: 1 }); net.initialize(); net.train([[1, 2]], { iterations: 1 }); expect(() => { net.run({ one: 1, two: 2 }); }).toThrow(); }); }); describe('when called with array,number data shape', () => { let runArraySpy: jest.SpyInstance; beforeEach(() => { runArraySpy = jest.spyOn(RNNTimeStep.prototype, 'runArray'); }); afterEach(() => { runArraySpy.mockRestore(); }); it('calls this.runArray() and returns value from there', () => { const net = new RNNTimeStep({ inputSize: 1, outputSize: 1 }); net.initialize(); net.train([[1, 2]], { iterations: 1 }); const result = net.run([1, 2]); expect(result).toBeGreaterThan(0); expect(runArraySpy).toHaveBeenCalledWith([1, 2]); }); }); describe('when called with array,array,number data shape', () => { let runArrayOfArraySpy: jest.SpyInstance; beforeEach(() => { runArrayOfArraySpy = jest.spyOn( RNNTimeStep.prototype, 'runArrayOfArray' ); }); afterEach(() => { runArrayOfArraySpy.mockRestore(); }); it('calls this.runArrayOfArray()', () => { const net = new RNNTimeStep({ inputSize: 4, outputSize: 4 }); net.initialize(); const item1 = [ [1, 2, 3, 4], [4, 3, 2, 1], ]; const item2 = [ [4, 3, 2, 1], [1, 2, 3, 4], ]; net.train([item1, item2], { iterations: 1 }); net.run(item1); expect(runArrayOfArraySpy).toHaveBeenCalledWith(item1); }); }); describe('when called with array,object,number data shape', () => { let runArrayOfObjectSpy: jest.SpyInstance; beforeEach(() => { runArrayOfObjectSpy = jest.spyOn( RNNTimeStep.prototype, 'runArrayOfObject' ); }); afterEach(() => { runArrayOfObjectSpy.mockRestore(); }); it('calls this.runArrayOfArray()', () => { const net = new RNNTimeStep({ inputSize: 4, outputSize: 4 }); net.initialize(); const oneToFour = { low: 1, high: 2, mid: 3, total: 4 }; const fourToOne = { low: 4, high: 3, mid: 2, total: 1 }; const item1 = [oneToFour, fourToOne]; const item2 = [fourToOne, oneToFour]; net.train([item1, item2], { iterations: 1 }); net.run(item1); expect(runArrayOfObjectSpy).toHaveBeenCalledWith(item1); }); }); }); describe('.runArrayOfArray()', () => { describe('when network is not runnable', () => { it('throws', () => { expect(() => { const net = new RNNTimeStep(); net.runArrayOfArray([Float32Array.from([1])]); }).toThrow(); }); }); describe('when network is runnable', () => { let runInputSpy: jest.SpyInstance; beforeEach(() => { runInputSpy = jest.spyOn(Equation.prototype, 'runInput'); }); afterEach(() => { runInputSpy.mockRestore(); }); it('sets up equations for length of input plus 1 for internal of 0', () => { const net = new RNNTimeStep({ inputSize: 2, hiddenLayers: [2], outputSize: 2, }); net.initialize(); net.bindEquation(); expect(net.model.equations.length).toBe(1); net.runArrayOfArray([ Float32Array.from([1, 3]), Float32Array.from([2, 2]), Float32Array.from([3, 1]), ]); expect(net.model.equations.length).toBe(4); }); it('sets calls equation.runInput() with value in array for each input plus 1 for 0 (to end) output', () => { const net = new RNNTimeStep({ inputSize: 2, hiddenLayers: [2], outputSize: 2, }); net.initialize(); net.bindEquation(); net.runArrayOfArray( [ [1, 3], [2, 2], [3, 1], ].map((v) => Float32Array.from(v)) ); expect(runInputSpy.mock.instances.length).toBe(4); expect(runInputSpy.mock.calls.length).toBe(4); expect(runInputSpy.mock.calls[0][0]).toEqual(Float32Array.from([1, 3])); expect(runInputSpy.mock.calls[1][0]).toEqual(Float32Array.from([2, 2])); expect(runInputSpy.mock.calls[2][0]).toEqual(Float32Array.from([3, 1])); expect(runInputSpy.mock.calls[3][0]).toEqual(Float32Array.from([0, 0])); }); it('sets calls this.end() after calls equations.runInput', () => { const net = new RNNTimeStep({ inputSize: 2, hiddenLayers: [2], outputSize: 2, }); const stub = (net.end = jest.fn()); net.initialize(); net.bindEquation(); net.runArrayOfArray([ Float32Array.from([1, 3]), Float32Array.from([2, 2]), Float32Array.from([3, 1]), ]); expect(stub).toBeCalled(); }); }); }); describe('.train()', () => { it('throws on array,datum,array w/ inputSize of 2', () => { const data = [{ input: [1, 2], output: [3, 4] }]; const net = new RNNTimeStep({ inputSize: 2, hiddenLayers: [10], outputSize: 1, }); expect(() => { net.train(data); }).toThrow('manually set inputSize and outputSize mismatch'); }); it('throws on array,datum,array w/ outputSize of 2', () => { const data = [{ input: [1, 2], output: [3, 4] }]; const net = new RNNTimeStep({ inputSize: 1, hiddenLayers: [10], outputSize: 2, }); expect(() => { net.train(data); }).toThrow('manually set inputSize and outputSize mismatch'); }); it('throws on array,datum,object w/ inputSize of 2', () => { const data = [{ input: { a: 1, b: 2 }, output: { c: 3, d: 4 } }]; const net = new RNNTimeStep({ inputSize: 2, hiddenLayers: [10], outputSize: 2, }); expect(() => { net.train(data); }).toThrow('inputSize must be 1 for this data size'); }); describe('automatically setting options.inputSize and options.outputSize', () => { describe('array', () => { it('will set inputSize & outputSize if from data', () => { const data = [[0.1, 0.2, 0.3, 0.4, 0.5]]; const options = { iterations: 0, }; const net = new RNNTimeStep(); net.train(data, options); expect(net.options.inputSize).toBe(1); expect(net.options.outputSize).toBe(1); }); }); describe('array of array', () => { it('will set inputSize & outputSize if from data', () => { const data = [ [ [0.1, 0.5], [0.2, 0.4], [0.3, 0.3], [0.4, 0.2], [0.5, 0.1], ], ]; const options = { iterations: 1, }; const net = new RNNTimeStep(); net.train(data, options); expect(net.options.inputSize).toBe(2); expect(net.options.outputSize).toBe(2); }); }); describe('array of object in single long array', () => { it('will set inputSize & outputSize if from data', () => { const data = [{ low: 0.1, med: 0.25, high: 0.5 }]; const options = { iterations: 1, }; const net = new RNNTimeStep(); net.train(data, options); expect(net.options.inputSize).toBe(1); expect(net.options.outputSize).toBe(1); }); }); describe('array of object in multiple array', () => { it('will set inputSize & outputSize if from data', () => { const data = [ [ { low: 0.1, med: 0.25, high: 0.5 }, { low: 0.5, med: 0.25, high: 0.1 }, ], ]; const options = { iterations: 1, }; const net = new RNNTimeStep(); net.train(data, options); expect(net.options.inputSize).toBe(3); expect(net.options.outputSize).toBe(3); }); }); describe('input/output numbers', () => { it('will set inputSize & outputSize if from data', () => { const data = [{ input: [0.1, 0.2, 0.3, 0.4], output: [0.5] }]; const options = { iterations: 1, }; const net = new RNNTimeStep(); net.train(data, options); expect(net.options.inputSize).toBe(1); expect(net.options.outputSize).toBe(1); }); }); describe('input/output arrays', () => { it('will set inputSize & outputSize if from data', () => { const data = [ { input: [[0.1, 0.5]], output: [[0.5, 0.1]], }, ]; const options = { iterations: 1, }; const net = new RNNTimeStep(); net.train(data, options); expect(net.options.inputSize).toBe(2); expect(net.options.outputSize).toBe(2); }); }); describe('input/output object', () => { it('will set inputSize & outputSize if from data', () => { const data = [ { input: { low: 0.1, high: 0.5 }, output: { low: 0.5, high: 0.1 }, }, ]; const options = { iterations: 1, }; const net = new RNNTimeStep(); net.train(data, options); expect(net.options.inputSize).toBe(1); expect(net.options.outputSize).toBe(1); }); }); describe('datum', () => { it('will set inputSize & outputSize if from data', () => { const data = [ { input: [{ low: 0.1, high: 0.5 }], output: [{ low: 0.5, high: 0.1 }], }, ]; const options = { iterations: 1, }; const net = new RNNTimeStep(); net.train(data, options); expect(net.options.inputSize).toBe(2); expect(net.options.outputSize).toBe(2); }); }); it('will not set inputSize & outputSize if already set larger than 1', () => { const net = new RNNTimeStep({ inputSize: 99, outputSize: 88 }); net.initialize = () => { throw new Error('got passed size check'); }; expect(() => { net.train([ [0, 1, 2, 3, 4], [4, 3, 2, 1, 0], ]); }).toThrow(); expect(net.options.inputSize).toBe(99); expect(net.options.outputSize).toBe(88); }); }); describe('calling using arrays', () => { describe('training data with 1D arrays', () => { describe('end to end', () => { let trainArraysSpy: jest.SpyInstance; let predictTargetSpy: jest.SpyInstance; beforeEach(() => { trainArraysSpy = jest.spyOn( RNNTimeStep.prototype, 'trainArrayOfArray' ); predictTargetSpy = jest.spyOn(Equation.prototype, 'predictTarget'); }); afterEach(() => { trainArraysSpy.mockRestore(); predictTargetSpy.mockRestore(); }); it('uses .runInputNumbers with correct arguments', () => { const net = new RNNTimeStep({ inputSize: 1, hiddenLayers: [1], outputSize: 1, }); const trainingData = [ [0.1, 0.2, 0.3, 0.4, 0.5], [0.5, 0.4, 0.3, 0.2, 0.1], ]; net.train(trainingData, { iterations: 1 }); expect(trainArraysSpy.mock.calls.length).toBe(2); expect(trainArraysSpy.mock.calls[0].length).toBe(1); expect(trainArraysSpy.mock.calls[0][0]).toEqual( trainingData[0].map((value) => Float32Array.from([value])) ); expect(trainArraysSpy.mock.calls[1][0]).toEqual( trainingData[1].map((value) => Float32Array.from([value])) ); expect(predictTargetSpy.mock.calls.length).toBe(8); expect(net.model.equations.length).toBe(5); // first array expect(predictTargetSpy.mock.calls[0][0]).toEqual( Float32Array.from([0.1]) ); expect(predictTargetSpy.mock.calls[0][1]).toEqual( Float32Array.from([0.2]) ); expect(predictTargetSpy.mock.calls[1][0]).toEqual( Float32Array.from([0.2]) ); expect(predictTargetSpy.mock.calls[1][1]).toEqual( Float32Array.from([0.3]) ); expect(predictTargetSpy.mock.calls[2][0]).toEqual( Float32Array.from([0.3]) ); expect(predictTargetSpy.mock.calls[2][1]).toEqual( Float32Array.from([0.4]) ); expect(predictTargetSpy.mock.calls[3][0]).toEqual( Float32Array.from([0.4]) ); expect(predictTargetSpy.mock.calls[3][1]).toEqual( Float32Array.from([0.5]) ); // second array expect(predictTargetSpy.mock.calls[4][0]).toEqual( Float32Array.from([0.5]) ); expect(predictTargetSpy.mock.calls[4][1]).toEqual( Float32Array.from([0.4]) ); expect(predictTargetSpy.mock.calls[5][0]).toEqual( Float32Array.from([0.4]) ); expect(predictTargetSpy.mock.calls[5][1]).toEqual( Float32Array.from([0.3]) ); expect(predictTargetSpy.mock.calls[6][0]).toEqual( Float32Array.from([0.3]) ); expect(predictTargetSpy.mock.calls[6][1]).toEqual( Float32Array.from([0.2]) ); expect(predictTargetSpy.mock.calls[7][0]).toEqual( Float32Array.from([0.2]) ); expect(predictTargetSpy.mock.calls[7][1]).toEqual( Float32Array.from([0.1]) ); }); }); it('can learn basic logic', () => { const net = new LSTMTimeStep({ inputSize: 1, hiddenLayers: [10], outputSize: 1, }); const trainingData = [ [0.1, 0.2, 0.3, 0.4, 0.5], [0.5, 0.4, 0.3, 0.2, 0.1], ]; const result = net.train(trainingData, { errorThresh: 0.005, iterations: 1000, }); expect(result.error).toBeLessThan(0.005); expect(result.iterations).toBeLessThan(1000); const result1 = net.forecast([0.1, 0.2, 0.3], 2); expect(result1[0]).toBeCloseTo(0.4, 1); expect(result1[1]).toBeCloseTo(0.5, 1); const result2 = net.forecast([0.5, 0.4, 0.3], 2); expect(result2[0]).toBeCloseTo(0.2, 1); expect(result2[1]).toBeCloseTo(0.1, 1); }); }); describe('training data with 2D arrays', () => { let trainArraysSpy: jest.SpyInstance; let predictTargetSpy: jest.SpyInstance; beforeEach(() => { trainArraysSpy = jest.spyOn( RNNTimeStep.prototype, 'trainArrayOfArray' ); predictTargetSpy = jest.spyOn(Equation.prototype, 'predictTarget'); }); afterEach(() => { trainArraysSpy.mockRestore(); predictTargetSpy.mockRestore(); }); it('uses .trainArrays with correct arguments', () => { const net = new RNNTimeStep({ inputSize: 2, hiddenLayers: [1], outputSize: 2, }); const trainingData = [ [0.1, 0.5], [0.2, 0.4], [0.3, 0.3], [0.4, 0.2], [0.5, 0.1], ]; const trainingDataFormatted = trainingData.map((array) => Float32Array.from(array) ); net.train(trainingData, { iterations: 1 }); expect(trainArraysSpy.mock.calls.length).toBe(1); expect(trainArraysSpy.mock.calls[0].length).toBe(1); expect(trainArraysSpy.mock.calls[0][0]).toEqual( trainingDataFormatted ); expect(predictTargetSpy.mock.calls.length).toBe(4); expect(net.model.equations.length).toBe(5); // first array expect(predictTargetSpy.mock.calls[0][0]).toEqual( Float32Array.from([0.1, 0.5]) ); expect(predictTargetSpy.mock.calls[0][1]).toEqual( Float32Array.from([0.2, 0.4]) ); // second array expect(predictTargetSpy.mock.calls[1][0]).toEqual( Float32Array.from([0.2, 0.4]) ); expect(predictTargetSpy.mock.calls[1][1]).toEqual( Float32Array.from([0.3, 0.3]) ); // third array expect(predictTargetSpy.mock.calls[2][0]).toEqual( Float32Array.from([0.3, 0.3]) ); expect(predictTargetSpy.mock.calls[2][1]).toEqual( Float32Array.from([0.4, 0.2]) ); // forth array expect(predictTargetSpy.mock.calls[3][0]).toEqual( Float32Array.from([0.4, 0.2]) ); expect(predictTargetSpy.mock.calls[3][1]).toEqual( Float32Array.from([0.5, 0.1]) ); }); it('can learn basic logic', () => { const net = new LSTMTimeStep({ inputSize: 2, hiddenLayers: [20], outputSize: 2, }); const trainingData = [ [0.1, 0.5], [0.2, 0.4], [0.3, 0.3], [0.4, 0.2], [0.5, 0.1], ]; const result = net.train(trainingData, { errorThresh: 0.05 }); expect(result.error).toBeLessThan(0.05); expect(result.iterations).toBeLessThan(4000); }); }); describe('training data with 3D arrays', () => { let trainArraysSpy: jest.SpyInstance; let predictTargetSpy: jest.SpyInstance; beforeEach(() => { trainArraysSpy = jest.spyOn( RNNTimeStep.prototype, 'trainArrayOfArray' ); predictTargetSpy = jest.spyOn(Equation.prototype, 'predictTarget'); }); afterEach(() => { trainArraysSpy.mockRestore(); predictTargetSpy.mockRestore(); }); it('uses .trainArrays with correct arguments', () => { const net = new RNNTimeStep({ inputSize: 2, hiddenLayers: [1], outputSize: 2, }); const trainingData = [ [ [0.1, 0.5], [0.2, 0.4], [0.3, 0.3], [0.4, 0.2], [0.5, 0.1], ], [ [0.5, 0.9], [0.6, 0.8], [0.7, 0.7], [0.8, 0.6], [0.9, 0.5], ], ]; const trainingDataFormatted0 = trainingData[0].map((array) => Float32Array.from(array) ); const trainingDataFormatted1 = trainingData[1].map((array) => Float32Array.from(array) ); net.train(trainingData, { iterations: 1 }); expect(trainArraysSpy.mock.calls.length).toBe(2); expect(trainArraysSpy.mock.calls[0].length).toBe(1); expect(trainArraysSpy.mock.calls[0][0]).toEqual( trainingDataFormatted0 ); expect(trainArraysSpy.mock.calls[1][0]).toEqual( trainingDataFormatted1 ); expect(predictTargetSpy.mock.calls.length).toBe(8); expect(net.model.equations.length).toBe(5); // first set, first array expect(predictTargetSpy.mock.calls[0][0]).toEqual( Float32Array.from([0.1, 0.5]) ); expect(predictTargetSpy.mock.calls[0][1]).toEqual( Float32Array.from([0.2, 0.4]) ); // first set, second array expect(predictTargetSpy.mock.calls[1][0]).toEqual( Float32Array.from([0.2, 0.4]) ); expect(predictTargetSpy.mock.calls[1][1]).toEqual( Float32Array.from([0.3, 0.3]) ); // first set, third array expect(predictTargetSpy.mock.calls[2][0]).toEqual( Float32Array.from([0.3, 0.3]) ); expect(predictTargetSpy.mock.calls[2][1]).toEqual( Float32Array.from([0.4, 0.2]) ); // first set, forth array expect(predictTargetSpy.mock.calls[3][0]).toEqual( Float32Array.from([0.4, 0.2]) ); expect(predictTargetSpy.mock.calls[3][1]).toEqual( Float32Array.from([0.5, 0.1]) ); // second set, first array expect(predictTargetSpy.mock.calls[4][0]).toEqual( Float32Array.from([0.5, 0.9]) ); expect(predictTargetSpy.mock.calls[4][1]).toEqual( Float32Array.from([0.6, 0.8]) ); // second set, second array expect(predictTargetSpy.mock.calls[5][0]).toEqual( Float32Array.from([0.6, 0.8]) ); expect(predictTargetSpy.mock.calls[5][1]).toEqual( Float32Array.from([0.7, 0.7]) ); // second set, third array expect(predictTargetSpy.mock.calls[6][0]).toEqual( Float32Array.from([0.7, 0.7]) ); expect(predictTargetSpy.mock.calls[6][1]).toEqual( Float32Array.from([0.8, 0.6]) ); // second set, forth array expect(predictTargetSpy.mock.calls[7][0]).toEqual( Float32Array.from([0.8, 0.6]) ); expect(predictTargetSpy.mock.calls[7][1]).toEqual( Float32Array.from([0.9, 0.5]) ); }); it('can learn basic logic', () => { const net = new LSTMTimeStep({ inputSize: 2, hiddenLayers: [30], outputSize: 2, }); const trainingData = [ [ [0.1, 0.5], [0.2, 0.4], [0.3, 0.3], [0.4, 0.2], [0.5, 0.1], ], [ [0.5, 0.9], [0.6, 0.8], [0.7, 0.7], [0.8, 0.6], [0.9, 0.5], ], ]; const result = net.train(trainingData, { errorThresh: 0.05 }); expect(result.error).toBeLessThan(0.05); expect(result.iterations).toBeLessThan(4000); }); }); }); describe('calling using training datum', () => { describe('training data with objects', () => { let trainArraysSpy: jest.SpyInstance; let predictTargetSpy: jest.SpyInstance; beforeEach(() => { trainArraysSpy = jest.spyOn( RNNTimeStep.prototype, 'trainArrayOfArray' ); predictTargetSpy = jest.spyOn(Equation.prototype, 'predictTarget'); }); afterEach(() => { trainArraysSpy.mockRestore(); predictTargetSpy.mockRestore(); }); it('uses .runInputOutput with correct arguments', () => { const net = new RNNTimeStep({ inputSize: 1, hiddenLayers: [1], outputSize: 1, }); // average temp const trainingData = [ // Washington DC { input: { jan: 42, feb: 44, mar: 53, apr: 64, }, output: { may: 75, jun: 83, }, }, // Bluff Utah { input: { jan: 44, feb: 52, mar: 63, apr: 72, }, output: { may: 82, jun: 92, }, }, ]; net.train(trainingData, { iterations: 1 }); expect(trainArraysSpy.mock.calls.length).toBe(2); expect(trainArraysSpy.mock.calls[0].length).toBe(1); expect(trainArraysSpy.mock.calls[0][0]).toEqual( [42, 44, 53, 64, 75, 83].map((v: number) => Float32Array.from([v])) ); expect(trainArraysSpy.mock.calls[1][0]).toEqual( [44, 52, 63, 72, 82, 92].map((v: number) => Float32Array.from([v])) ); expect(predictTargetSpy.mock.calls.length).toBe(10); expect(net.model.equations.length).toBe(6); // first array expect(predictTargetSpy.mock.calls[0][0]).toEqual( new Float32Array([42]) ); expect(predictTargetSpy.mock.calls[0][1]).toEqual( new Float32Array([44]) ); expect(predictTargetSpy.mock.calls[1][0]).toEqual( new Float32Array([44]) ); expect(predictTargetSpy.mock.calls[1][1]).toEqual( new Float32Array([53]) ); expect(predictTargetSpy.mock.calls[2][0]).toEqual( new Float32Array([53]) ); expect(predictTargetSpy.mock.calls[2][1]).toEqual( new Float32Array([64]) ); expect(predictTargetSpy.mock.calls[3][0]).toEqual( new Float32Array([64]) ); expect(predictTargetSpy.mock.calls[3][1]).toEqual( new Float32Array([75]) ); expect(predictTargetSpy.mock.calls[4][0]).toEqual( new Float32Array([75]) ); expect(predictTargetSpy.mock.calls[4][1]).toEqual( new Float32Array([83]) ); // second array expect(predictTargetSpy.mock.calls[5][0]).toEqual( new Float32Array([44]) ); expect(predictTargetSpy.mock.calls[5][1]).toEqual( new Float32Array([52]) ); expect(predictTargetSpy.mock.calls[6][0]).toEqual( new Float32Array([52]) ); expect(predictTargetSpy.mock.calls[6][1]).toEqual( new Float32Array([63]) ); expect(predictTargetSpy.mock.calls[7][0]).toEqual( new Float32Array([63]) ); expect(predictTargetSpy.mock.calls[7][1]).toEqual( new Float32Array([72]) ); expect(predictTargetSpy.mock.calls[8][0]).toEqual( new Float32Array([72]) ); expect(predictTargetSpy.mock.calls[8][1]).toEqual( new Float32Array([82]) ); expect(predictTargetSpy.mock.calls[9][0]).toEqual( new Float32Array([82]) ); expect(predictTargetSpy.mock.calls[9][1]).toEqual( new Float32Array([92]) ); }); }); describe('training data with 1D arrays', () => { let trainArraysSpy: jest.SpyInstance; let predictTargetSpy: jest.SpyInstance; beforeEach(() => { trainArraysSpy = jest.spyOn( RNNTimeStep.prototype, 'trainArrayOfArray' ); predictTargetSpy = jest.spyOn(Equation.prototype, 'predictTarget'); }); afterEach(() => { trainArraysSpy.mockRestore(); predictTargetSpy.mockRestore(); }); it('uses .runInputOutput with correct arguments', () => { const net = new RNNTimeStep({ inputSize: 1, hiddenLayers: [1], outputSize: 1, }); const trainingData = [ { input: [1, 2, 3, 4], output: [5] }, { input: [5, 4, 3, 2], output: [1] }, ]; const trainingDataFormatted0 = [1, 2, 3, 4, 5].map((v: number) => Float32Array.from([v]) ); const trainingDataFormatted1 = [5, 4, 3, 2, 1].map((v: number) => Float32Array.from([v]) ); net.train(trainingData, { iterations: 1 }); expect(trainArraysSpy.mock.calls.length).toBe(2); expect(trainArraysSpy.mock.calls[0].length).toBe(1); expect(trainArraysSpy.mock.calls[0][0]).toEqual( trainingDataFormatted0 ); expect(trainArraysSpy.mock.calls[1][0]).toEqual( trainingDataFormatted1 ); expect(predictTargetSpy.mock.calls.length).toBe(8); expect(net.model.equations.length).toBe(5); // first array expect(predictTargetSpy.mock.calls[0][0]).toEqual( Float32Array.from([1]) ); expect(predictTargetSpy.mock.calls[0][1]).toEqual( Float32Array.from([2]) ); expect(predictTargetSpy.mock.calls[1][0]).toEqual( Float32Array.from([2]) ); expect(predictTargetSpy.mock.calls[1][1]).toEqual( Float32Array.from([3]) ); expect(predictTargetSpy.mock.calls[2][0]).toEqual( Float32Array.from([3]) ); expect(predictTargetSpy.mock.calls[2][1]).toEqual( Float32Array.from([4]) ); expect(predictTargetSpy.mock.calls[3][0]).toEqual( Float32Array.from([4]) ); expect(predictTargetSpy.mock.calls[3][1]).toEqual( Float32Array.from([5]) ); // second array expect(predictTargetSpy.mock.calls[4][0]).toEqual( Float32Array.from([5]) ); expect(predictTargetSpy.mock.calls[4][1]).toEqual( Float32Array.from([4]) ); expect(predictTargetSpy.mock.calls[5][0]).toEqual( Float32Array.from([4]) ); expect(predictTargetSpy.mock.calls[5][1]).toEqual( Float32Array.from([3]) ); expect(predictTargetSpy.mock.calls[6][0]).toEqual( Float32Array.from([3]) ); expect(predictTargetSpy.mock.calls[6][1]).toEqual( Float32Array.from([2]) ); expect(predictTargetSpy.mock.calls[7][0]).toEqual( Float32Array.from([2]) ); expect(predictTargetSpy.mock.calls[7][1]).toEqual( Float32Array.from([1]) ); }); }); describe('training data with 2D arrays', () => { let trainArraysSpy: jest.SpyInstance; let predictTargetSpy: jest.SpyInstance; beforeEach(() => { trainArraysSpy = jest.spyOn( RNNTimeStep.prototype, 'trainArrayOfArray' ); predictTargetSpy = jest.spyOn(Equation.prototype, 'predictTarget'); }); afterEach(() => { trainArraysSpy.mockRestore(); predictTargetSpy.mockRestore(); }); it('uses .runInputOutputArray with correct arguments', () => { const net = new RNNTimeStep({ inputSize: 2, hiddenLayers: [1], outputSize: 2, }); const trainingData = [ { input: [ [0.1, 0.5], [0.2, 0.4], [0.3, 0.3], [0.4, 0.2], ], output: [[0.5, 0.1]], }, { input: [ [0.5, 0.9], [0.6, 0.8], [0.7, 0.7], [0.8, 0.6], ], output: [[0.9, 0.5]], }, ]; const trainingDataFormatted0 = [ ...trainingData[0].input.map((value) => Float32Array.from(value)), ...trainingData[0].output.map((value) => Float32Array.from(value)), ]; const trainingDataFormatted1 = [ ...trainingData[1].input.map((value) => Float32Array.from(value)), ...trainingData[1].output.map((value) => Float32Array.from(value)), ]; net.train(trainingData, { iterations: 1 }); expect(trainArraysSpy.mock.calls.length).toBe(2); expect(trainArraysSpy.mock.calls[0].length).toBe(1); expect(trainArraysSpy.mock.calls[0][0]).toEqual( trainingDataFormatted0 ); expect(trainArraysSpy.mock.calls[1][0]).toEqual( trainingDataFormatted1 ); expect(predictTargetSpy.mock.calls.length).toBe(8); expect(net.model.equations.length).toBe(5); // first set, first array expect(predictTargetSpy.mock.calls[0][0]).toEqual( Float32Array.from([0.1, 0.5]) ); expect(predictTargetSpy.mock.calls[0][1]).toEqual( Float32Array.from([0.2, 0.4]) ); // first set, second array expect(predictTargetSpy.mock.calls[1][0]).toEqual( Float32Array.from([0.2, 0.4]) ); expect(predictTargetSpy.mock.calls[1][1]).toEqual( Float32Array.from([0.3, 0.3]) ); // first set, third array expect(predictTargetSpy.mock.calls[2][0]).toEqual( Float32Array.from([0.3, 0.3]) ); expect(predictTargetSpy.mock.calls[2][1]).toEqual( Float32Array.from([0.4, 0.2]) ); // first set, forth array expect(predictTargetSpy.mock.calls[3][0]).toEqual( Float32Array.from([0.4, 0.2]) ); expect(predictTargetSpy.mock.calls[3][1]).toEqual( Float32Array.from([0.5, 0.1]) ); // second set, first array expect(predictTargetSpy.mock.calls[4][0]).toEqual( Float32Array.from([0.5, 0.9]) ); expect(predictTargetSpy.mock.calls[4][1]).toEqual( Float32Array.from([0.6, 0.8]) ); // second set, second array expect(predictTargetSpy.mock.calls[5][0]).toEqual( Float32Array.from([0.6, 0.8]) ); expect(predictTargetSpy.mock.calls[5][1]).toEqual( Float32Array.from([0.7, 0.7]) ); // second set, third array expect(predictTargetSpy.mock.calls[6][0]).toEqual( Float32Array.from([0.7, 0.7]) ); expect(predictTargetSpy.mock.calls[6][1]).toEqual( Float32Array.from([0.8, 0.6]) ); // second set, forth array expect(predictTargetSpy.mock.calls[7][0]).toEqual( Float32Array.from([0.8, 0.6]) ); expect(predictTargetSpy.mock.calls[7][1]).toEqual( Float32Array.from([0.9, 0.5]) ); }); }); }); describe('prediction using arrays', () => { it('can train and predict linear numeric, single input, 1 to 5, and 5 to 1', () => { const net = new LSTMTimeStep({ inputSize: 1, hiddenLayers: [20, 20], outputSize: 1, }); const trainingData = [ [0.1, 0.2, 0.3, 0.4, 0.5], [0.5, 0.4, 0.3, 0.2, 0.1], ]; const result = net.train(trainingData); expect(result.error).toBeLessThan(0.05); const closeToFive = net.run([0.1, 0.2, 0.3, 0.4]); const closeToOne = net.run([0.5, 0.4, 0.3, 0.2]); expect(closeToOne.toFixed(1)).toBe('0.1'); expect(closeToFive.toFixed(1)).toBe('0.5'); }); it('can train and predict single linear array, two input, 1 to 5, and 5 to 1', () => { const net = new LSTMTimeStep({ inputSize: 2, hiddenLayers: [20], outputSize: 2, }); // Same test as previous, but combined on a single set const trainingData = [ [0.1, 0.5], [0.2, 0.4], [0.3, 0.3], [0.4, 0.2], [0.5, 0.1], ]; const result = net.train(trainingData, { errorThresh: 0.01, }); expect(result.error).toBeLessThan(0.01); const closeToFiveAndOne = net.run([ [0.1, 0.5], [0.2, 0.4], [0.3, 0.3], [0.4, 0.2], ]); expect(closeToFiveAndOne[0].toFixed(1)).toBe('0.5'); expect(closeToFiveAndOne[1].toFixed(1)).toBe('0.1'); }); it('can train and predict multiple linear array, two input, 1 to 5, 5 to 1, 5 to 9, and 9 to 5', () => { const net = new LSTMTimeStep({ inputSize: 2, hiddenLayers: [40], outputSize: 2, }); // Same test as previous, but combined on a single set const trainingData = [ [ [0.1, 0.5], [0.2, 0.4], [0.3, 0.3], [0.4, 0.2], [0.5, 0.1], ], [ [0.5, 0.9], [0.6, 0.8], [0.7, 0.7], [0.8, 0.6], [0.9, 0.5], ], ]; const result = net.train(trainingData); expect(result.error).toBeLessThan(0.05); const closeToFiveAndOne = net.run([ [0.1, 0.5], [0.2, 0.4], [0.3, 0.3], [0.4, 0.2], ]); expect(closeToFiveAndOne[0].toFixed(1)).toBe('0.5'); expect(closeToFiveAndOne[1].toFixed(1)).toBe('0.1'); const closeToNineAndFive = net.run([ [0.5, 0.9], [0.6, 0.8], [0.7, 0.7], [0.8, 0.6], ]); expect(closeToNineAndFive[0].toFixed(1)).toBe('0.9'); expect(closeToNineAndFive[1].toFixed(1)).toBe('0.5'); }); }); describe('prediction using input/output', () => { describe('with objects', () => { it('can train and predict input/output linear array avg weather data', () => { const net = new LSTMTimeStep({ inputSize: 1, hiddenLayers: [5], outputSize: 1, }); // average temp const trainingData = [ // Washington DC { input: { jan: 0.42, feb: 0.44, mar: 0.53, apr: 0.64, }, output: { may: 0.75, jun: 0.83, }, }, // Bluff Utah { input: { jan: 0.44, feb: 0.52, mar: 0.63, apr: 0.72, }, output: { may: 0.82, jun: 0.92, }, }, ]; const result = net.train(trainingData); const washington = net.runObject({ jan: 0.42, feb: 0.44, mar: 0.53, apr: 0.64, }); const bluff = net.runObject({ jan: 0.44, feb: 0.52, mar: 0.63, apr: 0.72, }); expect(result.error).toBeLessThan(0.05); expect(washington.may.toFixed(2).indexOf('0.7')).toBeGreaterThan(-1); expect(washington.jun.toFixed(2).indexOf('0.8')).toBeGreaterThan(-1); expect(bluff.may.toFixed(2).indexOf('0.8')).toBeGreaterThan(-1); expect(bluff.jun.toFixed(2).indexOf('0.9')).toBeGreaterThan(-1); }); }); describe('with arrays', () => { it('can use inputs(4) and output(1)', () => { const net = new LSTMTimeStep({ inputSize: 1, hiddenLayers: [20, 20], outputSize: 1, }); // Same test as previous, but combined on a single set const trainingData = [ { input: [0.1, 0.2, 0.3, 0.4], output: [0.5], }, { input: [0.5, 0.4, 0.3, 0.2], output: [0.1], }, ]; const result = net.train(trainingData); expect(result.error).toBeLessThan(0.09); const closeToFive = net.run([0.1, 0.2, 0.3, 0.4]); const closeToOne = net.run([0.5, 0.4, 0.3, 0.2]); expect(closeToFive.toFixed(1)).toBe('0.5'); expect(closeToOne.toFixed(1)).toBe('0.1'); }); it('can train and predict using array of input and output, two input, 1 to 5, and 5 to 1', () => { const net = new LSTMTimeStep({ inputSize: 2, hiddenLayers: [20], outputSize: 2, }); // Same test as previous, but combined on a single set const trainingData = [ { input: [ [0.1, 0.5], [0.2, 0.4], [0.3, 0.3], [0.4, 0.2], ], output: [[0.5, 0.1]], }, ]; const result = net.train(trainingData, { errorThresh: 0.01 }); expect(result.error).toBeLessThan(0.01); const closeToFiveAndOne = net.run([ [0.1, 0.5], [0.2, 0.4], [0.3, 0.3], [0.4, 0.2], ]); expect(closeToFiveAndOne[0].toFixed(1)).toBe('0.5'); expect(closeToFiveAndOne[1].toFixed(1)).toBe('0.1'); }); }); }); }); describe('.trainArrayOfArray()', () => { describe('when preparing equation length', () => { let bindEquationSpy: jest.SpyInstance; beforeEach(() => { bindEquationSpy = jest.spyOn(RNNTimeStep.prototype, 'bindEquation'); }); afterEach(() => { bindEquationSpy.mockRestore(); }); it('calls .bindEquation() to match the input length', () => { const net = new RNNTimeStep({ inputSize: 1, outputSize: 1 }); net.initialize(); net.trainArrayOfArray([ Float32Array.from([1]), Float32Array.from([1]), Float32Array.from([1]), ]); expect(bindEquationSpy).toHaveBeenCalledTimes(3); }); }); describe('when reading in input', () => { let predictTargetSpy: jest.SpyInstance; beforeEach(() => { predictTargetSpy = jest.spyOn(Equation.prototype, 'predictTarget'); }); afterEach(() => { predictTargetSpy.mockRestore(); }); it('calls .predictTarget() with expected current and next values from input argument', () => { const net = new RNNTimeStep({ inputSize: 1, outputSize: 1 }); net.initialize(); net.trainArrayOfArray([ Float32Array.from([1]), Float32Array.from([2]), Float32Array.from([3]), ]); expect(predictTargetSpy.mock.calls.length).toBe(2); expect(predictTargetSpy.mock.calls[0]).toEqual([ Float32Array.from([1]), Float32Array.from([2]), ]); expect(predictTargetSpy.mock.calls[1]).toEqual([ Float32Array.from([2]), Float32Array.from([3]), ]); }); }); describe('after reading in input', () => { let endSpy: jest.SpyInstance; beforeEach(() => { endSpy = jest.spyOn(RNNTimeStep.prototype, 'end'); }); afterEach(() => { endSpy.mockRestore(); }); it('calls .end()', () => { const net = new RNNTimeStep({ inputSize: 1, outputSize: 1 }); net.initialize(); net.trainArrayOfArray([Float32Array.from([1]), Float32Array.from([1])]); expect(endSpy).toHaveBeenCalledTimes(1); }); }); describe('when given an array of length less than 2', () => { it('throws with descriptive message', () => { const net = new RNNTimeStep({ inputSize: 1, outputSize: 1 }); net.initialize(); expect(() => { net.trainArrayOfArray([Float32Array.from([1])]); }).toThrow('input must be an array of 2 or more'); }); }); it('returns a number that is the error', () => { const net = new RNNTimeStep({ inputSize: 1, outputSize: 1 }); net.initialize(); const error = net.trainArrayOfArray([ Float32Array.from([1]), Float32Array.from([2]), ]); expect(error).toBeGreaterThan(0); }); }); describe('.forecastArray()', () => { it('returns null when this.isRunnable returns false', () => { expect(() => { new RNNTimeStep().forecastArray(Float32Array.from([1])); }).toThrow(); }); it('sets up equations for length of input plus count plus 1 for internal of 0', () => { const net = new RNNTimeStep({ inputSize: 1, hiddenLayers: [1], outputSize: 1, }); net.initialize(); net.bindEquation(); expect(net.model.equations.length).toBe(1); net.forecastArray(Float32Array.from([1, 2, 3]), 2); expect(net.model.equations.length).toBe(6); }); it('sets calls this.end() after calls equations.runInput', () => { const net = new RNNTimeStep({ inputSize: 1, hiddenLayers: [1], outputSize: 1, }); const stub = (net.end = jest.fn()); net.initialize(); net.bindEquation(); net.forecastArray(Float32Array.from([1, 2, 3]), 2); expect(stub).toBeCalled(); }); it('outputs the length of required forecast', () => { const net = new RNNTimeStep({ inputSize: 1, hiddenLayers: [1], outputSize: 1, }); net.initialize(); net.bindEquation(); const result = net.forecastArray(Float32Array.from([1, 2, 3]), 2); expect(result.length).toBe(2); }); it('outputs a flat array of numbers', () => { const net = new RNNTimeStep({ inputSize: 1, hiddenLayers: [1], outputSize: 1, }); net.initialize(); net.bindEquation(); const result = net.forecastArray(Float32Array.from([1, 2, 3]), 2); expect(typeof result[0]).toBe('number'); expect(typeof result[1]).toBe('number'); }); }); describe('.forecastArrayOfArray', () => { it('returns null when this.isRunnable returns false', () => { expect(() => { new RNNTimeStep().forecastArrayOfArray([Float32Array.from([1])]); }).toThrow(); }); it('sets up equations for length of input plus count plus 1 for internal of 0', () => { const net = new RNNTimeStep({ inputSize: 3, hiddenLayers: [1], outputSize: 3, }); net.initialize(); net.bindEquation(); expect(net.model.equations.length).toBe(1); net.forecastArrayOfArray([Float32Array.from([1, 2, 3])], 2); expect(net.model.equations.length).toBe(4); }); it('sets calls this.end() after calls equations.runInput', () => { const net = new RNNTimeStep({ inputSize: 3, hiddenLayers: [1], outputSize: 3, }); const stub = (net.end = jest.fn()); net.initialize(); net.bindEquation(); net.forecastArrayOfArray([Float32Array.from([1, 2, 3])], 2); expect(stub).toBeCalled(); }); it('outputs the length of required forecast', () => { const net = new RNNTimeStep({ inputSize: 3, hiddenLayers: [1], outputSize: 3, }); net.initialize(); net.bindEquation(); const result = net.forecastArrayOfArray( [Float32Array.from([1, 2, 3])], 2 ); expect(result.length).toBe(2); }); it('outputs a nested array of numbers', () => { const net = new RNNTimeStep({ inputSize: 3, hiddenLayers: [1], outputSize: 3, }); net.initialize(); net.bindEquation(); const result = net.forecastArrayOfArray( [Float32Array.from([1, 2, 3])], 2 ); expect(result.length).toBe(2); expect(result[0].length).toBe(3); expect(result[1].length).toBe(3); expect(typeof result[0][0]).toBe('number'); expect(typeof result[0][1]).toBe('number'); expect(typeof result[0][2]).toBe('number'); expect(typeof result[1][0]).toBe('number'); expect(typeof result[1][1]).toBe('number'); expect(typeof result[1][2]).toBe('number'); }); }); describe('.forecastArrayOfObject()', () => { let forecastArrayObjectSpy: jest.SpyInstance; beforeEach(() => { forecastArrayObjectSpy = jest.spyOn( RNNTimeStep.prototype, 'forecastArrayOfObject' ); }); afterEach(() => { forecastArrayObjectSpy.mockRestore(); }); it('maps values correctly', () => { const trainingData = [ [ { low: 0.1, high: 0.9 }, { low: 0.2, high: 0.8 }, { low: 0.3, high: 0.7 }, ], [ { low: 0.9, high: 0.1 }, { low: 0.8, high: 0.2 }, { low: 0.7, high: 0.3 }, ], ]; const net = new RNNTimeStep({ inputSize: 2, outputSize: 2, }); net.train(trainingData, { iterations: 1000 }); const result = net.forecast([{ low: 0.1, high: 0.9 }], 2); expect(result.length).toBe(2); expect(result[0].low).toBeGreaterThan(0); expect(result[0].high).toBeGreaterThan(0); expect(result[1].low).toBeGreaterThan(0); expect(result[1].high).toBeGreaterThan(0); }); }); describe('.forecast()', () => { describe('when called with unrecognized data shape', () => { it('throws', () => { expect(() => { const net = new RNNTimeStep(); net.train([[1, 2, 3]], { iterations: 1 }); // @ts-expect-error need to infer types net.forecast({ one: [1] }, 2); }).toThrow('Unrecognized data shape object,array,number'); }); }); describe('when called with array,number', () => { let forecastArraysSpy: jest.SpyInstance; beforeEach(() => { forecastArraysSpy = jest.spyOn(RNNTimeStep.prototype, 'forecastArray'); }); afterEach(() => { forecastArraysSpy.mockRestore(); }); it('calls this.forecastArray with input and count', () => { const net = new RNNTimeStep(); net.train([[1, 2, 3]], { iterations: 1 }); net.forecast([1], 2); expect(forecastArraysSpy).toBeCalledWith([1], 2); }); }); describe('when called with array,array,number', () => { let forecastArraysOfArraySpy: jest.SpyInstance; beforeEach(() => { forecastArraysOfArraySpy = jest.spyOn( RNNTimeStep.prototype, 'forecastArrayOfArray' ); }); afterEach(() => { forecastArraysOfArraySpy.mockRestore(); }); it('calls this.forecastArrayOfArray with input and count', () => { const net = new RNNTimeStep(); net.train( [ [ [1, 2, 3], [4, 5, 6], [7, 8, 9], ], ], { iterations: 1 } ); net.forecast([[1, 2, 3]], 2); expect(forecastArraysOfArraySpy).toBeCalledWith([[1, 2, 3]], 2); }); }); describe('when called with array,object,number', () => { let forecastArrayOfObjectSpy: jest.SpyInstance; beforeEach(() => { forecastArrayOfObjectSpy = jest.spyOn( RNNTimeStep.prototype, 'forecastArrayOfObject' ); }); afterEach(() => { forecastArrayOfObjectSpy.mockRestore(); }); it('calls this.forecastArrayOfObject with input and count', () => { const net = new RNNTimeStep(); net.train( [ [ { low: 1, high: 2, med: 3 }, { low: 4, high: 5, med: 6 }, { low: 7, high: 8, med: 9 }, ], ], { iterations: 1 } ); net.forecast([{ low: 1, high: 2, med: 3 }], 2); expect(forecastArrayOfObjectSpy).toBeCalledWith( [{ low: 1, high: 2, med: 3 }], 2 ); }); }); }); describe('.formatData()', () => { describe('when called with array,number data shape', () => { let formatArraySpy: jest.SpyInstance; beforeEach(() => { formatArraySpy = jest.spyOn(RNNTimeStep.prototype, 'formatArray'); }); afterEach(() => { formatArraySpy.mockRestore(); }); it('calls this.formatNumber with data', () => { const net = new RNNTimeStep(); const data = [1]; net.formatData(data); expect(formatArraySpy).toHaveBeenCalledWith(data); }); }); describe('when called with array,array,number data shape', () => { let formatArrayOfArraySpy: jest.SpyInstance; beforeEach(() => { formatArrayOfArraySpy = jest.spyOn( RNNTimeStep.prototype, 'formatArrayOfArray' ); }); afterEach(() => { formatArrayOfArraySpy.mockRestore(); }); it('calls this.formatArrayOfArray with data', () => { const net = new RNNTimeStep({ inputSize: 1 }); const data = [[1]]; net.formatData(data); expect(formatArrayOfArraySpy).toHaveBeenCalledWith(data); }); }); describe('when called with array,object,number data shape', () => { describe('when this.inputSize = 1', () => { let formatArrayOfObjectSpy: jest.SpyInstance; beforeEach(() => { formatArrayOfObjectSpy = jest.spyOn( RNNTimeStep.prototype, 'formatArrayOfObject' ); }); afterEach(() => { formatArrayOfObjectSpy.mockRestore(); }); it('calls this.formatArrayOfObject with data', () => { const net = new RNNTimeStep({ inputSize: 1 }); const data = [{ low: 1, high: 2 }]; net.formatData(data); expect(formatArrayOfObjectSpy).toHaveBeenCalledWith(data); }); }); describe('when this.inputSize > 1', () => { let formatArrayOfObjectMultiSpy: jest.SpyInstance; beforeEach(() => { formatArrayOfObjectMultiSpy = jest.spyOn( RNNTimeStep.prototype, 'formatArrayOfObjectMulti' ); }); afterEach(() => { formatArrayOfObjectMultiSpy.mockRestore(); }); it('calls this.formatArrayOfObjectMulti with data', () => { const net = new RNNTimeStep({ inputSize: 2 }); const data = [{ low: 1, high: 2 }]; net.formatData(data); expect(formatArrayOfObjectMultiSpy).toHaveBeenCalledWith(data); }); }); }); describe('when called with array,datum,array,number data shape', () => { let formatArrayOfDatumOfArraySpy: jest.SpyInstance; beforeEach(() => { formatArrayOfDatumOfArraySpy = jest.spyOn( RNNTimeStep.prototype, 'formatArrayOfDatumOfArray' ); }); afterEach(() => { formatArrayOfDatumOfArraySpy.mockRestore(); }); it('calls this.formatArrayOfDatumOfArray with data', () => { const net = new RNNTimeStep(); const data = [ { input: [1, 2], output: [3, 4], }, ]; net.formatData(data); expect(formatArrayOfDatumOfArraySpy).toHaveBeenCalledWith(data); }); }); describe('when called with array,datum,object,number data shape', () => { let formatArrayOfDatumOfObjectSpy: jest.SpyInstance; beforeEach(() => { formatArrayOfDatumOfObjectSpy = jest.spyOn( RNNTimeStep.prototype, 'formatArrayOfDatumOfObject' ); }); afterEach(() => { formatArrayOfDatumOfObjectSpy.mockRestore(); }); it('calls this.formatArrayOfDatumOfArray with data', () => { const net = new RNNTimeStep(); const data = [ { input: { low: 1, high: 2 }, output: { low: 3, high: 4 }, }, ]; net.formatData(data); expect(formatArrayOfDatumOfObjectSpy).toHaveBeenCalledWith(data); }); }); describe('when called with array,array,array,number data shape', () => { let formatArrayOfArrayOfArraySpy: jest.SpyInstance; beforeEach(() => { formatArrayOfArrayOfArraySpy = jest.spyOn( RNNTimeStep.prototype, 'formatArrayOfArrayOfArray' ); }); afterEach(() => { formatArrayOfArrayOfArraySpy.mockRestore(); }); it('calls this.formatArrayOfArrayOfArray with data', () => { const net = new RNNTimeStep(); const data = [[[1, 2, 3]], [[3, 4, 5]]]; net.formatData(data); expect(formatArrayOfArrayOfArraySpy).toHaveBeenCalledWith(data); }); }); describe('when called with array,array,object,number data shape', () => { let formatArrayOfArrayOfObjectSpy: jest.SpyInstance; beforeEach(() => { formatArrayOfArrayOfObjectSpy = jest.spyOn( RNNTimeStep.prototype, 'formatArrayOfArrayOfObject' ); }); afterEach(() => { formatArrayOfArrayOfObjectSpy.mockRestore(); }); it('calls this.formatArrayOfArrayOfObject with data', () => { const net = new RNNTimeStep(); const data = [ [ { h: 1, l: 2, m: 3 }, { h: 3, l: 2, m: 3 }, ], [ { h: 3, l: 4, m: 5 }, { h: 4, l: 4, m: 4 }, ], ]; net.formatData(data); expect(formatArrayOfArrayOfObjectSpy).toHaveBeenCalledWith(data); }); }); describe('when called with array,datum,array,array,number data shape', () => { let formatArrayOfDatumOfArrayOfArraySpy: jest.SpyInstance; beforeEach(() => { formatArrayOfDatumOfArrayOfArraySpy = jest.spyOn( RNNTimeStep.prototype, 'formatArrayOfDatumOfArrayOfArray' ); }); afterEach(() => { formatArrayOfDatumOfArrayOfArraySpy.mockRestore(); }); it('calls this.formatArrayOfArrayOfObject with data', () => { const net = new RNNTimeStep({ inputSize: 2, outputSize: 2, }); const data = [ { input: [ [1, 2], [3, 4], ], output: [ [3, 4], [2, 1], ], }, ]; net.formatData(data); expect(formatArrayOfDatumOfArrayOfArraySpy).toHaveBeenCalledWith(data); }); }); describe('when called with array,datum,array,object,number data shape', () => { let formatArrayOfDatumOfArrayOfObjectSpy: jest.SpyInstance; beforeEach(() => { formatArrayOfDatumOfArrayOfObjectSpy = jest.spyOn( RNNTimeStep.prototype, 'formatArrayOfDatumOfArrayOfObject' ); }); afterEach(() => { formatArrayOfDatumOfArrayOfObjectSpy.mockRestore(); }); it('calls this.formatArrayOfDatumOfArrayOfObject with data', () => { const net = new RNNTimeStep(); const data = [ { input: [ { h: 1, l: 2 }, { h: 1, l: 2 }, ], output: [ { h: 2, l: 1 }, { h: 2, l: 1 }, ], }, ]; net.formatData(data); expect(formatArrayOfDatumOfArrayOfObjectSpy).toHaveBeenCalledWith(data); }); }); }); describe('.formatArray()', () => { it('returns a proper Float32Array[][]', () => { const net = new RNNTimeStep(); const result = net.formatArray([1, 2, 3]); expect(result).toEqual([ [[1], [2], [3]].map((v) => Float32Array.from(v)), ]); }); }); describe('.formatArrayOfArray()', () => { describe('when this.options.inputSize and this.options.outputSize = 1', () => { it('returns a proper Float32Array[][]', () => { const net = new RNNTimeStep(); const result = net.formatArrayOfArray([[1, 2, 3]]); expect(result).toEqual([ [[1], [2], [3]].map((v) => Float32Array.from(v)), ]); }); }); describe('when this.options.inputSize and this.options.outputSize > 1', () => { describe('when inputSize does not match data length', () => { const net = new RNNTimeStep({ inputSize: 2, outputSize: 3 }); it('throws', () => { expect(() => { net.formatArrayOfArray([[1, 2, 3]]); }).toThrow('inputSize must match data input size'); }); }); describe('when outputSize does not match data length', () => { const net = new RNNTimeStep({ inputSize: 3, outputSize: 2 }); it('throws', () => { expect(() => { net.formatArrayOfArray([[1, 2, 3]]); }).toThrow('outputSize must match data output size'); }); }); it('returns a proper Float32Array[][]', () => { const net = new RNNTimeStep({ inputSize: 3, outputSize: 3 }); const result = net.formatArrayOfArray([[1, 2, 3]]); expect(result).toEqual([[[1, 2, 3]].map((v) => Float32Array.from(v))]); }); }); }); describe('.formatArrayOfObject()', () => { describe('when this.options.inputSize > 1', () => { it('throws', () => { const net = new RNNTimeStep({ inputSize: 2, outputSize: 1 }); expect(() => { net.formatArrayOfObject([{ a: 1 }]); }).toThrow('inputSize must be 1 for this data size'); }); }); describe('when this.options.outputSize > 1', () => { it('throws', () => { const net = new RNNTimeStep({ inputSize: 1, outputSize: 2 }); expect(() => { net.formatArrayOfObject([{ a: 1 }]); }).toThrow('outputSize must be 1 for this data size'); }); }); describe('when this.inputLookup is null', () => { it('sets this.inputLookup & this.inputLookupLength', () => { const net = new RNNTimeStep(); expect(net.inputLookup).toBe(null); expect(net.inputLookupLength).toBe(0); net.formatArrayOfObject([{ a: 1 }]); expect(net.inputLookup).toEqual({ a: 0 }); expect(net.inputLookupLength).toBe(1); }); }); describe('when this.inputLookup is set', () => { it('does not set this.inputLookup or this.inputLookupLength', () => { const net = new RNNTimeStep(); const inputLookup = { a: 0 }; net.inputLookup = inputLookup; net.inputLookupLength = 2; net.formatArrayOfObject([{ a: 1 }]); expect(net.inputLookup).toBe(inputLookup); expect(net.inputLookupLength).toBe(2); }); }); it('returns a proper Float32Array[][]', () => { const net = new RNNTimeStep(); const result = net.formatArrayOfObject([{ one: 1, two: 2, three: 3 }]); expect(result).toEqual([ [[1], [2], [3]].map((v) => Float32Array.from(v)), ]); }); }); describe('.formatArrayOfObjectMulti()', () => { describe('when this.inputLookup is null', () => { it('sets this.inputLookup & this.inputLookupLength', () => { const net = new RNNTimeStep(); expect(net.inputLookup).toBe(null); expect(net.inputLookupLength).toBe(0); net.formatArrayOfObjectMulti([{ a: 1, b: 2 }]); expect(net.inputLookup).toEqual({ a: 0, b: 1 }); expect(net.inputLookupLength).toBe(2); }); }); describe('when this.inputLookup is set', () => { it('does not set this.inputLookup or this.inputLookupLength', () => { const net = new RNNTimeStep(); const inputLookup = { a: 0, b: 1 }; net.inputLookup = inputLookup; net.inputLookupLength = 3; net.formatArrayOfObjectMulti([{ a: 1, b: 2 }]); expect(net.inputLookup).toBe(inputLookup); expect(net.inputLookupLength).toBe(3); }); }); it('returns a proper Float32Array[][]', () => { const net = new RNNTimeStep(); const result = net.formatArrayOfObjectMulti([ { one: 1, two: 2, three: 3 }, ]); expect(result).toEqual([[Float32Array.from([1, 2, 3])]]); }); }); describe('.formatArrayOfDatumOfArray()', () => { describe('when this.options.inputSize > 1', () => { it('throws', () => { const net = new RNNTimeStep({ inputSize: 2, outputSize: 1 }); expect(() => { net.formatArrayOfDatumOfArray([]); }).toThrow('inputSize must be 1 for this data size'); }); }); describe('when this.options.outputSize > 1', () => { it('throws', () => { const net = new RNNTimeStep({ inputSize: 1, outputSize: 2 }); expect(() => { net.formatArrayOfDatumOfArray([]); }).toThrow('outputSize must be 1 for this data size'); }); }); it('returns a proper Float32Array[][]', () => { const net = new RNNTimeStep(); const result = net.formatArrayOfDatumOfArray([ { input: [1, 2, 3], output: [4, 5, 6] }, ]); expect(result).toEqual([ [[1], [2], [3], [4], [5], [6]].map((v) => Float32Array.from(v)), ]); }); }); describe('.formatArrayOfDatumOfObject()', () => { describe('when this.options.inputSize > 1', () => { it('throws', () => { const net = new RNNTimeStep({ inputSize: 2, outputSize: 1 }); expect(() => { net.formatArrayOfDatumOfObject([]); }).toThrow('inputSize must be 1 for this data size'); }); }); describe('when this.options.outputSize > 1', () => { it('throws', () => { const net = new RNNTimeStep({ inputSize: 1, outputSize: 2 }); expect(() => { net.formatArrayOfDatumOfObject([]); }).toThrow('outputSize must be 1 for this data size'); }); }); describe('when this.inputLookup is null', () => { it('sets this.inputLookup & this.inputLookupLength', () => { const net = new RNNTimeStep(); expect(net.inputLookup).toBe(null); expect(net.inputLookupLength).toBe(0); net.formatArrayOfDatumOfObject([ { input: { a: 1, b: 2 }, output: { a: 1, b: 2 }, }, ]); expect(net.inputLookup).toEqual({ a: 0, b: 1 }); expect(net.inputLookupLength).toBe(2); }); }); describe('when this.inputLookup is set', () => { it('does not set this.inputLookup or this.inputLookupLength', () => { const net = new RNNTimeStep(); const inputLookup = { a: 0, b: 1 }; net.inputLookup = inputLookup; net.inputLookupLength = 3; net.formatArrayOfDatumOfObject([ { input: { a: 1, b: 2 }, output: { a: 1, b: 2 }, }, ]); expect(net.inputLookup).toBe(inputLookup); expect(net.inputLookupLength).toBe(3); }); }); it('returns a proper Float32Array[][]', () => { const net = new RNNTimeStep(); const result = net.formatArrayOfDatumOfObject([ { input: { a: 1, b: 2 }, output: { a: 1, b: 2 } }, ]); expect(result).toEqual([ [[1], [2], [1], [2]].map((v) => Float32Array.from(v)), ]); }); }); describe('.formatArrayOfArrayOfArray()', () => { it('returns a proper Float32Array[][]', () => { const net = new RNNTimeStep(); const result = net.formatArrayOfArrayOfArray([ [ [1, 2, 3, 4], [4, 3, 2, 1], ], ]); expect(result).toEqual([ [ [1, 2, 3, 4], [4, 3, 2, 1], ].map((v) => Float32Array.from(v)), ]); }); }); describe('.formatArrayOfArrayOfObject()', () => { describe('when this.inputLookup is null', () => { it('sets this.inputLookup & this.inputLookupLength', () => { const net = new RNNTimeStep(); expect(net.inputLookup).toBe(null); expect(net.inputLookupLength).toBe(0); net.formatArrayOfArrayOfObject([ [ { a: 1, b: 2 }, { a: 2, b: 1 }, ], ]); expect(net.inputLookup).toEqual({ a: 0, b: 1 }); expect(net.inputLookupLength).toBe(2); }); }); describe('when this.inputLookup is set', () => { it('does not set this.inputLookup or this.inputLookupLength', () => { const net = new RNNTimeStep(); const inputLookup = { a: 0, b: 1 }; net.inputLookup = inputLookup; net.inputLookupLength = 3; net.formatArrayOfArrayOfObject([ [ { a: 1, b: 2 }, { a: 2, b: 1 }, ], ]); expect(net.inputLookup).toBe(inputLookup); expect(net.inputLookupLength).toBe(3); }); }); it('returns a proper Float32Array[][]', () => { const net = new RNNTimeStep(); const result = net.formatArrayOfArrayOfObject([ [ { a: 1, b: 2 }, { a: 2, b: 1 }, ], ]); expect(result).toEqual([ [ [1, 2], [2, 1], ].map((v) => Float32Array.from(v)), ]); }); }); describe('.formatArrayOfDatumOfArrayOfArray()', () => { describe('when inputSize does not match data length', () => { const net = new RNNTimeStep({ inputSize: 2, outputSize: 3 }); it('throws', () => { expect(() => { net.formatArrayOfDatumOfArrayOfArray([ { input: [[1, 2, 3]], output: [[1, 2, 3]] }, ]); }).toThrow('inputSize must match data input size'); }); }); describe('when outputSize does not match data length', () => { const net = new RNNTimeStep({ inputSize: 3, outputSize: 2 }); it('throws', () => { expect(() => { net.formatArrayOfDatumOfArrayOfArray([ { input: [[1, 2, 3]], output: [[1, 2, 3]] }, ]); }).toThrow('outputSize must match data output size'); }); }); it('returns a proper Float32Array[][]', () => { const net = new RNNTimeStep({ inputSize: 2, outputSize: 2 }); const result = net.formatArrayOfDatumOfArrayOfArray([ { input: [ [1, 2], [3, 4], ], output: [ [4, 3], [2, 1], ], }, { input: [ [4, 3], [2, 1], ], output: [ [1, 2], [3, 4], ], }, ]); expect(result).toEqual([ [ [1, 2], [3, 4], [4, 3], [2, 1], ].map((v) => Float32Array.from(v)), [ [4, 3], [2, 1], [1, 2], [3, 4], ].map((v) => Float32Array.from(v)), ]); }); }); describe('.toFunction()', () => { it('processes array same as net w/ inputSize of 1', () => { const data = [{ input: [1, 2], output: [3, 4] }]; const net = new LSTMTimeStep({ inputSize: 1, hiddenLayers: [10], outputSize: 1, }); net.train(data, { iterations: 100, errorThresh: 0.05 }); const fn = net.toFunction(); const expected = net.run(data[0].input); const result = fn(data[0].input); expect(typeof result).toBe('number'); expect(result).toEqual(expected); }); it('processes object same as net w/ inputSize of 1', () => { const data = [{ input: { a: 1, b: 2 }, output: { c: 3, d: 4 } }]; const net = new LSTMTimeStep({ inputSize: 1, hiddenLayers: [10], outputSize: 1, }); net.train(data, { iterations: 100, errorThresh: 0.05 }); const fn = net.toFunction(); const expected = net.run(data[0].input); expect(fn(data[0].input)).toEqual(expected); }); it('processes array,object same as net', () => { const data = [ { input: [ { a: 1, b: 4 }, { a: 2, b: 3 }, ], output: [ { c: 3, d: 2 }, { c: 4, d: 1 }, ], }, ]; const net = new LSTMTimeStep({ inputSize: 2, hiddenLayers: [10], outputSize: 2, }); net.train(data, { iterations: 100, errorThresh: 0.05 }); const fn = net.toFunction(); const expected = net.run(data[0].input); expect(fn(data[0].input)).toEqual(expected); }); it('processes array same as net', () => { const net = new LSTMTimeStep({ inputSize: 1, hiddenLayers: [10], outputSize: 1, }); // Same test as previous, but combined on a single set const trainingData = [ [0.1, 0.2, 0.3, 0.4, 0.5], [0.5, 0.4, 0.3, 0.2, 0.1], ]; const trainResult = net.train(trainingData); expect(trainResult.error).toBeLessThan(0.09); const closeToFive = net.run([0.1, 0.2, 0.3, 0.4]); const closeToOne = net.run([0.5, 0.4, 0.3, 0.2]); const fn = net.toFunction(); expect(closeToFive.toFixed(1)).toBe('0.5'); expect(closeToOne.toFixed(1)).toBe('0.1'); expect(fn([0.1, 0.2, 0.3, 0.4])).toBeCloseTo(closeToFive); expect(fn([0.5, 0.4, 0.3, 0.2])).toBeCloseTo(closeToOne); }); it('processes array,array same as net', () => { const net = new LSTMTimeStep({ inputSize: 2, hiddenLayers: [10], outputSize: 2, }); // Same test as previous, but combined on a single set const trainingData = [ [0.1, 0.5], [0.2, 0.4], [0.3, 0.3], [0.4, 0.2], [0.5, 0.1], ]; const trainResult = net.train(trainingData); expect(trainResult.error).toBeLessThan(0.09); const closeToFiveAndOne = net.run([ [0.1, 0.5], [0.2, 0.4], [0.3, 0.3], [0.4, 0.2], ]); const fn = net.toFunction(); const result = fn([ [0.1, 0.5], [0.2, 0.4], [0.3, 0.3], [0.4, 0.2], ]); expect(closeToFiveAndOne[0].toFixed(1)).toBe('0.5'); expect(closeToFiveAndOne[1].toFixed(1)).toBe('0.1'); expect(result[0]).toBe(closeToFiveAndOne[0]); expect(result[1]).toBe(closeToFiveAndOne[1]); }); it('processes object same as net', () => { const net = new LSTMTimeStep({ inputSize: 1, hiddenLayers: [10], outputSize: 1, }); // Same test as previous, but combined on a single set const trainingData = [ { input: { monday: 0.1, tuesday: 0.2, wednesday: 0.3, thursday: 0.4 }, output: { friday: 0.5 }, }, { input: { monday: 0.5, tuesday: 0.4, wednesday: 0.3, thursday: 0.2 }, output: { friday: 0.1 }, }, ]; const trainResult = net.train(trainingData); expect(trainResult.error).toBeLessThan(0.09); const closeToFive = net.runObject({ monday: 0.1, tuesday: 0.2, wednesday: 0.3, thursday: 0.4, }); const closeToOne = net.runObject({ monday: 0.5, tuesday: 0.4, wednesday: 0.3, thursday: 0.2, }); const fn = net.toFunction(); expect(closeToFive.friday.toFixed(1)).toBe('0.5'); expect(closeToOne.friday.toFixed(1)).toBe('0.1'); expect( (fn as (input: INumberHash) => INumberHash)({ monday: 0.1, tuesday: 0.2, wednesday: 0.3, thursday: 0.4, }).friday ).toBe(closeToFive.friday); expect( (fn as (input: INumberHash) => INumberHash)({ monday: 0.5, tuesday: 0.4, wednesday: 0.3, thursday: 0.2, }).friday ).toBe(closeToOne.friday); }); it('handles array,object to array,object with lookup tables being same w/ inputSize of 1', () => { const inputSize = 1; const hiddenLayers = [10]; const outputSize = 1; const net = new RNNTimeStep({ inputSize, hiddenLayers, outputSize, }); net.train([ { monday: 1, tuesday: 2, wednesday: 3, thursday: 4, friday: 5 }, ]); const fn = net.toFunction(); const result = (fn as (input: INumberHash) => INumberHash)({ monday: 1, tuesday: 2, wednesday: 3, thursday: 4, }); expect(result).toEqual( net.run({ monday: 1, tuesday: 2, wednesday: 3, thursday: 4 }) ); expect(Object.keys(result).length).toBe(1); expect(result.friday.toFixed(0)).toBe('5'); }); }); describe('.test()', () => { let runSpy: jest.SpyInstance; beforeEach(() => { runSpy = jest.spyOn(LSTMTimeStep.prototype, 'run'); }); afterEach(() => { runSpy.mockRestore(); }); describe('with any data shape', () => { let formatDataSpy: jest.SpyInstance; beforeEach(() => { formatDataSpy = jest.spyOn(RNNTimeStep.prototype, 'formatData'); }); afterEach(() => { formatDataSpy.mockRestore(); }); it('calls .formatData()', () => { const data = [[1, 2]]; const net = new RNNTimeStep(); net.train(data); formatDataSpy.mockClear(); net.test(data); expect(formatDataSpy).toHaveBeenCalledWith(data); }); }); describe('using array,array,number', () => { const trainingData = [[0.1, 0.2, 0.3, 0.4, 0.5]]; describe('inputSize of 1', () => { it('accumulates no error or misclasses when no error', () => { const net = new LSTMTimeStep({ inputSize: 1, hiddenLayers: [10], outputSize: 1, }); net.train(trainingData, { iterations: 500 }); const testResult = net.test(trainingData); expect(testResult.error).toBeLessThan(0.001); expect(testResult.misclasses.length).toBe(0); }); it('accumulates error and misclasses when error', () => { const net = new LSTMTimeStep({ inputSize: 1, hiddenLayers: [10], outputSize: 1, }); net.train(trainingData, { iterations: 500 }); const misclass = [1, 2, 3, 4, 5]; const testResult = net.test([misclass]); expect(testResult.error).toBeGreaterThan(0.1); expect(testResult.misclasses.length).toBe(1); expect(testResult.misclasses).toEqual([ { value: misclass, actual: runSpy.mock.results[0].value, }, ]); }); }); describe('inputSize of 2', () => { it('throws', () => { const net = new LSTMTimeStep({ inputSize: 2, hiddenLayers: [10], outputSize: 5, }); expect(() => { net.test(trainingData); }).toThrow('inputSize must match data input size'); }); }); describe('outputSize of 2', () => { it('throws', () => { const net = new LSTMTimeStep({ inputSize: 5, hiddenLayers: [10], outputSize: 2, }); expect(() => { net.test(trainingData); }).toThrow('outputSize must match data output size'); }); }); }); describe('using array,array,array,number', () => { const trainingData = [ [ [0.1, 0.5], [0.2, 0.4], [0.3, 0.3], [0.4, 0.2], [0.5, 0.1], ], ]; describe('inputSize of 2', () => { describe('no error', () => { it('can test', () => { const net = new LSTMTimeStep({ inputSize: 2, hiddenLayers: [10], outputSize: 2, }); net.train(trainingData, { iterations: 500 }); const testResult = net.test(trainingData); expect(testResult.error).toBeLessThan(0.001); expect(testResult.misclasses.length).toBe(0); }); }); describe('some error', () => { it('can test', () => { const net = new LSTMTimeStep({ inputSize: 2, hiddenLayers: [10], outputSize: 2, }); net.train(trainingData, { iterations: 500 }); const misclass = [ [1, 5], [2, 4], [3, 3], [4, 2], [5, 1], ]; const testResult = net.test([misclass]); expect(testResult.error).toBeGreaterThanOrEqual(0.1); expect(testResult.misclasses.length).toBe(1); expect(testResult.misclasses).toEqual([ { value: misclass, actual: runSpy.mock.results[0].value, }, ]); }); }); }); }); describe('using array,object,number', () => { const trainingData = [ { monday: 0.1, tuesday: 0.1, wednesday: 0.2, thursday: 0.3, friday: 0.4, }, ]; describe('inputSize of 1', () => { describe('no error', () => { it('can test w/ forecastNumbers of 1', () => { const net = new LSTMTimeStep({ inputSize: 1, hiddenLayers: [10], outputSize: 1, }); net.train(trainingData, { iterations: 500 }); const testResult = net.test(trainingData); expect(testResult.error).toBeLessThan(0.001); expect(testResult.misclasses.length).toBe(0); }); }); describe('some error', () => { it('can test w/ forecastNumbers of 1', () => { const net = new LSTMTimeStep({ inputSize: 1, hiddenLayers: [10], outputSize: 1, }); net.train(trainingData, { iterations: 500 }); const misclass = { monday: 1, tuesday: 2, wednesday: 3, thursday: 4, friday: 5, }; const testResult = net.test([misclass]); expect(testResult.error).toBeGreaterThanOrEqual(0.08); expect(testResult.misclasses.length).toBe(1); expect(testResult.misclasses).toEqual([ { value: misclass, actual: runSpy.mock.results[0].value, }, ]); }); }); }); }); describe('using array,array,object,number', () => { const trainingData = [ [ { low: 0.1, high: 0.5 }, { low: 0.2, high: 0.4 }, { low: 0.3, high: 0.3 }, { low: 0.4, high: 0.2 }, { low: 0.5, high: 0.1 }, ], ]; describe('inputSize of 2', () => { describe('no error', () => { it('can test w/ run of 1', () => { const net = new LSTMTimeStep({ inputSize: 2, hiddenLayers: [10], outputSize: 2, }); net.train(trainingData); const testResult = net.test(trainingData); expect(testResult.error).toBeLessThan(0.001); expect(testResult.misclasses.length).toBe(0); }); }); describe('some error', () => { it('can test w/ run of 1', () => { const net = new LSTMTimeStep({ inputSize: 2, hiddenLayers: [10], outputSize: 2, }); net.train(trainingData, { iterations: 500 }); const misclass = [ { low: 1, high: 5 }, { low: 2, high: 4 }, { low: 3, high: 3 }, { low: 4, high: 2 }, { low: 5, high: 1 }, ]; const testResult = net.test([misclass]); expect(testResult.error).toBeGreaterThan(0.3); expect(testResult.misclasses.length).toBe(1); expect(testResult.misclasses).toEqual([ { value: misclass, actual: runSpy.mock.results[0].value, }, ]); }); }); }); }); describe('using array,datum,array,number', () => { const trainingData = [{ input: [0.1, 0.2, 0.3, 0.4], output: [0.5] }]; describe('no error', () => { it('can test w/ forecast of 1', () => { const net = new LSTMTimeStep({ inputSize: 1, hiddenLayers: [10], outputSize: 1, }); net.train(trainingData, { iterations: 500 }); const testResult = net.test(trainingData); expect(testResult.error).toBeLessThan(0.001); expect(testResult.misclasses.length).toBe(0); }); }); describe('some error', () => { it('can test w/ forecast of 1', () => { const net = new LSTMTimeStep({ inputSize: 1, hiddenLayers: [10], outputSize: 1, }); net.train(trainingData, { iterations: 500 }); const misclass = { input: [1, 2, 3, 4], output: [5] }; const testResult = net.test([misclass]); expect(testResult.error).toBeGreaterThanOrEqual(0.08); expect(testResult.misclasses.length).toBe(1); expect(testResult.misclasses).toEqual([ { value: misclass, actual: runSpy.mock.results[0].value, }, ]); }); }); }); describe('using array,datum,object,number', () => { const trainingData = [ { input: { monday: 0.1, tuesday: 0.2, wednesday: 0.3, thursday: 0.4, }, output: { friday: 0.5 }, }, ]; describe('inputSize of 1', () => { describe('no error', () => { it('can test w/ forecastNumbers of 1', () => { const net = new LSTMTimeStep({ inputSize: 1, hiddenLayers: [10], outputSize: 1, }); net.train(trainingData, { iterations: 500 }); const testResult = net.test(trainingData); expect(testResult.error).toBeLessThan(0.001); expect(testResult.misclasses.length).toBe(0); }); }); describe('some error', () => { it('can test w/ forecastNumbers of 1', () => { const net = new LSTMTimeStep({ inputSize: 1, hiddenLayers: [10], outputSize: 1, }); net.train(trainingData); const misclass = { input: { monday: 1, tuesday: 2, wednesday: 3, thursday: 4, }, output: { friday: 5 }, }; const testResult = net.test([misclass]); expect(testResult.error).toBeGreaterThanOrEqual(0.08); expect(testResult.misclasses.length).toBe(1); expect(testResult.misclasses).toEqual([ { value: misclass, actual: runSpy.mock.results[0].value, }, ]); }); }); }); }); describe('using array,datum,array,array', () => { const trainingData1 = [ { input: [ [0.1, 0.5], [0.2, 0.4], [0.3, 0.3], [0.4, 0.2], ], output: [[0.5, 0.1]], }, ]; const trainingData2 = [ { input: [ [0.1, 0.5], [0.2, 0.4], [0.3, 0.3], ], output: [ [0.4, 0.2], [0.5, 0.1], ], }, ]; describe('no error', () => { it('can test w/ forecast of 1', () => { const net = new LSTMTimeStep({ inputSize: 2, hiddenLayers: [10], outputSize: 2, }); net.train(trainingData1, { iterations: 500 }); const testResult = net.test(trainingData1); expect(testResult.error).toBeLessThan(0.001); expect(testResult.misclasses.length).toBe(0); }); it('can test w/ forecast of 2', () => { const net = new LSTMTimeStep({ inputSize: 2, hiddenLayers: [10], outputSize: 2, }); net.train(trainingData2, { iterations: 500 }); const testResult = net.test(trainingData2); expect(testResult.error).toBeLessThan(0.001); expect(testResult.misclasses.length).toBe(0); }); }); describe('some error', () => { it('can test w/ forecast of 1', () => { const net = new LSTMTimeStep({ inputSize: 2, hiddenLayers: [10], outputSize: 2, }); net.train(trainingData1, { iterations: 500 }); const misclass = { input: [ [1, 5], [2, 4], [3, 3], [4, 2], ], output: [[5, 1]], }; const testResult = net.test([misclass]); expect(testResult.error).toBeGreaterThan(0.1); expect(testResult.misclasses.length).toBe(1); expect(testResult.misclasses).toEqual([ { value: misclass, actual: runSpy.mock.results[0].value, }, ]); }); it('can test w/ forecast of 2', () => { const net = new LSTMTimeStep({ inputSize: 2, hiddenLayers: [10], outputSize: 2, }); net.train(trainingData2, { iterations: 500 }); const misclass = { input: [ [1, 5], [2, 4], [3, 3], ], output: [ [4, 2], [5, 1], ], }; const testResult = net.test([misclass]); expect(testResult.error).toBeGreaterThanOrEqual(0.08); expect(testResult.misclasses.length).toBe(1); expect(testResult.misclasses).toEqual([ { value: misclass, actual: runSpy.mock.results[0].value, }, ]); }); }); }); describe('using array,datum,array,object,number', () => { const trainingData1 = [ { input: [ { low: 0.1, high: 0.5 }, { low: 0.2, high: 0.4 }, { low: 0.3, high: 0.3 }, { low: 0.4, high: 0.2 }, ], output: [{ low: 0.5, high: 0.1 }], }, ]; const trainingData2 = [ { input: [ { low: 0.1, high: 0.5 }, { low: 0.2, high: 0.4 }, { low: 0.3, high: 0.3 }, ], output: [ { low: 0.4, high: 0.2 }, { low: 0.5, high: 0.1 }, ], }, ]; describe('no error', () => { it('can test w/ forecast of 1', () => { const net = new LSTMTimeStep({ inputSize: 2, hiddenLayers: [10], outputSize: 2, }); net.train(trainingData1, { iterations: 500 }); const testResult = net.test(trainingData1); expect(testResult.error).toBeLessThan(0.001); expect(testResult.misclasses.length).toBe(0); }); it('can test w/ forecast of 2', () => { const net = new LSTMTimeStep({ inputSize: 2, hiddenLayers: [10], outputSize: 2, }); net.train(trainingData2, { iterations: 500 }); const testResult = net.test(trainingData2); expect(testResult.error).toBeLessThan(0.001); expect(testResult.misclasses.length).toBe(0); }); }); describe('some error', () => { it('can test w/ forecast of 1', () => { const net = new LSTMTimeStep({ inputSize: 2, hiddenLayers: [10], outputSize: 2, }); net.train(trainingData1, { iterations: 500 }); const misclass = { input: [ { low: 1, high: 5 }, { low: 2, high: 4 }, { low: 3, high: 3 }, { low: 4, high: 2 }, ], output: [{ low: 0.5, high: 0.1 }], }; const testResult = net.test([misclass]); expect(testResult.error).toBeGreaterThan(0.1); expect(testResult.misclasses.length).toBe(1); expect(testResult.misclasses).toEqual([ { value: misclass, actual: runSpy.mock.results[0].value, }, ]); }); it('can test w/ forecast of 2', () => { const net = new LSTMTimeStep({ inputSize: 2, hiddenLayers: [10], outputSize: 2, }); net.train(trainingData2, { iterations: 500 }); const misclass = { input: [ { low: 1, high: 5 }, { low: 2, high: 4 }, { low: 3, high: 3 }, ], output: [ { low: 4, high: 2 }, { low: 5, high: 1 }, ], }; const testResult = net.test([misclass]); expect(testResult.error).toBeGreaterThanOrEqual(0.08); expect(testResult.misclasses.length).toBe(1); expect(testResult.misclasses).toEqual([ { value: misclass, actual: runSpy.mock.results[0].value, }, ]); }); }); }); }); describe('.addFormat()', () => { it('array,array,number', () => { const instance = {}; RNNTimeStep.prototype.addFormat.call(instance, [[0]]); expect(instance).toEqual({}); }); it('datum,array,array,number', () => { const instance = {}; RNNTimeStep.prototype.addFormat.call(instance, { input: [[0]], output: [[0]], }); expect(instance).toEqual({}); }); it('array,number', () => { const instance = {}; RNNTimeStep.prototype.addFormat.call(instance, [0]); expect(instance).toEqual({}); }); it('datum,array,number', () => { const instance = {}; RNNTimeStep.prototype.addFormat.call(instance, { input: [0], output: [0], }); expect(instance).toEqual({}); }); it('datum,object,number', () => { const instance = { inputLookup: { inputOne: 0 }, outputLookup: { outputOne: 0 }, }; RNNTimeStep.prototype.addFormat.call(instance, { input: { inputTwo: 1, inputThree: 2 }, output: { outputTwo: 1, outputThree: 2 }, }); expect(instance).toEqual({ inputLookup: { inputOne: 0, inputTwo: 1, inputThree: 2 }, inputLookupLength: 3, outputLookup: { outputOne: 0, outputTwo: 1, outputThree: 2 }, outputLookupLength: 3, }); }); it('object,number', () => { const instance = { inputLookup: { inputOne: 0 }, }; RNNTimeStep.prototype.addFormat.call(instance, { inputTwo: 1, inputThree: 2, }); expect(instance).toEqual({ inputLookup: { inputOne: 0, inputTwo: 1, inputThree: 2 }, inputLookupLength: 3, outputLookup: { inputOne: 0, inputTwo: 1, inputThree: 2 }, outputLookupLength: 3, }); }); // it('array,object,number', () => {}); // it('datum,array,object,number', () => {}); }); describe('.toJSON()', () => { it('saves network dimensions to json', () => { const inputSize = 4; const hiddenLayers = [1, 2, 3]; const outputSize = 5; const net = new RNNTimeStep({ inputSize, hiddenLayers, outputSize, }); const { inputLookup, inputLookupLength, outputLookup, outputLookupLength, } = net; net.initialize(); const json = net.toJSON(); expect(json.options.inputSize).toBe(inputSize); expect(json.options.hiddenLayers).toEqual(hiddenLayers); expect(json.options.outputSize).toBe(outputSize); expect(json.inputLookup).toBe(inputLookup); expect(json.inputLookupLength).toBe(inputLookupLength); expect(json.outputLookup).toBe(outputLookup); expect(json.outputLookupLength).toBe(outputLookupLength); }); }); describe('.fromJSON()', () => { it('restores network dimensions from json', () => { const inputSize = 45; const hiddenLayers = [1, 2, 3, 4, 5, 6, 7, 8, 9]; const outputSize = 20; const net = new RNNTimeStep({ inputSize, hiddenLayers, outputSize, }); net.initialize(); const json = net.toJSON(); const { inputLookup, inputLookupLength, outputLookup, outputLookupLength, } = json; const serializedNet = new RNNTimeStep(); serializedNet.fromJSON(json); expect(serializedNet.options.inputSize).toBe(inputSize); expect(serializedNet.options.hiddenLayers).toEqual(hiddenLayers); expect(serializedNet.options.outputSize).toBe(outputSize); expect(serializedNet.inputLookup).toBe(inputLookup); expect(serializedNet.inputLookupLength).toBe(inputLookupLength); expect(serializedNet.outputLookup).toBe(outputLookup); expect(serializedNet.outputLookupLength).toBe(outputLookupLength); }); it('error rate stays same after serialization', () => { const inputSize = 1; const hiddenLayers = [10]; const outputSize = 1; const net = new RNNTimeStep({ inputSize, hiddenLayers, outputSize, }); let lastNetStatus: IRNNStatus = { error: Infinity, iterations: -1 }; const trainingData = [ { monday: 1, tuesday: 2, wednesday: 3, thursday: 4, friday: 5 }, ]; net.train(trainingData, { callback: (status) => { lastNetStatus = status; }, iterations: 50, }); net.run({ monday: 1, tuesday: 2, wednesday: 3, thursday: 4 }); const json = net.toJSON(); const serializedNet = new RNNTimeStep(); serializedNet.fromJSON(json); let lastSerializedNetStatus: IRNNStatus = { error: Infinity, iterations: -1, }; serializedNet.train(trainingData, { iterations: 1, callback: (status: IRNNStatus) => { lastSerializedNetStatus = status; }, }); expect(lastSerializedNetStatus.error).toBeLessThan(lastNetStatus.error); }); }); }); ================================================ FILE: src/recurrent/rnn-time-step.ts ================================================ import { FormattableData, InputOutputValue, INumberArray, INumberHash, ITrainingDatum, lookup, } from '../lookup'; import { ArrayLookupTable } from '../utilities/array-lookup-table'; import { arraysToFloat32Arrays, arrayToFloat32Arrays, inputOutputArraysToFloat32Arrays, inputOutputArrayToFloat32Arrays, inputOutputObjectsToFloat32Arrays, inputOutputObjectToFloat32Arrays, objectToFloat32Array, objectToFloat32Arrays, } from '../utilities/cast'; import { LookupTable } from '../utilities/lookup-table'; import { randomFloat } from '../utilities/random'; import { IMatrixJSON, Matrix } from './matrix'; import { Equation } from './matrix/equation'; import { maxI } from './matrix/max-i'; import { RandomMatrix } from './matrix/random-matrix'; import { sampleI } from './matrix/sample-i'; import { softmax } from './matrix/softmax'; import { defaults as rnnDefaults, IRNNHiddenLayer, IRNNHiddenLayerModel, IRNNOptions, IRNNStatus, IRNNTrainingOptions, last, RNN, trainDefaults as rnnTrainDefaults, } from './rnn'; export type ValuesOf< T extends InputOutputValue | InputOutputValue[] > = T[number]; export interface IRNNTimeStepOptions extends IRNNTimeStepJSONOptions { inputSize: number; inputRange: number; hiddenLayers: number[]; outputSize: number; decayRate: number; smoothEps: number; regc: number; clipval: number; maxPredictionLength: number; json?: IRNNTimeStepJSON; } export interface IRNNTimeStepJSONOptions { inputSize: number; inputRange: number; hiddenLayers: number[]; outputSize: number; decayRate: number; smoothEps: number; regc: number; clipval: number; maxPredictionLength: number; } export interface IRNNTimeStepJSON { type: string; options: IRNNTimeStepJSONOptions; hiddenLayers: Array<{ [index: string]: IMatrixJSON }>; outputConnector: IMatrixJSON; output: IMatrixJSON; inputLookup: INumberHash | null; inputLookupLength: number; outputLookup: INumberHash | null; outputLookupLength: number; } export interface IMisclass { value: FormattableData; actual: FormattableData; } export interface ITestResults { misclasses: IMisclass[]; error: number; total: number; } export interface IRNNTimeStepModel { isInitialized: boolean; hiddenLayers: IRNNHiddenLayer[]; output: Matrix; equations: Equation[]; allMatrices: Matrix[]; equationConnections: Matrix[][]; outputConnector: RandomMatrix | Matrix; } export const defaults = (): IRNNOptions => { return { ...rnnDefaults(), inputSize: 1, hiddenLayers: [20], outputSize: 1, inputRange: 0, }; }; export class RNNTimeStep extends RNN { inputLookupLength = 0; inputLookup: INumberHash | null = null; outputLookup: INumberHash | null = null; outputLookupLength = 0; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error model: IRNNTimeStepModel = Object.seal({ isInitialized: false, hiddenLayers: [], output: new Matrix(0, 0), equations: [], allMatrices: [], equationConnections: [], outputConnector: new RandomMatrix(0, 0, 0.08), }); // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error options: IRNNTimeStepOptions = defaults(); constructor( options: Partial = {} ) { super(); this.options = { ...this.options, ...options }; this.updateTrainingOptions({ ...trainDefaults, ...options, }); if (options.json) { this.fromJSON(options.json); } } createInputMatrix(): RandomMatrix { throw new Error('Input Matrices do not exist on RNNTimeStep'); } createOutputMatrices(): { outputConnector: RandomMatrix; output: Matrix } { const { outputSize } = this.options; const lastHiddenSize = last(this.options.hiddenLayers); // whd const outputConnector = new RandomMatrix(outputSize, lastHiddenSize, 0.08); // bd const output = new RandomMatrix(outputSize, 1, 0.08); return { output, outputConnector }; } bindEquation(): void { const { model, options } = this; const { hiddenLayers, inputSize } = options; const layers = model.hiddenLayers; const equation = new Equation(); const outputs = []; const equationConnection = model.equationConnections.length > 0 ? model.equationConnections[model.equationConnections.length - 1] : this.initialLayerInputs; // 0 index let output = this.getEquation( equation, equation.input(new Matrix(inputSize, 1)), equationConnection[0], layers[0] ); outputs.push(output); // 1+ indices for (let i = 1, max = hiddenLayers.length; i < max; i++) { output = this.getEquation( equation, output, equationConnection[i], layers[i] ); outputs.push(output); } model.equationConnections.push(outputs); equation.add( equation.multiply(model.outputConnector, output), model.output ); model.equations.push(equation); } initialize(): void { this.model = this.mapModel(); } // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error mapModel(): IRNNTimeStepModel { const allMatrices: Matrix[] = []; this.initialLayerInputs = this.options.hiddenLayers.map( (size) => new Matrix(size, 1) ); const hiddenLayers = this.createHiddenLayers(); for (let i = 0, max = hiddenLayers.length; i < max; i++) { const hiddenMatrix = hiddenLayers[i]; for (const property in hiddenMatrix) { if (!hiddenMatrix.hasOwnProperty(property)) continue; allMatrices.push(hiddenMatrix[property]); } } const { outputConnector, output } = this.createOutputMatrices(); allMatrices.push(outputConnector); allMatrices.push(output); return Object.seal({ isInitialized: true, hiddenLayers, output, equations: [], allMatrices, equationConnections: [], outputConnector, }); } backpropagate(): void { for (let i = this.model.equations.length - 1; i > -1; i--) { this.model.equations[i].backpropagate(); } } // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error run( rawInput: InputType ): ValuesOf { const shape = lookup.dataShape(rawInput).join(','); switch (shape) { case 'array,number': // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error return this.runArray(rawInput as Float32Array); case 'array,array,number': // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error return this.runArrayOfArray(rawInput as Float32Array[]); case 'object,number': // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error return this.runObject(rawInput as INumberHash); // Backward compatibility, will be result of `unknown` and need casting. Better to just use net.runObject() directly case 'array,object,number': // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error return this.runArrayOfObject(rawInput as INumberHash[]); default: throw new Error(`Unrecognized data shape ${shape}`); } } forecast( rawInput: InputType, count = 1 ): InputType { const shape = lookup.dataShape(rawInput).join(','); switch (shape) { case 'array,number': // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error return this.forecastArray(rawInput as Float32Array, count); case 'array,array,number': // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error return this.forecastArrayOfArray(rawInput as Float32Array[], count); case 'object,number': // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error return this.runObject(rawInput as INumberHash); case 'array,object,number': // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error return this.forecastArrayOfObject(rawInput as INumberHash[], count); default: throw new Error(`Unrecognized data shape ${shape}`); } } forecastArray(input: Float32Array, count = 1): Float32Array { this.checkRunnable(); const { model } = this; const { equations } = model; const length = input.length + count; while (equations.length <= length) { this.bindEquation(); } let lastOutput; let equationIndex = 0; if (this.options.inputSize === 1) { for (let i = 0; i < input.length; i++) { lastOutput = equations[equationIndex++].runInput( Float32Array.from([input[i]]) ); } } else { for (let i = 0; i < input.length; i++) { lastOutput = equations[equationIndex++].runInput(Float32Array.from([])); } } if (!lastOutput) { throw new Error('lastOutput not set'); } const result = [lastOutput.weights[0]]; for (let i = 0, max = count - 1; i < max; i++) { lastOutput = equations[equationIndex++].runInput(lastOutput.weights); result.push(lastOutput.weights[0]); } this.end(); return Float32Array.from(result); } forecastArrayOfArray(input: Float32Array[], count = 1): Float32Array[] { this.checkRunnable(); const { model } = this; const { equations } = model; const length = input.length + count; while (equations.length <= length) { this.bindEquation(); } let lastOutput; let equationIndex = 0; for (let i = 0; i < input.length; i++) { lastOutput = equations[equationIndex++].runInput(input[i]); } if (!lastOutput) { throw new Error('lastOutput not set'); } const result = [Float32Array.from(lastOutput.weights)]; for (let i = 0, max = count - 1; i < max; i++) { lastOutput = equations[equationIndex++].runInput(lastOutput.weights); result.push(Float32Array.from(lastOutput.weights.slice(0))); } this.end(); return result; } forecastArrayOfObject(input: INumberHash[], count = 1): INumberHash[] { if (!this.inputLookup) { throw new Error('this.inputLookup not set'); } if (!this.outputLookup) { throw new Error('this.outputLookup not set'); } const formattedData = input.map((value) => lookup.toArray( this.inputLookup as INumberHash, value, this.inputLookupLength ) ); return this.forecastArrayOfArray(formattedData, count).map((value) => lookup.toObject(this.outputLookup as INumberHash, value) ); } // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error train( data: FormattableData[], trainOpts: Partial = {} ): IRNNStatus { this.trainOpts = trainOpts = { ...rnnTrainDefaults, ...trainOpts, }; // Don't destructure here because this.setSize() can reset this.options. if (this.options.inputSize === 1 && this.options.outputSize === 1) { this.setSize(data); } this.verifySize(); const formattedData = this.formatData(data); let error = Infinity; let i; this.verifyIsInitialized(); const { iterations, errorThresh, logPeriod, callback, callbackPeriod, } = this.trainOpts; const log = trainOpts.log === true ? console.log : trainOpts.log; for (i = 0; i < iterations && error > errorThresh; i++) { let sum = 0; for (let j = 0; j < formattedData.length; j++) { const err = this.trainPattern(formattedData[j], true); sum += err; } error = sum / formattedData.length; if (isNaN(error)) throw new Error( 'Network error rate is unexpected NaN, check network configurations and try again. Most probably input format is not correct or training data is not enough. ' ); if (log && i % logPeriod === 0) { log(`iterations: ${i}, training error: ${error}`); } if (callback && i % callbackPeriod === 0) { callback({ error, iterations: i }); } } return { error, iterations: i, }; } trainArrayOfArray(input: Float32Array[]): number { if (input.length < 2) { throw new Error('input must be an array of 2 or more'); } const { equations } = this.model; while (equations.length < input.length) { this.bindEquation(); } let errorSum = 0; for (let i = 0, max = input.length - 1; i < max; i++) { errorSum += equations[i].predictTarget(input[i], input[i + 1]); } this.end(); return errorSum / input.length; } // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error trainPattern(input: Float32Array[], logErrorRate?: boolean): number { const error = this.trainArrayOfArray(input); this.backpropagate(); this.adjustWeights(); if (logErrorRate) { return error; } return 0; } setSize(data: FormattableData[]): void { let size = 0; const dataShape = lookup.dataShape(data).join(','); switch (dataShape) { case 'array,array,number': case 'array,object,number': case 'array,datum,array,number': case 'array,datum,object,number': size = 1; // probably 1 break; case 'array,array,array,number': size = (data as number[][][])[0][0].length; break; case 'array,array,object,number': // inputs and outputs should match size = Object.keys(lookup.toTable2D(data as INumberHash[][])).length; break; case 'array,datum,array,array,number': size = ((data as unknown) as Array<{ [key: string]: number[][]; }>)[0].input[0].length; break; case 'array,datum,array,object,number': size = Object.keys( lookup.toInputTable2D( data as Array<{ input: Array<{ [key: string]: number }> }> ) ).length; break; default: throw new Error('unknown data shape or configuration'); } this.options = Object.seal({ ...this.options, inputSize: size, outputSize: size, }); } verifySize(): void { if (this.options.inputSize || this.options.outputSize) { if (this.options.inputSize !== this.options.outputSize) { throw new Error('manually set inputSize and outputSize mismatch'); } } } runArray(input: Float32Array): number { this.checkRunnable(); const { equations } = this.model; while (equations.length <= input.length) { this.bindEquation(); } let lastOutput; for (let i = 0; i < input.length; i++) { lastOutput = equations[i].runInput(new Float32Array([input[i]])); } this.end(); return (lastOutput as Matrix).weights[0]; } runArrayOfArray(input: Float32Array[]): Float32Array { this.checkRunnable(); const { model } = this; const { equations } = model; while (equations.length <= input.length) { this.bindEquation(); } let lastOutput; for (let i = 0; i < input.length; i++) { const outputMatrix = equations[i].runInput(input[i]); lastOutput = outputMatrix.weights; } this.end(); return lastOutput ?? Float32Array.from([]); } runObject(input: INumberHash): INumberHash { if (!this.inputLookup) { throw new Error('this.inputLookup not set'); } if (!this.outputLookup) { throw new Error('this.outputLookup not set'); } if (!this.outputLookupLength) { throw new Error('this.outputLookupLength not set'); } if (this.inputLookup === this.outputLookup) { const inputArray = lookup.toArrayShort(this.inputLookup, input); return lookup.toObjectPartial( this.outputLookup, this.forecastArray( inputArray, this.outputLookupLength - inputArray.length ), inputArray.length ); } return lookup.toObject( this.outputLookup, this.forecastArray( lookup.toArray(this.inputLookup, input, this.inputLookupLength), this.outputLookupLength ) ); } runArrayOfObject(input: INumberHash[]): INumberHash { if (this.inputLookup === null) { throw new Error('this.inputLookup not set'); } if (this.outputLookup === null) { throw new Error('this.outputLookup not set'); } const formattedInput = input.map((value) => lookup.toArray( this.inputLookup as INumberHash, value, this.inputLookupLength ) ); return this.forecastArrayOfArray(formattedInput, 1).map((value) => lookup.toObject(this.outputLookup as INumberHash, value) )[0]; } runArrayOfObjectOfArray(input: INumberHash[]): INumberHash { if (!this.inputLookup) { throw new Error('this.inputLookup not set'); } if (!this.outputLookup) { throw new Error('this.outputLookup not set'); } return lookup.toObject( this.outputLookup, this.runArrayOfArray( lookup.toArrays(this.inputLookup, input, this.inputLookupLength) ) ); } end(): void { this.model.equations[this.model.equations.length - 1].runInput( new Float32Array(this.options.outputSize) ); } requireInputOutputOfOne(): void { if (this.options.inputSize !== 1) { throw new Error('inputSize must be 1 for this data size'); } if (this.options.outputSize !== 1) { throw new Error('outputSize must be 1 for this data size'); } } // Handles data shape of 'array,number' formatArray(data: number[]): Float32Array[][] { const result = []; this.requireInputOutputOfOne(); for (let i = 0; i < data.length; i++) { result.push(Float32Array.from([data[i]])); } return [result]; } // Handles data shape of 'array,array,number' formatArrayOfArray(data: number[][]): Float32Array[][] { const result = []; const { inputSize, outputSize } = this.options; if (inputSize === 1 && outputSize === 1) { for (let i = 0; i < data.length; i++) { result.push(arrayToFloat32Arrays(data[i])); } return result; } if (inputSize !== data[0].length) { throw new Error('inputSize must match data input size'); } if (outputSize !== data[0].length) { throw new Error('outputSize must match data output size'); } for (let i = 0; i < data.length; i++) { result.push(Float32Array.from(data[i])); } return [result]; } // Handles data shape of 'array,object,number' formatArrayOfObject(data: INumberHash[]): Float32Array[][] { this.requireInputOutputOfOne(); if (!this.inputLookup) { const lookupTable = new LookupTable(data); this.inputLookup = this.outputLookup = lookupTable.table; this.inputLookupLength = this.outputLookupLength = lookupTable.length; } const result = []; for (let i = 0; i < data.length; i++) { result.push(objectToFloat32Arrays(data[i])); } return result; } // Handles data shape of 'array,object,number' when this.options.inputSize > 1 formatArrayOfObjectMulti(data: INumberHash[]): Float32Array[][] { if (!this.inputLookup) { const lookupTable = new LookupTable(data); this.inputLookup = this.outputLookup = lookupTable.table; this.inputLookupLength = this.outputLookupLength = lookupTable.length; } const result = []; for (let i = 0; i < data.length; i++) { result.push([ objectToFloat32Array(data[i], this.inputLookup, this.inputLookupLength), ]); } return result; } // Handles data shape of 'array,datum,array,number' formatArrayOfDatumOfArray(data: ITrainingDatum[]): Float32Array[][] { const result = []; this.requireInputOutputOfOne(); for (let i = 0; i < data.length; i++) { const datum = data[i]; result.push( inputOutputArrayToFloat32Arrays( datum.input as number[], datum.output as number[] ) ); } return result; } // Handles data shape of 'array,datum,object,number' formatArrayOfDatumOfObject(data: ITrainingDatum[]): Float32Array[][] { this.requireInputOutputOfOne(); if (!this.inputLookup) { const inputLookup = new LookupTable(data, 'input'); this.inputLookup = inputLookup.table; this.inputLookupLength = inputLookup.length; } if (!this.outputLookup) { const outputLookup = new LookupTable(data, 'output'); this.outputLookup = outputLookup.table; this.outputLookupLength = outputLookup.length; } const result = []; for (let i = 0; i < data.length; i++) { const datum = data[i]; result.push( inputOutputObjectToFloat32Arrays( datum.input as INumberHash, datum.output as INumberHash ) ); } return result; } // Handles data shape of 'array,array,array,number' formatArrayOfArrayOfArray(data: number[][][]): Float32Array[][] { const result = []; for (let i = 0; i < data.length; i++) { result.push(arraysToFloat32Arrays(data[i])); } return result; } // Handles data shape of 'array,array,object,number' formatArrayOfArrayOfObject(data: INumberHash[][]): Float32Array[][] { if (!this.inputLookup) { const lookupTable = new LookupTable(data); this.inputLookup = this.outputLookup = lookupTable.table; this.inputLookupLength = this.outputLookupLength = lookupTable.length; } const result = []; for (let i = 0; i < data.length; i++) { const array = []; for (let j = 0; j < data[i].length; j++) { array.push( objectToFloat32Array( data[i][j], this.inputLookup, this.inputLookupLength ) ); } result.push(array); } return result; } // Handles data shape of 'array,datum,array,array,number' formatArrayOfDatumOfArrayOfArray(data: ITrainingDatum[]): Float32Array[][] { const result = []; const { inputSize, outputSize } = this.options; if (inputSize !== (data[0].input as INumberArray[][])[0].length) { throw new Error('inputSize must match data input size'); } if (outputSize !== (data[0].output as INumberArray[][])[0].length) { throw new Error('outputSize must match data output size'); } for (let i = 0; i < data.length; i++) { const datum = data[i]; result.push( inputOutputArraysToFloat32Arrays( datum.input as number[][], datum.output as number[][] ) ); } return result; } // 'Handles data shape of array,datum,array,object,number' formatArrayOfDatumOfArrayOfObject( data: Array<{ input: Array>; output: Array>; }> ): Float32Array[][] { if (!this.inputLookup) { const inputLookup = new ArrayLookupTable(data, 'input'); this.inputLookup = inputLookup.table; this.inputLookupLength = inputLookup.length; } if (!this.outputLookup) { const outputLookup = new ArrayLookupTable(data, 'output'); this.outputLookup = outputLookup.table; this.outputLookupLength = outputLookup.length; } if (!this.outputLookupLength) { throw new Error('this.outputLookupLength not set to usable number'); } const result = []; for (let i = 0; i < data.length; i++) { const datum = data[i]; result.push( inputOutputObjectsToFloat32Arrays( datum.input, datum.output, this.inputLookup, this.outputLookup, this.inputLookupLength, this.outputLookupLength ) ); } return result; } // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error formatData(data: FormattableData[]): Float32Array[][] { const dataShape = lookup.dataShape(data).join(','); switch (dataShape) { case 'array,number': return this.formatArray(data as number[]); case 'array,array,number': return this.formatArrayOfArray(data as number[][]); case 'array,object,number': if (this.options.inputSize === 1) { return this.formatArrayOfObject(data as INumberHash[]); } else { return this.formatArrayOfObjectMulti(data as INumberHash[]); } case 'array,datum,array,number': return this.formatArrayOfDatumOfArray(data as ITrainingDatum[]); case 'array,datum,object,number': return this.formatArrayOfDatumOfObject(data as ITrainingDatum[]); case 'array,array,array,number': return this.formatArrayOfArrayOfArray(data as number[][][]); case 'array,array,object,number': return this.formatArrayOfArrayOfObject(data as INumberHash[][]); case 'array,datum,array,array,number': return this.formatArrayOfDatumOfArrayOfArray(data as ITrainingDatum[]); case 'array,datum,array,object,number': return this.formatArrayOfDatumOfArrayOfObject( data as Array<{ input: Array>; output: Array>; }> ); default: throw new Error('unknown data shape or configuration'); } } test(data: FormattableData[]): ITestResults { // for classification problems const misclasses = []; // run each pattern through the trained network and collect // error and misclassification statistics let errorSum = 0; const formattedData = this.formatData(data); for (let i = 0; i < formattedData.length; i++) { const input = formattedData[i]; const output = this.run(input.splice(0, input.length - 1)); const target = input[input.length - 1]; let errors = 0; let errorCount = 0; for (let j = 0; j < output.length; j++) { errorCount++; const error = target[j] - output[j]; // mse errors += error * error; } errorSum += errors / errorCount; const errorsAbs = Math.abs(errors); if (errorsAbs > this.trainOpts.errorThresh) { const misclass = (data as number[][][])[i]; misclasses.push({ value: misclass, actual: output, }); } } return { error: errorSum / formattedData.length, misclasses, total: formattedData.length, }; } // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error addFormat(value: FormattableData): void { const dataShape = lookup.dataShape(value).join(','); switch (dataShape) { case 'array,array,number': case 'datum,array,array,number': case 'array,number': case 'datum,array,number': return; case 'datum,object,number': { this.inputLookup = lookup.addKeys( (value as ITrainingDatum).input as INumberHash, this.inputLookup ?? {} ); if (this.inputLookup) { this.inputLookupLength = Object.keys(this.inputLookup).length; } this.outputLookup = lookup.addKeys( (value as ITrainingDatum).output as INumberHash, this.outputLookup ?? {} ); if (this.outputLookup) { this.outputLookupLength = Object.keys(this.outputLookup).length; } break; } case 'object,number': { this.inputLookup = this.outputLookup = lookup.addKeys( value as INumberHash, this.inputLookup ?? {} ); if (this.inputLookup) { this.inputLookupLength = this.outputLookupLength = Object.keys( this.inputLookup ).length; } break; } case 'array,object,number': { const typedValue = value as INumberHash[]; for (let i = 0; i < typedValue.length; i++) { this.inputLookup = this.outputLookup = lookup.addKeys( typedValue[i], this.inputLookup ?? {} ); if (this.inputLookup) { this.inputLookupLength = this.outputLookupLength = Object.keys( this.inputLookup ).length; } } break; } case 'datum,array,object,number': { const typedValue = value as ITrainingDatum; const typedInput = typedValue.input as INumberHash[]; for (let i = 0; i < typedInput.length; i++) { this.inputLookup = lookup.addKeys( typedInput[i], this.inputLookup ?? {} ); if (this.inputLookup) { this.inputLookupLength = Object.keys(this.inputLookup).length; } } const typedOutput = typedValue.output as INumberHash[]; for (let i = 0; i < typedOutput.length; i++) { this.outputLookup = lookup.addKeys( typedOutput[i], this.outputLookup ?? {} ); if (this.outputLookup) { this.outputLookupLength = Object.keys(this.outputLookup).length; } } break; } default: throw new Error('unknown data shape or configuration'); } } // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error toJSON(): IRNNTimeStepJSON { if (!this.model) { this.initialize(); } const { model } = this; const options = { ...this.options, ...rnnDefaults }; return { type: this.constructor.name, options, hiddenLayers: model.hiddenLayers.map((hiddenLayer) => { const layers: { [index: string]: IMatrixJSON } = {}; for (const p in hiddenLayer) { if (!hiddenLayer.hasOwnProperty(p)) continue; layers[p] = hiddenLayer[p].toJSON(); } return layers; }), outputConnector: model.outputConnector.toJSON(), output: model.output.toJSON(), inputLookup: this.inputLookup, inputLookupLength: this.inputLookupLength, outputLookup: this.outputLookup, outputLookupLength: this.outputLookupLength, }; } // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error fromJSON(json: IRNNTimeStepJSON): this { const { options } = json; const allMatrices = []; const hiddenLayers: IRNNHiddenLayerModel[] = []; // backward compatibility for hiddenSizes json.hiddenLayers.forEach((hiddenLayer) => { const layers: { [index: string]: Matrix } = {}; for (const p in hiddenLayer) { layers[p] = Matrix.fromJSON(hiddenLayer[p]); allMatrices.push(layers[p]); } hiddenLayers.push(layers as IRNNHiddenLayerModel); }); const outputConnector = Matrix.fromJSON(json.outputConnector); allMatrices.push(outputConnector); const output = Matrix.fromJSON(json.output); allMatrices.push(output); // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error this.options = { ...defaults(), ...options }; this.inputLookup = json.inputLookup; this.inputLookupLength = json.inputLookupLength; this.outputLookup = json.outputLookup; this.outputLookupLength = json.outputLookupLength; this.model = Object.seal({ isInitialized: true, hiddenLayers, output, allMatrices, outputConnector, equations: [], equationConnections: [], }); this.initialLayerInputs = options.hiddenLayers.map( (size) => new Matrix(size, 1) ); this.bindEquation(); return this; } // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error toFunction(cb?: (src: string) => string): RNNTimeStepFunction { const { model, inputLookup, inputLookupLength, outputLookup, outputLookupLength, } = this; const { inputSize } = this.options; const { equations } = model; const equation = equations[1]; const { states } = equation; const jsonString = JSON.stringify(this.toJSON()); function previousConnectionIndex(m: Matrix) { const connection = model.equationConnections[0]; const { states } = equations[0]; for (let i = 0, max = states.length; i < max; i++) { if (states[i].product === m) { return i; } } return connection.indexOf(m); } function matrixOrigin(m: Matrix, stateIndex: number): string { for (let i = 0, max = states.length; i < max; i++) { const state = states[i]; if (i === stateIndex) { const j = previousConnectionIndex(m); switch (m) { case state.left: if (j > -1) { return `typeof prevStates[${j}] === 'object' ? prevStates[${j}].product : new Matrix(${m.rows}, ${m.columns})`; } // eslint-disable-next-line no-fallthrough case state.right: if (j > -1) { return `typeof prevStates[${j}] === 'object' ? prevStates[${j}].product : new Matrix(${m.rows}, ${m.columns})`; } // eslint-disable-next-line no-fallthrough case state.product: return `new Matrix(${m.rows}, ${m.columns})`; default: throw Error('unknown state'); } } if (m === state.product) return `states[${i}].product`; if (m === state.right) return `states[${i}].right`; if (m === state.left) return `states[${i}].left`; } return ''; } function matrixToString(m: Matrix, stateIndex: number): string { if (!m || !m.rows || !m.columns) return 'null'; if (m === model.outputConnector) return `json.outputConnector`; if (m === model.output) return `json.output`; for (let i = 0, max = model.hiddenLayers.length; i < max; i++) { const hiddenLayer = model.hiddenLayers[i]; for (const p in hiddenLayer) { if (!hiddenLayer.hasOwnProperty(p)) continue; if (hiddenLayer[p] !== m) continue; return `json.hiddenLayers[${i}].${p}`; } } return matrixOrigin(m, stateIndex); } function formatInputData() { if (!inputLookup) return ''; if (inputSize === 1) { if (inputLookup === outputLookup) { return `function lookupInput(input) { var table = ${JSON.stringify(inputLookup)}; var result = []; for (var p in table) { if (!input.hasOwnProperty(p)) break; result.push(Float32Array.from([input[p]])); } return result; }`; } return `function lookupInput(input) { var table = ${JSON.stringify(inputLookup)}; var result = []; for (var p in table) { result.push(Float32Array.from([input[p]])); } return result; }`; } return `function lookupInput(rawInputs) { var table = ${JSON.stringify(inputLookup)}; var result = []; for (var i = 0; i < rawInputs.length; i++) { var rawInput = rawInputs[i]; var input = new Float32Array(${inputLookupLength}); for (var p in table) { input[table[p]] = rawInput.hasOwnProperty(p) ? rawInput[p] : 0; } result.push(input); } return result; }`; } function formatOutputData() { if (!outputLookup) return ''; if (inputSize === 1) { if (inputLookup === outputLookup) { return `function lookupOutputPartial(output, input) { var table = ${JSON.stringify(outputLookup)}; var offset = input.length; var result = {}; var i = 0; for (var p in table) { if (i++ < offset) continue; result[p] = output[table[p] - offset][0]; } return result; }`; } return `function lookupOutput(output) { var table = ${JSON.stringify(outputLookup)}; var result = {}; for (var p in table) { result[p] = output[table[p]][0]; } return result; }`; } return `function lookupOutput(output) { var table = ${JSON.stringify(outputLookup)}; var result = {}; for (var p in table) { result[p] = output[table[p]]; } return result; }`; } function toInner(fnString: string) { // crude, but should be sufficient for now // function() { body } // crude, but should be sufficient for now // function() { body } const fnParts = fnString.toString().split('{'); fnParts.shift(); // body } const fnBodyString = fnParts.join('{'); const fnBodyParts = fnBodyString.split('}'); fnBodyParts.pop(); // body return fnBodyParts .join('}') .split('\n') .join('\n ') .replace('product.deltas[i] = 0;', '') .replace('product.deltas[column] = 0;', '') .replace('left.deltas[leftIndex] = 0;', '') .replace('right.deltas[rightIndex] = 0;', '') .replace('product.deltas = left.deltas.slice(0);', ''); } function fileName(fnName: string) { return `src/recurrent/matrix/${fnName.replace(/[A-Z]/g, function (value) { return `-${value.toLowerCase()}`; })}.js`; } const statesRaw = []; const usedFunctionNames: { [methodName: string]: boolean } = {}; const innerFunctionsSwitch = []; for (let i = 0, max = states.length; i < max; i++) { const state = states[i]; statesRaw.push(`states[${i}] = { name: '${state.forwardFn.name}', left: ${state.left ? matrixToString(state.left, i) : 'undefined'}, right: ${state.right ? matrixToString(state.right, i) : 'undefined'}, product: ${matrixToString(state.product, i)} }`); const fnName = state.forwardFn.name; if (!usedFunctionNames[fnName]) { usedFunctionNames[fnName] = true; if (state.name === 'input') { innerFunctionsSwitch.push(`case '${fnName}':`); innerFunctionsSwitch.push( inputLookup && inputSize === 1 ? 'product.weights = _i < input.length ? input[_i]: prevStates[prevStates.length - 1].product.weights;' : inputSize === 1 ? 'product.weights = [input[_i]];' : 'product.weights = input[_i];' ); innerFunctionsSwitch.push('break;'); } else { innerFunctionsSwitch.push( ` case '${fnName}':${ fnName !== 'forwardFn' ? ` //compiled from ${fileName(fnName)}` : '' } ${toInner(state.forwardFn.toString())} break;` ); } } } const forceForecast = inputSize === 1 && this.outputLookup; const src = ` var input = ${this.inputLookup ? 'lookupInput(rawInput)' : 'rawInput'}; var json = ${jsonString}; var output = []; var states = []; var prevStates; var state; var max = ${ forceForecast ? inputLookup === outputLookup ? inputLookupLength : `input.length + ${outputLookupLength - 1}` : 'input.length' }; for (var _i = 0; _i < max; _i++) { prevStates = states; states = []; ${statesRaw.join(';\n ')}; for (var stateIndex = 0, stateMax = ${ statesRaw.length }; stateIndex < stateMax; stateIndex++) { state = states[stateIndex]; var product = state.product; var left = state.left; var right = state.right; switch (state.name) { ${innerFunctionsSwitch.join('\n')} } } ${ inputSize === 1 && inputLookup ? 'if (_i >= input.length - 1) { output.push(state.product.weights); }' : 'output = state.product.weights;' } } ${ outputLookup ? outputLookup === inputLookup ? 'return lookupOutputPartial(output, input)' : 'return lookupOutput(output)' : inputSize === 1 ? 'return output[0]' : 'return output' }; ${formatInputData()} ${formatOutputData()} function Matrix(rows, columns) { this.rows = rows; this.columns = columns; this.weights = new Float32Array(rows * columns); } ${softmax.toString().replace('_2.default', 'Matrix')} ${randomFloat.toString()} ${sampleI.toString()} ${maxI.toString()}`; // eslint-disable-next-line return new Function('rawInput', cb ? cb(src) : src) as RNNTimeStepFunction; } } export type RNNTimeStepFunction = < InputType extends InputOutputValue | InputOutputValue[] >( rawInput?: InputType, isSampleI?: boolean, temperature?: number ) => ValuesOf; export const trainDefaults = { ...rnnTrainDefaults }; ================================================ FILE: src/recurrent/rnn.test.ts ================================================ import { IMatrixJSON, Matrix } from './matrix'; import { Equation } from './matrix/equation'; import { defaults, RNN, RNNFunction } from './rnn'; import { DataFormatter } from '../utilities/data-formatter'; import { allMatrices } from '../test-utils'; function notZero(v: number) { return v !== 0; } describe('RNN', () => { describe('.constructor()', () => { describe('when called without options.json', () => { it('does not initialize model', () => { const net = new RNN(); expect(net.model.isInitialized).toBe(false); }); }); describe('when called with options.json', () => { const getJSON = () => { const net = new RNN({ hiddenLayers: [3], inputSize: 3, inputRange: 2, outputSize: 2, }); net.initialize(); return net.toJSON(); }; let fromJSONMock: jest.SpyInstance; beforeEach(() => { fromJSONMock = jest.spyOn(RNN.prototype, 'fromJSON'); }); afterEach(() => { fromJSONMock.mockRestore(); }); it('calls this.fromJSON() with it', () => { const json = getJSON(); const net = new RNN({ json }); expect(net).toBeInstanceOf(RNN); expect(fromJSONMock).toBeCalledWith(json); }); }); }); describe('.initialize()', () => { describe('when creating hidden layers', () => { let createHiddenLayersMock: jest.SpyInstance; let getHiddenLayerMock: jest.SpyInstance; beforeEach(() => { createHiddenLayersMock = jest.spyOn( RNN.prototype, 'createHiddenLayers' ); getHiddenLayerMock = jest.spyOn(RNN.prototype, 'getHiddenLayer'); }); afterEach(() => { createHiddenLayersMock.mockRestore(); getHiddenLayerMock.mockRestore(); }); it('calls createHiddenLayers', () => { const net = new RNN(); net.initialize(); expect(createHiddenLayersMock).toBeCalled(); }); it('calls static getHiddenLayer method', () => { const net = new RNN(); net.initialize(); expect(getHiddenLayerMock).toBeCalled(); }); }); it('initializes model', () => { const net = new RNN(); net.initialize(); expect(net.model).not.toBe(null); }); it('can setup different size hiddenLayers', () => { const inputSize = 2; const hiddenLayers = [5, 4, 3]; const networkOptions = { learningRate: 0.001, decayRate: 0.75, inputSize: inputSize, hiddenLayers, outputSize: inputSize, }; const net = new RNN(networkOptions); net.initialize(); net.bindEquation(); expect(net.model.hiddenLayers.length).toBe(3); expect(net.model.hiddenLayers[0].weight.columns).toBe(inputSize); expect(net.model.hiddenLayers[0].weight.rows).toBe(hiddenLayers[0]); expect(net.model.hiddenLayers[1].weight.columns).toBe(hiddenLayers[0]); expect(net.model.hiddenLayers[1].weight.rows).toBe(hiddenLayers[1]); expect(net.model.hiddenLayers[2].weight.columns).toBe(hiddenLayers[1]); expect(net.model.hiddenLayers[2].weight.rows).toBe(hiddenLayers[2]); }); }); describe('.createHiddenLayers()', () => { it('creates hidden layers in the expected size', () => { const net = new RNN({ inputSize: 1, hiddenLayers: [15, 20], outputSize: 1, }); const hiddenLayers = net.createHiddenLayers(); expect(hiddenLayers.length).toEqual(2); expect(hiddenLayers[0].weight.rows).toEqual(15); expect(hiddenLayers[0].weight.columns).toEqual(1); expect(hiddenLayers[0].bias.rows).toEqual(15); expect(hiddenLayers[0].bias.columns).toEqual(1); expect(hiddenLayers[0].transition.rows).toEqual(15); expect(hiddenLayers[0].transition.columns).toEqual(15); expect(hiddenLayers[1].weight.rows).toEqual(20); expect(hiddenLayers[1].weight.columns).toEqual(15); expect(hiddenLayers[1].bias.rows).toEqual(20); expect(hiddenLayers[1].bias.columns).toEqual(1); expect(hiddenLayers[1].transition.rows).toEqual(20); expect(hiddenLayers[1].transition.columns).toEqual(20); }); }); describe('.createOutputMatrices()', () => { it('creates output layers in the expected size', () => { const net = new RNN({ inputSize: 1, hiddenLayers: [22], outputSize: 1, }); const { output, outputConnector } = net.createOutputMatrices(); expect(outputConnector.rows).toBe(2); expect(outputConnector.columns).toBe(22); expect(output.rows).toBe(2); expect(output.columns).toBe(1); }); }); describe('basic operations', () => { it('starts with zeros in input.deltas', () => { const net = new RNN(); net.initialize(); net.model.input.deltas.forEach((v) => { expect(v === 0).toBeTruthy(); }); }); it('after initial run, does not have zeros in weights and produces error', () => { const net = new RNN({ hiddenLayers: [3], inputSize: 3, inputRange: 2, outputSize: 2, }); net.initialize(); const error = net.trainInput([1, 1, 0]); expect(net.model.input.weights.some(notZero)).toBeTruthy(); expect( net.model.hiddenLayers[0].weight.weights.some(notZero) ).toBeTruthy(); expect(net.model.outputConnector.weights.some(notZero)).toBeTruthy(); expect(error).toBeGreaterThan(0); expect(error).toBeLessThan(Infinity); }); it('after initial run, input does not have zeros in deltas', () => { const net = new RNN({ hiddenLayers: [3], inputSize: 3, inputRange: 2, outputSize: 2, }); net.initialize(); net.trainInput([1, 1, 0]); net.model.input.deltas.forEach((v) => { expect(v).toBe(0); }); net.backpropagate([1, 1, 0]); net.backpropagate([0, 1, 1]); net.backpropagate([1, 0, 1]); net.backpropagate([1, 1, 0]); expect(net.model.input.deltas.some(notZero)).toBeTruthy(); }); it('can handle unrecognized input characters', () => { const net = new RNN({ hiddenLayers: [3] }); net.train([ { input: '1', output: '2' }, { input: '2', output: '3' }, ]); expect(() => { net.run('7'); }).not.toThrow(); }); }); describe('xor', () => { jest.retryTimes(5); function xorNet() { const net = new RNN({ hiddenLayers: [20, 20], inputSize: 3, inputRange: 3, outputSize: 3, }); net.initialize(); net.train(xorNetValues, { iterations: 1 }); return net; } const xorNetValues = [ [0, 0, 0], [0, 1, 1], [1, 0, 1], [1, 1, 0], ]; let predictTargetIndex: jest.SpyInstance; let backpropagateIndex: jest.SpyInstance; beforeEach(() => { predictTargetIndex = jest.spyOn(Equation.prototype, 'predictTargetIndex'); backpropagateIndex = jest.spyOn(Equation.prototype, 'backpropagateIndex'); }); afterEach(() => { predictTargetIndex.mockRestore(); backpropagateIndex.mockRestore(); }); it('properly provides values to equations[].predictTargetIndex', () => { const net = xorNet(); predictTargetIndex.mockReset(); net.trainInput([0, 0, 0]); // called in reverse expect(predictTargetIndex).toHaveBeenNthCalledWith(1, 0, 1); expect(predictTargetIndex).toHaveBeenNthCalledWith(2, 1, 1); expect(predictTargetIndex).toHaveBeenNthCalledWith(3, 1, 1); expect(predictTargetIndex).toHaveBeenNthCalledWith(4, 1, 0); predictTargetIndex.mockReset(); net.trainInput([0, 1, 1]); // called in reverse expect(predictTargetIndex).toHaveBeenNthCalledWith(1, 0, 1); expect(predictTargetIndex).toHaveBeenNthCalledWith(2, 1, 2); expect(predictTargetIndex).toHaveBeenNthCalledWith(3, 2, 2); expect(predictTargetIndex).toHaveBeenNthCalledWith(4, 2, 0); }); it('properly provides values to equations[].runBackpropagate', () => { const net = xorNet(); backpropagateIndex.mockReset(); net.trainInput([0, 0, 0]); net.backpropagate([0, 0, 0]); expect(backpropagateIndex).toBeCalledTimes(4); // called in reverse expect(backpropagateIndex).toHaveBeenNthCalledWith(1, 1); expect(backpropagateIndex).toHaveBeenNthCalledWith(2, 1); expect(backpropagateIndex).toHaveBeenNthCalledWith(3, 1); expect(backpropagateIndex).toHaveBeenNthCalledWith(4, 0); net.trainInput([0, 1, 1]); backpropagateIndex.mockReset(); net.backpropagate([0, 1, 1]); expect(backpropagateIndex).toBeCalledTimes(4); // called in reverse expect(backpropagateIndex).toHaveBeenNthCalledWith(1, 2); expect(backpropagateIndex).toHaveBeenNthCalledWith(2, 2); expect(backpropagateIndex).toHaveBeenNthCalledWith(3, 1); expect(backpropagateIndex).toHaveBeenNthCalledWith(4, 0); }); it('is fully connected and gives values in deltas', () => { const net = xorNet(); net.initialize(); const input = xorNetValues[2]; net.model.allMatrices.forEach((m) => { m.deltas.forEach((value) => { expect(value).toBe(0); }); }); net.trainInput(input); net.model.input.deltas.forEach((v) => { expect(v).toBe(0); }); net.model.hiddenLayers.forEach((layer) => { for (const p in layer) { if (!layer.hasOwnProperty(p)) continue; layer[p].deltas.forEach((v) => { expect(v).toBe(0); }); } }); net.model.output.deltas.forEach((v) => { expect(v).toBe(0); }); net.backpropagate(input); expect(net.model.input.deltas.some(notZero)).toBeTruthy(); net.model.hiddenLayers.forEach((layer) => { for (const p in layer) { if (!layer.hasOwnProperty(p)) continue; // if (!layer[p].deltas.some(notZero)) console.log(p); // assert(layer[p].deltas.some(notZero)); } }); expect(net.model.output.deltas.some(notZero)).toBeTruthy(); net.model.equations.forEach((equation) => { equation.states.forEach((state) => { if (state.left?.deltas) state.left.deltas.some(notZero); if (state.right?.deltas) state.right.deltas.some(notZero); if (state.product?.deltas) state.product.deltas.some(notZero); }); }); }); it('deltas and weights do not explode', () => { const net = xorNet(); const input = xorNetValues[2]; function checkExploded() { allMatrices(net.model, (values: Float32Array) => { values.forEach((value) => { if (isNaN(value)) throw new Error('exploded'); }); }); } expect(() => { for (let i = 0; i < 100; i++) { checkExploded(); net.trainInput(input); checkExploded(); net.backpropagate(input); checkExploded(); net.adjustWeights(); checkExploded(); } }).not.toThrow(); }); it('can learn xor (error goes down)', () => { const net = xorNet(); let initialError = Infinity; let error; for (let i = 0; i < 10; i++) { error = 0; for (let j = 0; j < 4; j++) { error += net.trainPattern(xorNetValues[j], true); } if (i === 0) { initialError = error; } } expect(error).toBeLessThan(initialError); }); it('can predict xor', () => { const net = xorNet(); for (let i = 0; i < 10; i++) { xorNetValues.forEach(function (value) { net.trainPattern(value, true); }); } expect(net.run().length).toBe(3); }); }); describe('json', () => { describe('.toJSON()', () => { it('can export model as json', () => { const net = new RNN({ inputSize: 6, inputRange: 12, outputSize: 6, }); const json = net.toJSON(); function compare(left: IMatrixJSON, right: Matrix) { left.weights.forEach((value, i) => { expect(value).toBe(right.weights[i]); }); expect(left.rows).toBe(right.rows); expect(left.columns).toBe(right.columns); } compare(json.input, net.model.input); net.model.hiddenLayers.forEach((layer, i) => { compare(json.hiddenLayers[i].weight, layer.weight); compare(json.hiddenLayers[i].transition, layer.transition); compare(json.hiddenLayers[i].bias, layer.bias); }); compare(json.output, net.model.output); compare(json.outputConnector, net.model.outputConnector); }); }); describe('.fromJSON()', () => { it('can import model from json', () => { const inputSize = 7; const hiddenLayers = [10, 20]; const dataFormatter = new DataFormatter('abcdef'.split('')); const jsonString = JSON.stringify( new RNN({ inputSize, // <- length hiddenLayers, inputRange: dataFormatter.characters.length, outputSize: dataFormatter.characters.length, // <- length dataFormatter, }).toJSON(), null, 2 ); const clone = new RNN(); clone.fromJSON(JSON.parse(jsonString)); const cloneString = JSON.stringify(clone.toJSON(), null, 2); expect(jsonString).toBe(cloneString); expect(clone.options.inputSize).toBe(dataFormatter.characters.length); expect(clone.options.inputRange).toBe(dataFormatter.characters.length); expect(clone.options.outputSize).toBe(dataFormatter.characters.length); expect(clone.model.hiddenLayers.length).toBe(2); expect(clone.model.hiddenLayers[0].weight.columns).toBe(inputSize); expect(clone.model.hiddenLayers[0].weight.rows).toBe(hiddenLayers[0]); expect(clone.model.hiddenLayers[1].weight.columns).toBe( hiddenLayers[0] ); expect(clone.model.hiddenLayers[1].weight.rows).toBe(hiddenLayers[1]); }); it('can import model from json using .fromJSON()', () => { const dataFormatter = new DataFormatter('abcdef'.split('')); const jsonString = JSON.stringify( new RNN({ inputSize: dataFormatter.characters.length, // <- length inputRange: dataFormatter.characters.length, outputSize: dataFormatter.characters.length, // <- length }).toJSON() ); const clone = new RNN(); clone.fromJSON(JSON.parse(jsonString)); expect(jsonString).toBe(JSON.stringify(clone.toJSON())); expect(clone.options.inputSize).toBe(7); expect(clone.options.inputRange).toBe(dataFormatter.characters.length); expect(clone.options.outputSize).toBe(dataFormatter.characters.length); }); it('will not initialize when importing json', () => { const dataFormatter = new DataFormatter('abcdef'.split('')); const original = new RNN({ inputSize: 6, // <- length inputRange: dataFormatter.characters.length, hiddenLayers: [3, 3], outputSize: dataFormatter.characters.length, // <- length dataFormatter, }); original.initialize(); const jsonString = JSON.stringify(original.toJSON()); const json = JSON.parse(jsonString); const clone = new RNN(); clone.fromJSON(json); expect(jsonString).toBe(JSON.stringify(clone.toJSON())); expect(clone.options.inputSize).toBe(7); expect(clone.options.inputRange).toBe(dataFormatter.characters.length); expect(clone.options.outputSize).toBe(dataFormatter.characters.length); }); it('can import model from json and train again', () => { const dataFormatter = new DataFormatter('abcdef'.split('')); const net = new RNN({ inputSize: 6, // <- length inputRange: dataFormatter.characters.length, outputSize: dataFormatter.characters.length, // <- length dataFormatter, }); net.initialize(); // over fit on purpose for (let i = 0; i < 10; i++) { net.trainPattern([0, 1, 1]); net.trainPattern([1, 0, 1]); net.trainPattern([1, 1, 0]); net.trainPattern([0, 0, 0]); } const error = net.trainPattern([0, 1, 1], true); const jsonString = JSON.stringify(net.toJSON()); const clone = new RNN(); clone.fromJSON(JSON.parse(jsonString)); expect(jsonString).toBe(JSON.stringify(clone.toJSON())); const newError = clone.trainPattern([0, 1, 1], true); expect(error - newError < 0.02).toBeTruthy(); expect(jsonString).not.toBe(JSON.stringify(clone.toJSON())); expect(clone.options.inputSize).toBe(7); expect(clone.options.inputRange).toBe(dataFormatter.characters.length); expect(clone.options.outputSize).toBe(dataFormatter.characters.length); }); }); }); describe('.trainPattern()', () => { it('changes the neural net when ran', () => { const net = new RNN({ dataFormatter: new DataFormatter([0, 1]), hiddenLayers: [2], }); const netBeforeTraining = JSON.stringify(net.toJSON()); net.train( [ [0, 0, 0], [0, 1, 1], [1, 0, 1], [1, 1, 0], ], { iterations: 10 } ); const netAfterTraining = JSON.stringify(net.toJSON()); expect(netBeforeTraining).not.toBe(netAfterTraining); }); }); describe('maxPredictionLength', () => { it('gets a default value', () => { expect(new RNN().options.maxPredictionLength).toBe( defaults().maxPredictionLength ); }); it('restores option', () => { const maxPredictionLength = Math.random(); expect(new RNN({ maxPredictionLength }).options.maxPredictionLength).toBe( maxPredictionLength ); }); it('can be set multiple times', () => { const net = new RNN({ maxPredictionLength: 5 }); expect(net.options.maxPredictionLength).toBe(5); net.options.maxPredictionLength = 1; expect(net.options.maxPredictionLength).toBe(1); }); it('shortens returned values', () => { const net = new RNN({ maxPredictionLength: 3 }); net.train([{ input: '123', output: '456' }], { errorThresh: 0.011 }); const output1 = net.run('123'); expect(output1.length).toBe(3); net.options.maxPredictionLength = 1; const output2 = net.run('123'); expect(output2.length).toBe(1); }); }); describe('.toFunction()', () => { describe('without callback argument', () => { it('returns function that works still produces stable output', () => { const net = new RNN({ hiddenLayers: [3], }); net.train([ { input: '1', output: '2' }, { input: '2', output: '3' }, ]); const expected1 = net.run('1'); const expected2 = net.run('2'); const fn = net.toFunction(); expect(fn('1')).toBe(expected1); expect(fn('2')).toBe(expected2); }); }); describe('with callback argument', () => { it('returns string, which expects a result that makes a function that works still produces stable output', async () => { const net = new RNN({ hiddenLayers: [3], }); net.train([ { input: '1', output: '2' }, { input: '2', output: '3' }, ]); const expected1 = net.run('1'); const expected2 = net.run('2'); const fn = await new Promise((resolve, reject) => { try { resolve( net.toFunction((_fnBody) => { expect(typeof _fnBody).toBe('string'); return _fnBody; }) ); } catch (e) { reject(e); } }); expect(fn('1')).toBe(expected1); expect(fn('2')).toBe(expected2); }); }); it('can output same as run method', () => { const dataFormatter = new DataFormatter(['h', 'i', ' ', 'm', 'o', '!']); const net = new RNN({ inputSize: 7, inputRange: dataFormatter.characters.length, outputSize: 7, dataFormatter, }); net.initialize(); for (let i = 0; i < 100; i++) { net.trainPattern(dataFormatter.toIndexes('hi mom!')); // if (i % 10) { // console.log(dataFormatter.toCharacters(net.run()).join('')); // } } const lastOutput = net.run(); expect(net.toFunction()()).toBe(lastOutput); }); it('can include the DataFormatter', () => { const net = new RNN(); net.train(['hi mom!'], { errorThresh: 0.011 }); const expected = net.run('hi'); const newNet = net.toFunction(); expect(newNet('hi')).toBe(expected); }); }); describe('.bindEquation()', () => { let getEquation: jest.SpyInstance; beforeEach(() => { getEquation = jest.spyOn(RNN.prototype, 'getEquation'); }); afterEach(() => { getEquation.mockRestore(); }); it('calls static getEquation method', () => { const net = new RNN(); net.initialize(); net.bindEquation(); expect(getEquation).toBeCalled(); }); }); }); ================================================ FILE: src/recurrent/rnn.ts ================================================ import { Log } from '../feed-forward'; import { INeuralNetworkTrainOptions } from '../neural-network'; import { DataFormatter, IDataFormatter, IDataFormatterJSON, } from '../utilities/data-formatter'; import { randomFloat } from '../utilities/random'; import { zeros } from '../utilities/zeros'; import { IMatrixJSON, Matrix } from './matrix'; import { copy } from './matrix/copy'; import { Equation } from './matrix/equation'; import { maxI } from './matrix/max-i'; import { RandomMatrix } from './matrix/random-matrix'; import { sampleI } from './matrix/sample-i'; import { softmax } from './matrix/softmax'; import { IRNNDatum, Value } from './rnn-data-types'; export interface IRNNModel { isInitialized: boolean; input: Matrix; hiddenLayers: IRNNHiddenLayerModel[]; output: Matrix; equations: Equation[]; allMatrices: Matrix[]; equationConnections: Matrix[][]; outputConnector: RandomMatrix | Matrix; } export interface IRNNOptions { inputSize: number; inputRange: number; hiddenLayers: number[]; outputSize: number; decayRate: number; smoothEps: number; regc: number; clipval: number; maxPredictionLength: number; dataFormatter: IDataFormatter; json?: IRNNJSON; } export interface IRNNJSONOptions { inputSize: number; inputRange: number; hiddenLayers: number[]; outputSize: number; decayRate: number; smoothEps: number; regc: number; clipval: number; maxPredictionLength: number; dataFormatter: IDataFormatterJSON; } export interface IRNNTrainingOptions { iterations: number; errorThresh: number; log: boolean | ((status: string) => void); logPeriod: number; learningRate: number; callback?: (status: IRNNStatus) => void; callbackPeriod: number; timeout: number; } export interface IRNNJSONTrainOptions { iterations: number; errorThresh: number; log: boolean | ((status: string) => void); logPeriod: number; learningRate: number; callback?: (status: IRNNStatus) => void; callbackPeriod: number; timeout: number | 'Infinity'; } export const trainDefaults: IRNNTrainingOptions = { iterations: 20000, errorThresh: 0.005, log: false, logPeriod: 10, learningRate: 0.01, callbackPeriod: 10, timeout: Infinity, }; export interface IRNNHiddenLayer { [key: string]: RandomMatrix | Matrix; } export interface IRNNHiddenLayerModel extends IRNNHiddenLayer { // wxh weight: RandomMatrix; // whh transition: RandomMatrix; // bhh bias: Matrix; } export const defaults = (): IRNNOptions => { return { inputSize: 20, inputRange: 20, hiddenLayers: [20, 20], outputSize: 20, decayRate: 0.999, smoothEps: 1e-8, regc: 0.000001, clipval: 5, maxPredictionLength: 100, dataFormatter: new DataFormatter(), }; }; export interface IRNNStatus { iterations: number; error: number; } export interface IRNNPreppedTrainingData { status: IRNNStatus; preparedData: number[][]; endTime: number; } export class RNN { options: IRNNOptions = { ...defaults() }; trainOpts: IRNNTrainingOptions = { ...trainDefaults }; stepCache: { [index: number]: Float32Array } = {}; runs = 0; ratioClipped = 0; model: IRNNModel = Object.seal({ isInitialized: false, input: new Matrix(0, 0), hiddenLayers: [], output: new Matrix(0, 0), equations: [], allMatrices: [], equationConnections: [], outputConnector: new RandomMatrix(0, 0, 0.08), }); initialLayerInputs: Matrix[] = []; constructor(options: Partial = {}) { this.options = { ...this.options, ...options }; this.updateTrainingOptions({ ...trainDefaults, // ...options, }); if (options.json) { this.fromJSON(options.json); } } initialize(): void { const { dataFormatter } = this.options; if (dataFormatter?.characters.length) { this.options.inputSize = this.options.inputRange = this.options.outputSize = dataFormatter.characters.length; } this.model = this.mapModel(); } createHiddenLayers(): IRNNHiddenLayer[] { const { hiddenLayers, inputSize } = this.options; const hiddenLayersModel: IRNNHiddenLayer[] = []; // 0 is end, so add 1 to offset hiddenLayersModel.push(this.getHiddenLayer(hiddenLayers[0], inputSize)); let prevSize = hiddenLayers[0]; for (let d = 1; d < hiddenLayers.length; d++) { // loop over depths const hiddenSize = hiddenLayers[d]; hiddenLayersModel.push(this.getHiddenLayer(hiddenSize, prevSize)); prevSize = hiddenSize; } return hiddenLayersModel; } getHiddenLayer(hiddenSize: number, prevSize: number): IRNNHiddenLayer { return { // wxh weight: new RandomMatrix(hiddenSize, prevSize, 0.08), // whh transition: new RandomMatrix(hiddenSize, hiddenSize, 0.08), // bhh bias: new Matrix(hiddenSize, 1), }; } getEquation( equation: Equation, inputMatrix: Matrix, previousResult: Matrix, hiddenLayer: IRNNHiddenLayer ): Matrix { if (!hiddenLayer.weight || !hiddenLayer.transition || !hiddenLayer.bias) { throw new Error('hiddenLayer does not have expected properties'); } const relu = equation.relu.bind(equation); const add = equation.add.bind(equation); const multiply = equation.multiply.bind(equation); return relu( add( add( multiply(hiddenLayer.weight, inputMatrix), multiply(hiddenLayer.transition, previousResult) ), hiddenLayer.bias ) ); } createInputMatrix(): RandomMatrix { const { inputRange, inputSize } = this.options; if (inputRange < 1) throw new Error('this.options.inputRange not an expected number'); if (inputSize < 1) throw new Error('this.options.inputSize not an expected number'); // 0 is end, so add 1 to offset return new RandomMatrix(inputRange + 1, inputSize, 0.08); } createOutputMatrices(): { outputConnector: RandomMatrix; output: Matrix } { const { outputSize, hiddenLayers } = this.options; const lastHiddenSize = last(hiddenLayers); // 0 is end, so add 1 to offset return { // whd outputConnector: new RandomMatrix(outputSize + 1, lastHiddenSize, 0.08), // 0 is end, so add 1 to offset // bd output: new Matrix(outputSize + 1, 1), }; } bindEquation(): void { const { model } = this; const { hiddenLayers } = this.options; const equation = new Equation(); const outputs: Matrix[] = []; const equationConnection = model.equationConnections.length > 0 ? last(model.equationConnections) : this.initialLayerInputs; // 0 index let output = this.getEquation( equation, equation.inputMatrixToRow(model.input), equationConnection[0], model.hiddenLayers[0] ); outputs.push(output); // 1+ indices for (let i = 1, max = hiddenLayers.length; i < max; i++) { if (!equationConnection[i]) { throw new Error(`Cannot find equation at index ${i}`); } output = this.getEquation( equation, output, equationConnection[i], model.hiddenLayers[i] ); outputs.push(output); } model.equationConnections.push(outputs); equation.add( equation.multiply(model.outputConnector, output), model.output ); model.equations.push(equation); } mapModel(): IRNNModel { const allMatrices: Matrix[] = []; this.initialLayerInputs = this.options.hiddenLayers.map( (size) => new Matrix(size, 1) ); const input = this.createInputMatrix(); allMatrices.push(input); const hiddenLayers = this.createHiddenLayers() as IRNNHiddenLayerModel[]; if (!hiddenLayers.length) throw new Error('net.hiddenLayers not set'); for (let i = 0, max = hiddenLayers.length; i < max; i++) { const hiddenMatrix: IRNNHiddenLayerModel = hiddenLayers[i]; for (const property in hiddenMatrix) { if (!hiddenMatrix.hasOwnProperty(property)) continue; allMatrices.push(hiddenMatrix[property]); } } const { output, outputConnector } = this.createOutputMatrices(); allMatrices.push(outputConnector); allMatrices.push(output); return Object.seal({ isInitialized: true, input, hiddenLayers, output, equations: [], allMatrices, equationConnections: [], outputConnector, }); } trainInput(input: number[]): number { this.runs++; const { model } = this; const max = input.length; let log2ppl = 0; let equation; while (model.equations.length <= input.length + 1) { // last is zero this.bindEquation(); } for ( let inputIndex = -1, inputMax = input.length; inputIndex < inputMax; inputIndex++ ) { // start and end tokens are zeros const equationIndex = inputIndex + 1; equation = model.equations[equationIndex]; const source = inputIndex === -1 ? 0 : input[inputIndex] + 1; // first step: start with START token const target = inputIndex === max - 1 ? 0 : input[inputIndex + 1] + 1; // last step: end with END token log2ppl += equation.predictTargetIndex(source, target); } return Math.pow(2, log2ppl / (max - 1)) / 100; } backpropagate(input: number[]): void { let i = input.length; const { model } = this; const { equations } = model; while (i > 0) { equations[i].backpropagateIndex(input[i - 1] + 1); i--; } equations[0].backpropagateIndex(0); } adjustWeights(): void { const { regc, clipval, decayRate, smoothEps } = this.options; const { trainOpts, model, stepCache } = this; const { learningRate } = trainOpts; const { allMatrices } = model; let numClipped = 0; let numTot = 0; for (let matrixIndex = 0; matrixIndex < allMatrices.length; matrixIndex++) { const matrix = allMatrices[matrixIndex]; const { weights, deltas } = matrix; if (!(matrixIndex in stepCache)) { stepCache[matrixIndex] = zeros(matrix.rows * matrix.columns); } const cache = stepCache[matrixIndex]; for (let i = 0; i < weights.length; i++) { let r = deltas[i]; const w = weights[i]; // rmsprop adaptive learning rate cache[i] = cache[i] * decayRate + (1 - decayRate) * r * r; // gradient clip if (r > clipval) { r = clipval; numClipped++; } else if (r < -clipval) { r = -clipval; numClipped++; } numTot++; // update (and regularize) weights[i] = w + (-learningRate * r) / Math.sqrt(cache[i] + smoothEps) - regc * w; } } this.ratioClipped = numClipped / numTot; } get isRunnable(): boolean { if (this.model && this.model.equations.length === 0) { console.error(`No equations bound, did you run train()?`); return false; } return true; } checkRunnable(): void { if (!this.isRunnable) { throw new Error('Network not runnable'); } } run(rawInput: Value = [], isSampleI = false, temperature = 1): string { const maxPredictionLength: number = this.options.maxPredictionLength + (rawInput !== null ? (rawInput as string).length : 0) + (this.options.dataFormatter ? this.options.dataFormatter.specialIndexes.length : 0); this.checkRunnable(); const input: number[] = this.options.dataFormatter && (rawInput as string).length > 0 ? this.options.dataFormatter.formatDataIn(rawInput) : (rawInput as number[]); const { model } = this; const output = []; let i = 0; while (true) { const previousIndex = i === 0 ? 0 : i < input.length ? input[i - 1] + 1 : output[i - 1]; while (model.equations.length <= i) { this.bindEquation(); } const equation = model.equations[i]; // sample predicted letter const outputMatrix = equation.runIndex(previousIndex); const logProbabilities = new Matrix( model.output.rows, model.output.columns ); copy(logProbabilities, outputMatrix); if (temperature !== 1 && isSampleI) { /** * scale log probabilities by temperature and re-normalize * if temperature is high, logProbabilities will go towards zero * and the softmax outputs will be more diffuse. if temperature is * very low, the softmax outputs will be more peaky */ for (let j = 0, max = logProbabilities.weights.length; j < max; j++) { logProbabilities.weights[j] /= temperature; } } const probs = softmax(logProbabilities); const nextIndex = isSampleI ? sampleI(probs) : maxI(probs); i++; if (nextIndex === 0) { // END token predicted, break out break; } if (i >= maxPredictionLength) { // something is wrong break; } output.push(nextIndex); } /** * we slice the input length here, not because output contains it, but it will be erroneous as we are sending the * network what is contained in input, so the data is essentially guessed by the network what could be next, till it * locks in on a value. * Kind of like this, values are from input: * 0 -> 4 (or in English: "beginning on input" -> "I have no idea? I'll guess what they want next!") * 2 -> 2 (oh how interesting, I've narrowed down values...) * 1 -> 9 (oh how interesting, I've now know what the values are...) * then the output looks like: [4, 2, 9,...] * so we then remove the erroneous data to get our true output */ return this.options.dataFormatter.formatDataOut( input, output.slice(input.length).map((value) => value - 1) ); } /** * * Verifies network sizes are initialized * If they are not it will initialize them */ verifyIsInitialized(): void { if (!this.model.isInitialized) { this.initialize(); } } /** * * @param options * Supports all `trainDefaults` properties * also supports: * learningRate: (number), * momentum: (number), * activation: 'sigmoid', 'relu', 'leaky-relu', 'tanh' */ updateTrainingOptions(options: Partial): void { this.trainOpts = { ...trainDefaults, ...options }; this.validateTrainingOptions(this.trainOpts as INeuralNetworkTrainOptions); this.setLogMethod(options.log ?? this.trainOpts.log); // TODO: Remove this? // this.activation = options.activation || this.activation; } validateTrainingOptions(options: INeuralNetworkTrainOptions): void { const validations: { [fnName: string]: () => boolean } = { iterations: () => { const val = options.iterations; return typeof val === 'number' && val > 0; }, errorThresh: () => { const val = options.errorThresh; return typeof val === 'number' && val > 0 && val < 1; }, log: () => { const val = options.log; return typeof val === 'function' || typeof val === 'boolean'; }, logPeriod: () => { const val = options.logPeriod; return typeof val === 'number' && val > 0; }, learningRate: () => { const val = options.learningRate; return typeof val === 'number' && val > 0 && val < 1; }, callback: () => { const val = options.callback; return typeof val === 'function' || val === undefined; }, callbackPeriod: () => { const val = options.callbackPeriod; return typeof val === 'number' && val > 0; }, timeout: () => { const val = options.timeout; return typeof val === 'number' && val > 0; }, }; for (const p in validations) { const v = (options as unknown) as { [v: string]: string }; if (!validations[p]()) { throw new Error( `[${p}, ${v[p]}] is out of normal training range, your network will probably not train.` ); } } } setLogMethod(log: Log | undefined | boolean): void { if (typeof log === 'function') { this.trainOpts.log = log; } else if (log) { this.trainOpts.log = console.log; } else { this.trainOpts.log = false; } } protected prepTraining( data: Array, options: Partial ): IRNNPreppedTrainingData { this.updateTrainingOptions(options); const preparedData = this.options.dataFormatter.format(data); const endTime = Date.now() + (this.trainOpts.timeout ?? 0); const status = { error: 1, iterations: 0, }; this.verifyIsInitialized(); return { preparedData, status, endTime, }; } train( data: Array, trainOpts: Partial = {} ): IRNNStatus { this.trainOpts = trainOpts = { ...trainDefaults, ...trainOpts, }; const { iterations, errorThresh, logPeriod, callback, callbackPeriod, } = this.trainOpts; const log = trainOpts.log === true ? console.log : trainOpts.log; let error = Infinity; let i; let inputs: number[][]; if (this.options?.dataFormatter) { inputs = this.options.dataFormatter.format(data); } else if ( Array.isArray(data) && Array.isArray(data[0]) && typeof (data as number[][])[0][0] === 'number' ) { inputs = data as number[][]; } else { throw new Error('training not in expected format of number[][]'); } this.verifyIsInitialized(); for (i = 0; i < iterations && error > errorThresh; i++) { let sum = 0; for (let j = 0; j < inputs.length; j++) { const err = this.trainPattern(inputs[j], true); sum += err; } error = sum / data.length; if (isNaN(error)) { throw new Error( 'Network error rate is unexpected NaN, check network configurations and try again. Most probably input format is not correct or training data is not enough. ' ); } if (log && i % logPeriod === 0) { log(`iterations: ${i}, training error: ${error}`); } if (callback && i % callbackPeriod === 0) { callback({ error, iterations: i }); } } return { error, iterations: i, }; } addFormat(data: Value): void {} formatData(data: Value[]): number[][] { const result = []; for (let i = 0; i < data.length; i++) { result.push(this.options.dataFormatter.formatDataIn(data[i])); } return result; } toJSON(): IRNNJSON { if (!this.model.isInitialized) { this.initialize(); } const { model, options } = this; return { type: this.constructor.name, options: { ...options, dataFormatter: options.dataFormatter.toJSON() }, trainOpts: { ...this.trainOpts, timeout: this.trainOpts.timeout === Infinity ? 'Infinity' : this.trainOpts.timeout, }, input: model.input.toJSON(), hiddenLayers: model.hiddenLayers.map((hiddenLayer) => { const layers: { [index: string]: IMatrixJSON } = {}; for (const p in hiddenLayer) { if (!hiddenLayer.hasOwnProperty(p)) continue; layers[p] = hiddenLayer[p].toJSON(); } return layers; }), outputConnector: this.model.outputConnector.toJSON(), output: this.model.output.toJSON(), }; } fromJSON(json: IRNNJSON): this { const { options } = json; const allMatrices = []; const input = Matrix.fromJSON(json.input); allMatrices.push(input); const hiddenLayers: IRNNHiddenLayerModel[] = []; json.hiddenLayers.forEach((hiddenLayer) => { const layers: { [index: string]: Matrix } = {}; for (const p in hiddenLayer) { layers[p] = Matrix.fromJSON(hiddenLayer[p]); allMatrices.push(layers[p]); } hiddenLayers.push(layers as IRNNHiddenLayerModel); }); const outputConnector = Matrix.fromJSON(json.outputConnector); allMatrices.push(outputConnector); const output = Matrix.fromJSON(json.output); allMatrices.push(output); if (options.dataFormatter) { this.options = { ...defaults(), ...options, dataFormatter: DataFormatter.fromJSON(options.dataFormatter), }; } else { this.options = { ...defaults(), ...options, dataFormatter: new DataFormatter(), }; } this.model = Object.seal({ isInitialized: true, input, hiddenLayers, output, allMatrices, outputConnector, equations: [], equationConnections: [], }); this.initialLayerInputs = this.options.hiddenLayers.map( (size) => new Matrix(size, 1) ); this.bindEquation(); return this; } toFunction(cb?: (src: string) => string): RNNFunction { const { model } = this; const { equations } = this.model; const equation = equations[1]; const { states } = equation; const jsonString = JSON.stringify(this.toJSON()); function previousConnectionIndex(m: Matrix): number { const connection = model.equationConnections[0]; const { states } = equations[0]; for (let i = 0, max = states.length; i < max; i++) { if (states[i].product === m) { return i; } } return connection.indexOf(m); } function matrixOrigin(m: Matrix, stateIndex: number): string { for (let i = 0, max = states.length; i < max; i++) { const state = states[i]; if (i === stateIndex) { const j = previousConnectionIndex(m); if (j > -1 && (m === state.left || m === state.right)) { return `typeof prevStates[${j}] === 'object' ? prevStates[${j}].product : new Matrix(${m.rows}, ${m.columns})`; } return `new Matrix(${m.rows}, ${m.columns})`; } if (m === state.product) return `states[${i}].product`; if (m === state.right) return `states[${i}].right`; if (m === state.left) return `states[${i}].left`; } return ''; } function matrixToString(m: Matrix, stateIndex: number): string { if (!m || !m.rows || !m.columns) return 'null'; if (m === model.input) return `json.input`; if (m === model.outputConnector) return `json.outputConnector`; if (m === model.output) return `json.output`; for (let i = 0, max = model.hiddenLayers.length; i < max; i++) { const hiddenLayer = model.hiddenLayers[i]; for (const p in hiddenLayer) { if (!hiddenLayer.hasOwnProperty(p)) continue; if (hiddenLayer[p] !== m) continue; return `json.hiddenLayers[${i}].${p}`; } } return matrixOrigin(m, stateIndex); } function toInner(fnString: string): string { // crude, but should be sufficient for now // function() { body } const fnParts = fnString.toString().split('{'); fnParts.shift(); // body } const fnBodyString = fnParts.join('{'); const fnBodyParts = fnBodyString.split('}'); fnBodyParts.pop(); // body return fnBodyParts .join('}') .split('\n') .join('\n ') .replace('product.deltas[i] = 0;', '') .replace('product.deltas[column] = 0;', '') .replace('left.deltas[leftIndex] = 0;', '') .replace('right.deltas[rightIndex] = 0;', '') .replace('product.deltas = left.deltas.slice(0);', ''); } function fileName(fnName: string): string { return `src/recurrent/matrix/${fnName.replace(/[A-Z]/g, function (value) { return `-${value.toLowerCase()}`; })}.js`; } const statesRaw = []; const usedFunctionNames: { [methodName: string]: boolean } = {}; const innerFunctionsSwitch = []; for (let i = 0, max = states.length; i < max; i++) { const state = states[i]; statesRaw.push(`states[${i}] = { name: '${state.forwardFn.name}', left: ${state.left ? matrixToString(state.left, i) : 'undefined'}, right: ${state.right ? matrixToString(state.right, i) : 'undefined'}, product: ${matrixToString(state.product, i)} }`); const fnName = state.forwardFn.name; if (!usedFunctionNames[fnName]) { usedFunctionNames[fnName] = true; innerFunctionsSwitch.push( ` case '${fnName}': //compiled from ${fileName(fnName)} ${toInner(state.forwardFn.toString())} break;` ); } } const src = ` if (typeof rawInput === 'undefined') rawInput = []; if (typeof isSampleI === 'undefined') isSampleI = false; if (typeof temperature === 'undefined') temperature = 1; var json = ${jsonString}; ${ this.options.dataFormatter ? `${this.options.dataFormatter.toFunctionString()}; Object.assign(dataFormatter, json.options.dataFormatter);` : '' } ${ this.options.dataFormatter && typeof this.options.dataFormatter.formatDataIn === 'function' ? `const formatDataIn = function (input, output) { ${toInner( this.options.dataFormatter.formatDataIn.toString() )} }.bind(dataFormatter);` : '' } ${ this.options.dataFormatter !== null && typeof this.options.dataFormatter.formatDataOut === 'function' ? `const formatDataOut = function formatDataOut(input, output) { ${toInner( this.options.dataFormatter.formatDataOut.toString() )} }.bind(dataFormatter);` : '' } var maxPredictionLength = ${this.options.maxPredictionLength} + rawInput.length + ${ this.options.dataFormatter ? this.options.dataFormatter.specialIndexes.length : 0 }; var input = ${ this.options.dataFormatter && typeof this.options.dataFormatter.formatDataIn === 'function' ? 'formatDataIn(rawInput)' : 'rawInput' }; var _i = 0; var output = []; var states = []; var prevStates; while (true) { var previousIndex = (_i === 0 ? 0 : _i < input.length ? input[_i - 1] + 1 : output[_i - 1]) ; var rowPluckIndex = previousIndex; prevStates = states; states = []; ${statesRaw.join(';\n ')}; for (var stateIndex = 0, stateMax = ${ statesRaw.length }; stateIndex < stateMax; stateIndex++) { var state = states[stateIndex]; var product = state.product; var left = state.left; var right = state.right; switch (state.name) { ${innerFunctionsSwitch.join('\n')} } } var logProbabilities = state.product; if (temperature !== 1 && isSampleI) { for (var q = 0, nq = logProbabilities.weights.length; q < nq; q++) { logProbabilities.weights[q] /= temperature; } } var probs = softmax(logProbabilities); var nextIndex = isSampleI ? sampleI(probs) : maxI(probs); _i++; if (nextIndex === 0) { break; } if (_i >= maxPredictionLength) { break; } output.push(nextIndex); } ${ this.options.dataFormatter && typeof this.options.dataFormatter.formatDataOut === 'function' ? 'return formatDataOut(input, output.slice(input.length).map(function(value) { return value - 1; }))' : 'return output.slice(input.length).map(function(value) { return value - 1; })' }; function Matrix(rows, columns) { this.rows = rows; this.columns = columns; this.weights = zeros(rows * columns); } ${zeros.toString()} ${softmax.toString().replace('_1.Matrix', 'Matrix')} ${randomFloat.toString()} ${sampleI.toString()} ${maxI.toString()}`; // eslint-disable-next-line return new Function( 'rawInput', 'isSampleI', 'temperature', cb ? cb(src) : src ) as RNNFunction; } trainPattern(input: number[], logErrorRate?: boolean): number { const error = this.trainInput(input); this.backpropagate(input); this.adjustWeights(); if (logErrorRate) { return error; } return 0; } } export interface IRNNJSON { type: string; options: IRNNJSONOptions; trainOpts: IRNNJSONTrainOptions; input: IMatrixJSON; hiddenLayers: Array<{ [index: string]: IMatrixJSON }>; outputConnector: IMatrixJSON; output: IMatrixJSON; } export function last(values: T[]): T { return values[values.length - 1]; } export type RNNFunction = ( rawInput?: Array | string, isSampleI?: boolean, temperature?: number ) => string; ================================================ FILE: src/recurrent.baseline.test.ts ================================================ import { GPU, Input, KernelOutput } from 'gpu.js'; import { Matrix } from './recurrent/matrix'; import { input, output, rnnCell, ILayer, IRecurrentInput } from './layer'; import { IMomentumRootMeanSquaredPropagationSettings } from './praxis/momentum-root-mean-squared-propagation'; import { Recurrent } from './recurrent'; import { RNNTimeStep } from './recurrent/rnn-time-step'; import { setup, teardown } from './utilities/kernel'; jest.mock('./utilities/randos', () => { return { randos2D: (width: number, height: number) => { const weights: Float32Array[] = []; let value = 1; for (let rowIndex = 0; rowIndex < height; rowIndex++) { weights[rowIndex] = new Float32Array(width); for (let columnIndex = 0; columnIndex < width; columnIndex++) { weights[rowIndex][columnIndex] = value++; } } return weights; }, }; }); jest.mock('./recurrent/matrix/random-matrix', () => { class MockRandomMatrix extends Matrix { constructor(rows: number, columns: number, std: number) { super(rows, columns); let value = 1; for (let row = 0; row < rows; row++) { for (let column = 0; column < columns; column++) { this.setWeight(row, column, value++); } } console.log(this.weights); } } return { RandomMatrix: MockRandomMatrix, }; }); function asArrayOfArrayOfNumber(v: KernelOutput | Input): number[][] { if (!Array.isArray(v) || typeof (v as number[][])[0][0] !== 'number') { throw new Error('unexpected value'); } return v as number[][]; } function asMatrix(v?: Matrix): Matrix { if (!v) throw new Error('undefined Matrix'); return v; } describe('Recurrent Class: Baseline', () => { beforeEach(() => { setup( new GPU({ mode: 'cpu', }) ); }); afterEach(() => { teardown(); }); describe('when configured like RNNTimeStep', () => { function setupNets(): { timeStep: RNNTimeStep; recurrentNet: Recurrent; } { const timeStep: RNNTimeStep = new RNNTimeStep({ regc: 0.000001, inputSize: 1, hiddenLayers: [3], outputSize: 1, }); const praxisOpts: Partial = { regularizationStrength: timeStep.options.regc, learningRate: timeStep.trainOpts.learningRate, }; const recurrentNet = new Recurrent({ praxisOpts, inputLayer: () => input({ height: 1 }), hiddenLayers: [ (inputLayer: ILayer, recurrentInput: IRecurrentInput) => { return rnnCell({ width: 1, height: 3 }, inputLayer, recurrentInput); }, ], outputLayer: (inputLayer: ILayer) => output({ height: 1 }, inputLayer), }); timeStep.initialize(); recurrentNet.initialize(); timeStep.bindEquation(); return { timeStep, recurrentNet }; } describe('forward propagation', () => { function testRecurrentLayerSet( timeStep: RNNTimeStep, recurrentNet: Recurrent, index: number ) { const layerSet = recurrentNet._layerSets[index]; expect(layerSet.length).toBe(15); expect(asArrayOfArrayOfNumber(layerSet[0].weights)[0][0]).toBe( (timeStep.model.equations[index].inputValue as Float32Array)[0] ); expect(asArrayOfArrayOfNumber(layerSet[1].weights)[0][0]).toBe( timeStep.model.hiddenLayers[0].weight.weights[0] ); expect(asArrayOfArrayOfNumber(layerSet[1].weights)[1][0]).toBe( timeStep.model.hiddenLayers[0].weight.weights[1] ); expect(asArrayOfArrayOfNumber(layerSet[1].weights)[2][0]).toBe( timeStep.model.hiddenLayers[0].weight.weights[2] ); expect(asArrayOfArrayOfNumber(layerSet[2].weights)[0][0]).toBe( timeStep.model.equations[index].states[1].product.weights[0] ); expect(asArrayOfArrayOfNumber(layerSet[2].weights)[1][0]).toBe( timeStep.model.equations[index].states[1].product.weights[1] ); expect(asArrayOfArrayOfNumber(layerSet[2].weights)[2][0]).toBe( timeStep.model.equations[index].states[1].product.weights[2] ); expect(asArrayOfArrayOfNumber(layerSet[3].weights)[0][0]).toBe( timeStep.model.hiddenLayers[0].transition.weights[0] ); expect(asArrayOfArrayOfNumber(layerSet[3].weights)[0][1]).toBe( timeStep.model.hiddenLayers[0].transition.weights[1] ); expect(asArrayOfArrayOfNumber(layerSet[3].weights)[0][2]).toBe( timeStep.model.hiddenLayers[0].transition.weights[2] ); expect(asArrayOfArrayOfNumber(layerSet[3].weights)[1][0]).toBe( timeStep.model.hiddenLayers[0].transition.weights[3] ); expect(asArrayOfArrayOfNumber(layerSet[3].weights)[1][1]).toBe( timeStep.model.hiddenLayers[0].transition.weights[4] ); expect(asArrayOfArrayOfNumber(layerSet[3].weights)[1][2]).toBe( timeStep.model.hiddenLayers[0].transition.weights[5] ); expect(asArrayOfArrayOfNumber(layerSet[3].weights)[2][0]).toBe( timeStep.model.hiddenLayers[0].transition.weights[6] ); expect(asArrayOfArrayOfNumber(layerSet[3].weights)[2][1]).toBe( timeStep.model.hiddenLayers[0].transition.weights[7] ); expect(asArrayOfArrayOfNumber(layerSet[3].weights)[2][2]).toBe( timeStep.model.hiddenLayers[0].transition.weights[8] ); expect(asArrayOfArrayOfNumber(layerSet[4].weights)[0][0]).toBe( asMatrix(timeStep.model.equations[index].states[2].right).weights[0] ); expect(asArrayOfArrayOfNumber(layerSet[4].weights)[1][0]).toBe( asMatrix(timeStep.model.equations[index].states[2].right).weights[1] ); expect(asArrayOfArrayOfNumber(layerSet[4].weights)[2][0]).toBe( asMatrix(timeStep.model.equations[index].states[2].right).weights[2] ); expect(asArrayOfArrayOfNumber(layerSet[5].weights)[0][0]).toBe( timeStep.model.equations[index].states[2].product.weights[0] ); expect(asArrayOfArrayOfNumber(layerSet[5].weights)[1][0]).toBe( timeStep.model.equations[index].states[2].product.weights[1] ); expect(asArrayOfArrayOfNumber(layerSet[5].weights)[2][0]).toBe( timeStep.model.equations[index].states[2].product.weights[2] ); expect(asArrayOfArrayOfNumber(layerSet[6].weights)[0][0]).toBe( timeStep.model.equations[index].states[3].product.weights[0] ); expect(asArrayOfArrayOfNumber(layerSet[6].weights)[1][0]).toBe( timeStep.model.equations[index].states[3].product.weights[1] ); expect(asArrayOfArrayOfNumber(layerSet[6].weights)[2][0]).toBe( timeStep.model.equations[index].states[3].product.weights[2] ); expect(asArrayOfArrayOfNumber(layerSet[8].weights)[0][0]).toBe( timeStep.model.equations[index].states[4].product.weights[0] ); expect(asArrayOfArrayOfNumber(layerSet[8].weights)[1][0]).toBe( timeStep.model.equations[index].states[4].product.weights[1] ); expect(asArrayOfArrayOfNumber(layerSet[8].weights)[2][0]).toBe( timeStep.model.equations[index].states[4].product.weights[2] ); expect(asArrayOfArrayOfNumber(layerSet[9].weights)[0][0]).toBe( timeStep.model.equations[index].states[5].product.weights[0] ); expect(asArrayOfArrayOfNumber(layerSet[9].weights)[1][0]).toBe( timeStep.model.equations[index].states[5].product.weights[1] ); expect(asArrayOfArrayOfNumber(layerSet[9].weights)[2][0]).toBe( timeStep.model.equations[index].states[5].product.weights[2] ); expect(asArrayOfArrayOfNumber(layerSet[10].weights)[0][0]).toBe( timeStep.model.outputConnector.weights[0] ); expect(asArrayOfArrayOfNumber(layerSet[10].weights)[0][1]).toBe( timeStep.model.outputConnector.weights[1] ); expect(asArrayOfArrayOfNumber(layerSet[10].weights)[0][2]).toBe( timeStep.model.outputConnector.weights[2] ); expect(asArrayOfArrayOfNumber(layerSet[11].weights)[0][0]).toBe( timeStep.model.equations[index].states[6].product.weights[0] ); expect(asArrayOfArrayOfNumber(layerSet[12].weights)[0][0]).toBe( timeStep.model.output.weights[0] ); expect(asArrayOfArrayOfNumber(layerSet[13].weights)[0][0]).toBe( timeStep.model.equations[index].states[7].product.weights[0] ); expect(asArrayOfArrayOfNumber(layerSet[14].weights)[0][0]).toBe( timeStep.model.equations[index].states[7].product.weights[0] ); } test('.run() is equivalent to baseline', () => { const { timeStep, recurrentNet } = setupNets(); const timeStepResult = timeStep.run([100, 500]); const recurrentResult = recurrentNet.run([[100], [500]]); expect(recurrentNet._layerSets.length).toBe( timeStep.model.equations.length ); testRecurrentLayerSet(timeStep, recurrentNet, 0); testRecurrentLayerSet(timeStep, recurrentNet, 1); testRecurrentLayerSet(timeStep, recurrentNet, 2); expect(recurrentResult[0][0]).toBe(timeStepResult); }); test('.train() is equivalent to baseline', () => { const { timeStep, recurrentNet } = setupNets(); timeStep.adjustWeights = () => {}; recurrentNet.adjustWeights = () => {}; timeStep.train([[100, 500, 1000]], { iterations: 1 }); recurrentNet.train([[[100], [500], [1000]]], { iterations: 1, errorCheckInterval: 1, logPeriod: 1, }); expect(recurrentNet._layerSets.length).toBe( timeStep.model.equations.length ); testRecurrentLayerSet(timeStep, recurrentNet, 0); testRecurrentLayerSet(timeStep, recurrentNet, 1); testRecurrentLayerSet(timeStep, recurrentNet, 2); }); }); describe('back propagation', () => { test('.compare() via .train() is equivalent to baseline', () => { const { timeStep, recurrentNet } = setupNets(); function testRecurrentLayerSet(index: number) { expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][14].deltas )[0][0] ).toBe( asMatrix(timeStep.model.equations[index].states[7].product) .deltas[0] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][13].deltas )[0][0] ).toBe( asMatrix(timeStep.model.equations[index].states[7].product) .deltas[0] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][12].deltas )[0][0] ).toBe( asMatrix(timeStep.model.equations[index].states[7].right).deltas[0] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][11].deltas )[0][0] ).toBe( asMatrix(timeStep.model.equations[index].states[6].product) .deltas[0] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][10].deltas )[0][0] ).toBe( asMatrix(timeStep.model.equations[index].states[6].left).deltas[0] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][10].deltas )[0][1] ).toBe( asMatrix(timeStep.model.equations[index].states[6].left).deltas[1] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][10].deltas )[0][2] ).toBe( asMatrix(timeStep.model.equations[index].states[6].left).deltas[2] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][9].deltas )[0][0] ).toBe( asMatrix(timeStep.model.equations[index].states[5].product) .deltas[0] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][9].deltas )[1][0] ).toBe( asMatrix(timeStep.model.equations[index].states[5].product) .deltas[1] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][9].deltas )[2][0] ).toBe( asMatrix(timeStep.model.equations[index].states[5].product) .deltas[2] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][8].deltas )[0][0] ).toBe( asMatrix(timeStep.model.equations[index].states[4].product) .deltas[0] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][8].deltas )[1][0] ).toBe( asMatrix(timeStep.model.equations[index].states[4].product) .deltas[1] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][8].deltas )[2][0] ).toBe( asMatrix(timeStep.model.equations[index].states[4].product) .deltas[2] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][6].deltas )[0][0] ).toBe( asMatrix(timeStep.model.equations[index].states[3].product) .deltas[0] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][6].deltas )[1][0] ).toBe( asMatrix(timeStep.model.equations[index].states[3].product) .deltas[1] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][6].deltas )[2][0] ).toBe( asMatrix(timeStep.model.equations[index].states[3].product) .deltas[2] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][5].deltas )[0][0] ).toBe( asMatrix(timeStep.model.equations[index].states[2].product) .deltas[0] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][5].deltas )[1][0] ).toBe( asMatrix(timeStep.model.equations[index].states[2].product) .deltas[1] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][5].deltas )[2][0] ).toBe( asMatrix(timeStep.model.equations[index].states[2].product) .deltas[2] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][4].deltas )[0][0] ).toBe( asMatrix(timeStep.model.equations[index].states[2].right).deltas[0] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][4].deltas )[1][0] ).toBe( asMatrix(timeStep.model.equations[index].states[2].right).deltas[1] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][4].deltas )[2][0] ).toBe( asMatrix(timeStep.model.equations[index].states[2].right).deltas[2] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][3].deltas )[0][0] ).toBe( asMatrix(timeStep.model.equations[index].states[2].left).deltas[0] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][3].deltas )[0][1] ).toBe( asMatrix(timeStep.model.equations[index].states[2].left).deltas[1] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][3].deltas )[0][2] ).toBe( asMatrix(timeStep.model.equations[index].states[2].left).deltas[2] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][3].deltas )[1][0] ).toBe( asMatrix(timeStep.model.equations[index].states[2].left).deltas[3] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][3].deltas )[1][1] ).toBe( asMatrix(timeStep.model.equations[index].states[2].left).deltas[4] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][3].deltas )[1][2] ).toBe( asMatrix(timeStep.model.equations[index].states[2].left).deltas[5] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][3].deltas )[2][0] ).toBe( asMatrix(timeStep.model.equations[index].states[2].left).deltas[6] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][3].deltas )[2][1] ).toBe( asMatrix(timeStep.model.equations[index].states[2].left).deltas[7] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][3].deltas )[2][2] ).toBe( asMatrix(timeStep.model.equations[index].states[2].left).deltas[8] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][2].deltas )[0][0] ).toBe( asMatrix(timeStep.model.equations[index].states[1].product) .deltas[0] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][2].deltas )[1][0] ).toBe( asMatrix(timeStep.model.equations[index].states[1].product) .deltas[1] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][2].deltas )[2][0] ).toBe( asMatrix(timeStep.model.equations[index].states[1].product) .deltas[2] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][1].deltas )[0][0] ).toBe( asMatrix(timeStep.model.equations[index].states[1].left).deltas[0] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][1].deltas )[1][0] ).toBe( asMatrix(timeStep.model.equations[index].states[1].left).deltas[1] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][1].deltas )[2][0] ).toBe( asMatrix(timeStep.model.equations[index].states[1].left).deltas[2] ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][0].deltas )[0][0] ).toBe( asMatrix(timeStep.model.equations[index].states[1].right).deltas[0] ); } timeStep.adjustWeights = () => {}; const timeStepResult = timeStep.train([[100, 500, 1000]], { iterations: 1, }); recurrentNet.adjustWeights = () => {}; const recurrentNetResult = recurrentNet.train( [[[100], [500], [1000]]], { iterations: 1, errorCheckInterval: 1, logPeriod: 1 } ); expect(recurrentNetResult.error.toFixed(2)).toBe( timeStepResult.error.toFixed(2) ); expect(recurrentNet._layerSets.length).toBe( timeStep.model.equations.length ); testRecurrentLayerSet(2); testRecurrentLayerSet(1); testRecurrentLayerSet(0); }); test('.learn() via .train() is equivalent to baseline', () => { const { timeStep, recurrentNet } = setupNets(); function testRecurrentModel() { const model = recurrentNet._model; if (!model) return; expect(asArrayOfArrayOfNumber(model[0].weights)[0][0]).toBe( timeStep.model.allMatrices[0].weights[0] ); expect(asArrayOfArrayOfNumber(model[0].weights)[1][0]).toBe( timeStep.model.allMatrices[0].weights[1] ); expect(asArrayOfArrayOfNumber(model[0].weights)[2][0]).toBe( timeStep.model.allMatrices[0].weights[2] ); expect(asArrayOfArrayOfNumber(model[1].weights)[0][0]).toBe( timeStep.model.allMatrices[1].weights[0] ); expect(asArrayOfArrayOfNumber(model[1].weights)[0][1]).toBe( timeStep.model.allMatrices[1].weights[1] ); expect(asArrayOfArrayOfNumber(model[1].weights)[0][2]).toBe( timeStep.model.allMatrices[1].weights[2] ); expect(asArrayOfArrayOfNumber(model[1].weights)[1][0]).toBe( timeStep.model.allMatrices[1].weights[3] ); expect(asArrayOfArrayOfNumber(model[1].weights)[1][1]).toBe( timeStep.model.allMatrices[1].weights[4] ); expect(asArrayOfArrayOfNumber(model[1].weights)[1][2]).toBe( timeStep.model.allMatrices[1].weights[5] ); expect(asArrayOfArrayOfNumber(model[1].weights)[2][0]).toBe( timeStep.model.allMatrices[1].weights[6] ); expect(asArrayOfArrayOfNumber(model[1].weights)[2][1]).toBe( timeStep.model.allMatrices[1].weights[7] ); expect(asArrayOfArrayOfNumber(model[1].weights)[2][2]).toBe( timeStep.model.allMatrices[1].weights[8] ); expect(asArrayOfArrayOfNumber(model[2].weights)[0][0]).toBe( timeStep.model.allMatrices[2].weights[0] ); expect(asArrayOfArrayOfNumber(model[2].weights)[1][0]).toBeCloseTo( timeStep.model.allMatrices[2].weights[1], 0.00000000009 ); expect(asArrayOfArrayOfNumber(model[2].weights)[2][0]).toBeCloseTo( timeStep.model.allMatrices[2].weights[2], 0.00000000009 ); expect(asArrayOfArrayOfNumber(model[3].weights)[0][0]).toBe( timeStep.model.allMatrices[3].weights[0] ); expect(asArrayOfArrayOfNumber(model[3].weights)[0][1]).toBe( timeStep.model.allMatrices[3].weights[1] ); expect(asArrayOfArrayOfNumber(model[3].weights)[0][2]).toBe( timeStep.model.allMatrices[3].weights[2] ); expect(asArrayOfArrayOfNumber(model[4].weights)[0][0]).toBe( timeStep.model.allMatrices[4].weights[0] ); } const timeStepResult = timeStep.train([[100, 500, 1000]], { iterations: 1, }); const recurrentNetResult = recurrentNet.train( [[[100], [500], [1000]]], { iterations: 1, errorCheckInterval: 1, logPeriod: 1, } ); expect(recurrentNetResult.iterations).toBe(timeStepResult.iterations); expect(recurrentNetResult.error).toBeCloseTo( timeStepResult.error, 0.005 ); expect(recurrentNet._layerSets.length).toBe( timeStep.model.equations.length ); testRecurrentModel(); expect(recurrentNet.run([[2]])[0][0]).toBe(timeStep.run([2])); }); }); describe('forward propagate and backpropagate', () => { test('.train() is equivalent to baseline', () => { const { timeStep, recurrentNet } = setupNets(); function testRecurrentLayerSetWeights( timeStep: RNNTimeStep, recurrentNet: Recurrent, index: number ) { expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][14].weights )[0][0].toFixed(5) ).toBe( timeStep.model.equations[ index ].states[7].product.weights[0].toFixed(5) ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][13].weights )[0][0].toFixed(5) ).toBe( timeStep.model.equations[ index ].states[7].product.weights[0].toFixed(5) ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][12].weights )[0][0].toFixed(5) ).toBe(timeStep.model.output.weights[0].toFixed(5)); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][11].weights )[0][0].toFixed(5) ).toBe( timeStep.model.equations[ index ].states[6].product.weights[0].toFixed(5) ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][10].weights )[0][0].toFixed(5) ).toBe(timeStep.model.outputConnector.weights[0].toFixed(5)); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][10].weights )[0][1].toFixed(5) ).toBe(timeStep.model.outputConnector.weights[1].toFixed(5)); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][10].weights )[0][2].toFixed(5) ).toBe(timeStep.model.outputConnector.weights[2].toFixed(5)); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][9].weights )[0][0].toFixed(5) ).toBe( timeStep.model.equations[ index ].states[5].product.weights[0].toFixed(5) ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][9].weights )[1][0].toFixed(5) ).toBe( timeStep.model.equations[ index ].states[5].product.weights[1].toFixed(5) ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][9].weights )[2][0].toFixed(5) ).toBe( timeStep.model.equations[ index ].states[5].product.weights[2].toFixed(5) ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][8].weights )[0][0].toFixed(5) ).toBe( timeStep.model.equations[ index ].states[4].product.weights[0].toFixed(5) ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][8].weights )[1][0].toFixed(5) ).toBe( timeStep.model.equations[ index ].states[4].product.weights[1].toFixed(5) ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][8].weights )[2][0].toFixed(5) ).toBe( timeStep.model.equations[ index ].states[4].product.weights[2].toFixed(5) ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][6].weights )[0][0].toFixed(5) ).toBe( timeStep.model.equations[ index ].states[3].product.weights[0].toFixed(5) ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][6].weights )[1][0].toFixed(5) ).toBe( timeStep.model.equations[ index ].states[3].product.weights[1].toFixed(5) ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][6].weights )[2][0].toFixed(5) ).toBe( timeStep.model.equations[ index ].states[3].product.weights[2].toFixed(5) ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][5].weights )[0][0].toFixed(5) ).toBe( timeStep.model.equations[ index ].states[2].product.weights[0].toFixed(5) ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][5].weights )[1][0].toFixed(5) ).toBe( timeStep.model.equations[ index ].states[2].product.weights[1].toFixed(5) ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][5].weights )[2][0].toFixed(5) ).toBe( timeStep.model.equations[ index ].states[2].product.weights[2].toFixed(5) ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][4].weights )[0][0].toFixed(5) ).toBe( asMatrix( timeStep.model.equations[index].states[2].right ).weights[0].toFixed(5) ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][4].weights )[1][0].toFixed(5) ).toBe( asMatrix( timeStep.model.equations[index].states[2].right ).weights[1].toFixed(5) ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][4].weights )[2][0].toFixed(5) ).toBe( asMatrix( timeStep.model.equations[index].states[2].right ).weights[2].toFixed(5) ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][3].weights )[0][0].toFixed(5) ).toBe( timeStep.model.hiddenLayers[0].transition.weights[0].toFixed(5) ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][3].weights )[0][1].toFixed(5) ).toBe( timeStep.model.hiddenLayers[0].transition.weights[1].toFixed(5) ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][3].weights )[0][2].toFixed(5) ).toBe( timeStep.model.hiddenLayers[0].transition.weights[2].toFixed(5) ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][3].weights )[1][0].toFixed(5) ).toBe( timeStep.model.hiddenLayers[0].transition.weights[3].toFixed(5) ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][3].weights )[1][1].toFixed(5) ).toBe( timeStep.model.hiddenLayers[0].transition.weights[4].toFixed(5) ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][3].weights )[1][2].toFixed(5) ).toBe( timeStep.model.hiddenLayers[0].transition.weights[5].toFixed(5) ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][3].weights )[2][0].toFixed(5) ).toBe( timeStep.model.hiddenLayers[0].transition.weights[6].toFixed(5) ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][3].weights )[2][1].toFixed(5) ).toBe( timeStep.model.hiddenLayers[0].transition.weights[7].toFixed(5) ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][3].weights )[2][2].toFixed(5) ).toBe( timeStep.model.hiddenLayers[0].transition.weights[8].toFixed(5) ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][2].weights )[0][0].toFixed(5) ).toBe( timeStep.model.equations[ index ].states[1].product.weights[0].toFixed(5) ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][2].weights )[1][0].toFixed(5) ).toBe( timeStep.model.equations[ index ].states[1].product.weights[1].toFixed(5) ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][2].weights )[2][0].toFixed(5) ).toBe( timeStep.model.equations[ index ].states[1].product.weights[2].toFixed(5) ); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][1].weights )[0][0].toFixed(5) ).toBe(timeStep.model.hiddenLayers[0].weight.weights[0].toFixed(5)); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][1].weights )[1][0].toFixed(5) ).toBe(timeStep.model.hiddenLayers[0].weight.weights[1].toFixed(5)); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][1].weights )[2][0].toFixed(5) ).toBe(timeStep.model.hiddenLayers[0].weight.weights[2].toFixed(5)); expect( asArrayOfArrayOfNumber( recurrentNet._layerSets[index][0].weights )[0][0].toFixed(5) ).toBe( (timeStep.model.equations[index] .inputValue as Float32Array)[0].toFixed(5) ); } timeStep.train( [ [100, 500, 1000], [1000, 500, 100], ], { iterations: 100 } ); recurrentNet.train( [ [[100], [500], [1000]], [[1000], [500], [100]], ], { iterations: 100 } ); expect(recurrentNet._layerSets.length).toBe( timeStep.model.equations.length ); testRecurrentLayerSetWeights(timeStep, recurrentNet, 0); testRecurrentLayerSetWeights(timeStep, recurrentNet, 1); testRecurrentLayerSetWeights(timeStep, recurrentNet, 2); console.log(recurrentNet.run([[100], [500]]), timeStep.run([100, 500])); console.log( recurrentNet.run([[1000], [500]]), timeStep.run([1000, 500]) ); expect(recurrentNet.run([[100], [500]])[0][0]).toBe( timeStep.run([100, 500]) ); expect(recurrentNet.run([[1000], [500]])[0][0]).toBe( timeStep.run([1000, 500]) ); }); }); }); }); ================================================ FILE: src/recurrent.end-to-end.test.ts ================================================ import { GPU } from 'gpu.js'; import { add, random, input, lstmCell, multiply, output, rnnCell, ILayer, IRecurrentInput, } from './layer'; import { Recurrent } from './recurrent'; import { setup, teardown } from './utilities/kernel'; describe('Recurrent Class: End to End', () => { beforeEach(() => { setup( new GPU({ mode: 'cpu', }) ); }); afterEach(() => { teardown(); }); describe('training life-cycle', () => { test('properly instantiates starts with random weights and zero deltas and back propagates values through weights', () => { const net = new Recurrent({ inputLayer: () => input({ height: 1 }), hiddenLayers: [ (inputLayer: ILayer, recurrentInput: IRecurrentInput) => { if (recurrentInput.setDimensions) { recurrentInput.setDimensions(1, 3); } return add( multiply(random({ height: 3 }), inputLayer), recurrentInput ); }, ], outputLayer: (inputLayer: ILayer) => output({ height: 1 }, inputLayer), }); net.initialize(); net.initializeDeep(); const datum = [[1], [1]]; net.runInputs(datum); expect(net._model?.length).toEqual(3); expect(net._layerSets.length).toEqual(2); expect(net._layerSets[0].length).toEqual(10); expect(net._layerSets[1].length).toEqual(10); const clonedModelWeights = net._model?.map( (l: ILayer): Float32Array[] => { return (l.weights as Float32Array[]).map( (row: Float32Array): Float32Array => { return row.slice(0); } ); } ); function deltasAreZero() { expect( net._layerSets[0].every((l: ILayer) => (l.deltas as number[][]).every((row) => row.every((delta: number) => delta === 0) ) ) ).toBeTruthy(); } function deltasAreSet() { expect( net._layerSets[0].every((l: ILayer) => (l.deltas as number[][]).every((row) => row.every((delta: number) => delta !== 0) ) ) ).toBeTruthy(); } function modelWeightsAreUpdated() { expect( clonedModelWeights?.every( (oldLayerWeights: Float32Array[], layerIndex: number) => oldLayerWeights.every((row: Float32Array, rowIndex: number) => row.every((oldWeight, columnIndex) => { const model = net._model; if (!model) return true; const newLayerWeights = model[layerIndex] .weights as Float32Array[]; if (layerIndex === 0) return true; return oldWeight !== newLayerWeights[rowIndex][columnIndex]; }) ) ) ).toBeTruthy(); } function modelDeltasAreZero() { expect( net._model?.every((l: ILayer) => (l.deltas as number[][]).every((row) => row.every((delta: number) => delta === 0) ) ) ).toBeTruthy(); } deltasAreZero(); // two arbitrary values that are not zero net._calculateDeltas(datum); deltasAreSet(); net.adjustWeights(); modelWeightsAreUpdated(); modelDeltasAreZero(); }); }); describe('.initializeDeep()', () => { describe('structure', () => { test('can create new hidden layers in the correct structure', () => { const inputLayer = input({ height: 1 }); const weights = random({ height: 3 }); let recurrentInput: IRecurrentInput | null = null; const net = new Recurrent({ inputLayer: () => inputLayer, hiddenLayers: [ (inputLayer: ILayer, _recurrentInput: IRecurrentInput) => { if (_recurrentInput.setDimensions) { _recurrentInput.setDimensions(1, 3); } recurrentInput = _recurrentInput; return add(multiply(weights, inputLayer), _recurrentInput); }, ], outputLayer: (inputLayer: ILayer) => output({ height: 1 }, inputLayer), }); // single net.initialize(); if (!recurrentInput) throw new Error('recurrentInput is not defined'); expect(net._layerSets.length).toEqual(1); expect(net._layerSets[0].length).toEqual(10); expect(net._layerSets[0][0]).toEqual(inputLayer); expect(net._layerSets[0].indexOf(weights)).toBe(1); expect(net._layerSets[0].indexOf(recurrentInput)).toBe(3); // double net.initializeDeep(); expect(net._layerSets.length).toEqual(2); expect(net._layerSets[1].length).toEqual(10); expect(net._layerSets[1][0]).not.toBe(inputLayer); expect(net._layerSets[1][0].constructor).toEqual( inputLayer.constructor ); // new instance of same type NOT model expect(net._layerSets[1].indexOf(weights)).toBe(1); // direct reference IMPORTANT because model expect(net._layerSets[1].indexOf(recurrentInput)).toBe(-1); expect(net._layerSets[1][3].deltas).toBe(net._layerSets[0][4].deltas); // recurrence expect(net._layerSets[1][3].weights).toBe(net._layerSets[0][4].weights); // recurrence // triple net.initializeDeep(); expect(net._layerSets.length).toEqual(3); expect(net._layerSets[2].length).toEqual(10); expect(net._layerSets[2][0]).not.toBe(inputLayer); expect(net._layerSets[2][0].constructor).toEqual( inputLayer.constructor ); // new instance of same type NOT model expect(net._layerSets[2].indexOf(weights)).toBe(1); // direct reference IMPORTANT because model expect(net._layerSets[2][3].constructor.name).toBe('RecurrentInput'); expect(net._layerSets[2][3].deltas).toBe(net._layerSets[1][4].deltas); // recurrence expect(net._layerSets[2][3].weights).toBe(net._layerSets[1][4].weights); // recurrence }); }); }); test('can learn', () => { const net = new Recurrent({ inputLayer: () => input({ width: 1 }), hiddenLayers: [ (inputLayer: ILayer, recurrentInput: IRecurrentInput) => rnnCell({ width: 1, height: 1 }, inputLayer, recurrentInput), ], outputLayer: (inputLayer: ILayer) => output({ width: 1, height: 1 }, inputLayer), }); net.initialize(); net.initializeDeep(); expect(net._layerSets.length).toEqual(2); expect(net._layerSets[0].length).toEqual(15); expect(net._layerSets[1].length).toEqual(15); const errors = []; for (let i = 0; i < 20; i++) { errors.push( (net._trainPattern( [ [1, 2], [1, 3], ], true ) as number[])[0] ); } expect(errors[0]).toBeGreaterThan(errors[errors.length - 1]); }); test('can have more than one hiddenLayer', () => { expect(() => { try { const net = new Recurrent({ inputLayer: () => input({ width: 1 }), hiddenLayers: [ (inputLayer: ILayer, recurrentInput: IRecurrentInput) => rnnCell({ height: 3, width: 1 }, inputLayer, recurrentInput), (inputLayer: ILayer, recurrentInput: IRecurrentInput) => rnnCell({ height: 1, width: 1 }, inputLayer, recurrentInput), ], outputLayer: (inputLayer: ILayer) => output({ height: 1 }, inputLayer), }); net.initialize(); } catch (e) { throw new Error(); } }).not.toThrow(); }); test('can learn to increment', () => { const net = new Recurrent({ inputLayer: () => input({ height: 1 }), hiddenLayers: [ (inputLayer: ILayer, recurrentInput: IRecurrentInput) => rnnCell({ height: 3 }, inputLayer, recurrentInput), ], outputLayer: (inputLayer: ILayer) => output({ height: 1 }, inputLayer), }); net.initialize(); net.initializeDeep(); expect(net._model?.length).toEqual(5); expect(net._layerSets.length).toEqual(2); expect(net._layerSets[0].length).toEqual(15); expect(net._layerSets[1].length).toEqual(15); let error; for (let i = 0; i < 100; i++) { error = (net._trainPattern([[0], [1]], true) as number[])[0]; } expect(error as number).toBeLessThan(0.005); }); it('can learn xor', () => { const net = new Recurrent({ inputLayer: () => input({ height: 1 }), hiddenLayers: [ (inputLayer: ILayer, recurrentInput: IRecurrentInput) => lstmCell({ height: 10 }, inputLayer, recurrentInput), ], outputLayer: (inputLayer: ILayer) => output({ height: 1 }, inputLayer), }); const xorNetValues = [ [[0.001], [0.001], [0.001]], [[0.001], [1], [1]], [[1], [0.001], [1]], [[1], [1], [0.001]], ]; const errorThresh = 0.03; const iterations = 5000; const status = net.train(xorNetValues, { errorThresh, iterations }); expect( status.error <= errorThresh || status.iterations <= iterations ).toBeTruthy(); expect(net.run([[0.001], [0.001]])[0][0]).toBeLessThan(0.1); expect(net.run([[0.001], [1]])[0][0]).toBeGreaterThan(0.9); expect(net.run([[1], [0.001]])[0][0]).toBeGreaterThan(0.9); expect(net.run([[1], [1]])[0][0]).toBeLessThan(0.1); }); test('can learn 1,2,3', () => { const net = new Recurrent({ inputLayer: () => input({ height: 1 }), hiddenLayers: [ (inputLayer: ILayer, recurrentInput: IRecurrentInput) => lstmCell({ height: 10 }, inputLayer, recurrentInput), ], outputLayer: (inputLayer: ILayer) => output({ height: 1 }, inputLayer), }); const iterations = 101; const errorThresh = 0.005; const status = net.train([[[1], [2], [3]]], { iterations: 101, errorThresh, }); expect( status.iterations <= iterations || status.error < errorThresh ).toBeTruthy(); }); test('can learn 1,2,3 using .train()', () => { const net = new Recurrent({ inputLayer: () => input({ height: 1 }), hiddenLayers: [ (inputLayer: ILayer, recurrentInput: IRecurrentInput) => lstmCell({ height: 3 }, inputLayer, recurrentInput), ], outputLayer: (inputLayer: ILayer) => output({ height: 1 }, inputLayer), }); const results = net.train([[[1], [2], [3]]]); expect(results.error < 0.01).toBeTruthy(); expect(Math.round(net.run([[1], [2]])[0][0])).toBe(3); }); }); ================================================ FILE: src/recurrent.ts ================================================ import { RecurrentConnection } from './layer/recurrent-connection'; import { IRecurrentInput, RecurrentInput, RecurrentZeros, ILayer, ILayerSettings, } from './layer'; import { Activation, EntryPoint, EntryPointType, Filter, Internal, InternalModel, Model, Modifier, Operator, Target, } from './layer/types'; import { flattenLayers } from './utilities/flatten-layers'; import { FeedForward, IFeedForwardOptions, IFeedForwardTrainingOptions, ITrainingStatus, } from './feed-forward'; import { release, clone } from './utilities/kernel'; import { KernelOutput, Texture, TextureArrayOutput } from 'gpu.js'; import { OperatorType } from './layer/operator'; import { ModifierType } from './layer/modifier'; import { FilterType } from './layer/filter'; import { ActivationType } from './layer/activation'; import { TargetType } from './layer/target'; export interface IRecurrentTrainingOptions extends IFeedForwardTrainingOptions {} // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error export interface IRecurrentOptions extends IFeedForwardOptions { hiddenLayers: Array< ( inputLayer: ILayer, recurrentInput: IRecurrentInput, index: number ) => ILayer >; } export interface IRecurrentPreppedTrainingData { status: ITrainingStatus; preparedData: T[][]; endTime: number; } export class Recurrent< T extends TextureArrayOutput = TextureArrayOutput > extends FeedForward { trainOpts: IRecurrentTrainingOptions = {}; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error options: IRecurrentOptions; _outputConnection: RecurrentConnection | null = null; _layerSets: ILayer[][] = []; _hiddenLayerOutputIndices: number[] = []; _model: ILayer[] | null = null; // TODO: use generics in extend constructor( options: Partial = {} ) { // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error super(options); } _connectLayers(): { inputLayer: ILayer; hiddenLayers: ILayer[]; outputLayer: ILayer; } { if (!this.options.inputLayer) { throw new Error('inputLayer not found'); } if (!this.options.outputLayer) { throw new Error('outputLayer not found'); } const inputLayer = this.options.inputLayer(); const hiddenLayers = this._connectHiddenLayers(inputLayer); const outputLayer = this.options.outputLayer( hiddenLayers[hiddenLayers.length - 1], -1 ); return { inputLayer, hiddenLayers, outputLayer, }; } _connectLayersDeep(): ILayer[] { const layers: ILayer[] = []; const previousLayers = this._layerSets[this._layerSets.length - 1]; let usedHiddenLayerOutputIndex = 0; function findInputLayer(inputLayer: ILayer) { const index = previousLayers.indexOf(inputLayer); if (index < 0) throw new Error('unable to find layer'); return layers[index]; } function layerSettings(layer: ILayer): ILayerSettings { return { ...layer.settings, weights: null, deltas: null, praxis: null, }; } for (let i = 0; i < previousLayers.length; i++) { const previousLayer = previousLayers[i]; let layer: ILayer; if (previousLayer instanceof Activation) { layer = new (previousLayer.constructor as ActivationType)( findInputLayer(previousLayer.inputLayer), layerSettings(previousLayer) ); } else if (previousLayer instanceof EntryPoint) { layer = new (previousLayer.constructor as EntryPointType)( layerSettings(previousLayer) ); } else if (previousLayer instanceof Filter) { layer = new (previousLayer.constructor as FilterType)( layerSettings(previousLayer.inputLayer), findInputLayer(previousLayer.inputLayer) ); } else if (previousLayer instanceof Internal) { const previousHiddenLayerOutput = previousLayers[ this._hiddenLayerOutputIndices[usedHiddenLayerOutputIndex++] ]; if (previousLayer instanceof RecurrentConnection) { throw new Error('unfinished'); } else if (previousLayer instanceof RecurrentInput) { layer = new RecurrentInput(previousHiddenLayerOutput); } else if (previousLayer instanceof RecurrentZeros) { layer = new RecurrentInput(previousHiddenLayerOutput); } else { throw new Error( `hidden layer ${previousLayer.constructor.name} extends unknown hidden layer` ); } } else if ( previousLayer instanceof InternalModel || previousLayer instanceof Model ) { layer = previousLayer; } else if (previousLayer instanceof Modifier) { layer = new (previousLayer.constructor as ModifierType)( findInputLayer(previousLayer.inputLayer), layerSettings(previousLayer.inputLayer) ); } else if (previousLayer instanceof Operator) { layer = new (previousLayer.constructor as OperatorType)( findInputLayer(previousLayer.inputLayer1), findInputLayer(previousLayer.inputLayer2), layerSettings(previousLayer) ); } else if (previousLayer instanceof Target) { layer = new (previousLayer.constructor as TargetType)( layerSettings(previousLayer), findInputLayer(previousLayer.inputLayer) ); } else { throw new Error( `hidden layer ${previousLayer.constructor.name} extends unknown hidden layer` ); } layers.push(layer); } return layers; } _connectHiddenLayers(previousLayer: ILayer): ILayer[] { const hiddenLayers = []; if (!this.options.hiddenLayers) throw new Error('hiddenLayers not defined'); for (let i = 0; i < this.options.hiddenLayers.length; i++) { const recurrentInput = new RecurrentZeros(); const hiddenLayer = this.options.hiddenLayers[i]( previousLayer, recurrentInput, i ); previousLayer = hiddenLayer; hiddenLayers.push(hiddenLayer); } return hiddenLayers; } initialize(): void { this._outputConnection = new RecurrentConnection(); let layerSet: ILayer[]; if (this.options.layers) { layerSet = this._connectOptionsLayers(); } else { const { inputLayer, hiddenLayers, outputLayer } = this._connectLayers(); layerSet = flattenLayers([inputLayer, ...hiddenLayers, outputLayer]); this._hiddenLayerOutputIndices = hiddenLayers.map((l) => layerSet.indexOf(l) ); this._inputLayer = inputLayer; this._hiddenLayers = hiddenLayers; this._outputLayer = outputLayer; } this.layers = layerSet; this._layerSets = [layerSet]; this._model = layerSet.filter( (l) => l instanceof Model || l instanceof InternalModel ); this.initializeLayers(layerSet); } initializeDeep(): void { const layers = this._connectLayersDeep(); for (let i = 0; i < layers.length; i++) { const layer = layers[i]; layer.setupKernels(true); layer.reuseKernels(this._layerSets[0][i]); } this._layerSets.push(layers); } // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error run(inputs: T[]): T[] { while (this._layerSets.length <= inputs.length) { this.initializeDeep(); } const result = this.runInputs(inputs); if (result instanceof Texture) { return result.toArray() as T[]; } return result as T[]; } runInput(input: KernelOutput): KernelOutput { throw new Error('use .runInputs()'); } runInputs(inputs: T[]): KernelOutput { while (this._layerSets.length < inputs.length) { this.initializeDeep(); } const max = inputs.length - 1; // last output will be compared with last index for (let x = 0; x <= max; x++) { const layerSet = this._layerSets[x]; layerSet[0].predict(inputs[x]); for (let i = 1; i < layerSet.length; i++) { layerSet[i].predict(); } } const lastLayerUsed = this._layerSets[max]; const result = lastLayerUsed[lastLayerUsed.length - 1].weights; this.end(); return result as KernelOutput; } // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error train( data: T[][], options: Partial = {} ): ITrainingStatus { const { preparedData, status, endTime } = this._prepTraining(data, options); let continueTicking = true; const calculateError = (): number => this._calculateTrainingError(preparedData); const trainPatters = (): void => this._trainPatterns(preparedData); while (continueTicking) { continueTicking = this._trainingTick( status, endTime, calculateError, trainPatters ); } return status; } end(): void { const x = this._layerSets.length - 1; const lastLayerSet = this._layerSets[x]; lastLayerSet[0].predict([new Float32Array([0])]); for (let i = 1; i < lastLayerSet.length; i++) { lastLayerSet[i].predict(); } } // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error transferData(formattedData: T[][]): T[][] { return formattedData; } // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error _prepTraining( data: T[][], options: Partial ): IRecurrentPreppedTrainingData { this._updateTrainingOptions(options); const endTime = this.trainOpts.timeout ? Date.now() + this.trainOpts.timeout : 0; const status = { error: 1, iterations: 0, }; this.verifyIsInitialized(); return { preparedData: this.transferData(data), status, endTime, }; } // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error _calculateTrainingError(data: T[][]): number { if (!this.meanSquaredError) { throw new Error('this.meanSquaredError not setup'); } let sum: KernelOutput = new Float32Array(1); for (let i = 0; i < data.length; ++i) { const prevSum = sum; const error = this._trainPattern(data[i], true) as KernelOutput; sum = this.meanSquaredError.add(sum, error); release(error); release(prevSum); } const result = this.meanSquaredError.divide(data.length, sum); release(sum); if (result instanceof Texture) { const resultArray = result.toArray() as number[]; return resultArray[0]; } return (result as number[])[0]; } // TODO: more types // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error formatData(data: Float32Array): Float32Array { return data; } // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error _calculateDeltas(target: T[]): void { const lastLayerSet = this._layerSets[this._layerSets.length - 1]; // Iterate from the second to last layer backwards, propagating 0's for (let i = lastLayerSet.length - 2; i >= 0; i--) { lastLayerSet[i].compare(); } for (let x = target.length - 2; x >= 0; x--) { const layerSet = this._layerSets[x]; layerSet[layerSet.length - 1].compare(target[x + 1]); for (let i = layerSet.length - 2; i >= 0; i--) { layerSet[i].compare(); } } } adjustWeights(): void { const _model = this._model as ILayer[]; for (let i = 0; i < _model.length; i++) { _model[i].learn(this.options.learningRate ?? 0); } } // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error _trainPatterns(data: T[][]): void { for (let i = 0; i < data.length; ++i) { this._trainPattern(data[i], false); } } // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error _trainPattern(inputs: T[], logErrorRate: boolean): KernelOutput | null { // forward propagate this.runInputs(inputs); // back propagate this._calculateDeltas(inputs); this.adjustWeights(); if (logErrorRate) { if (!this.meanSquaredError) { throw new Error('this.meanSquaredError not setup'); } let error: KernelOutput = new Float32Array(1); for (let i = 0, max = inputs.length - 2; i <= max; i++) { const layerSet = this._layerSets[i]; const lastLayer = layerSet[layerSet.length - 1]; const prevError: KernelOutput = error; error = this.meanSquaredError.addAbsolute( prevError, lastLayer.errors as KernelOutput ); release(prevError); } return clone(this.meanSquaredError.divide(inputs.length, error)); } return null; } } ================================================ FILE: src/recurrent.unit.test.ts ================================================ import { GPU, KernelOutput } from 'gpu.js'; import { add, input, multiply, output, random, rnnCell, IRecurrentInput, ILayer, } from './layer'; import { Filter } from './layer/filter'; import { Recurrent } from './recurrent'; import { Matrix } from './recurrent/matrix'; import { setup, teardown } from './utilities/kernel'; function copy2D(matrix: Partial & number[][]) { return matrix.map((row) => Float32Array.from(row)); } describe('Recurrent Class: Unit', () => { beforeEach(() => { setup( new GPU({ mode: 'cpu', }) ); }); afterEach(() => { teardown(); }); describe('.initialize()', () => { test('can validate a simple recurrent neural network', () => { const net = new Recurrent({ inputLayer: () => input({ height: 2 }), hiddenLayers: [ (inputLayer: ILayer, recurrentInput: IRecurrentInput) => { if (recurrentInput.setDimensions) { recurrentInput.setDimensions(1, 3); } return rnnCell({ height: 3 }, inputLayer, recurrentInput); }, ], outputLayer: (inputLayer: ILayer) => output({ height: 1 }, inputLayer), }); net.initialize(); const layers = net._layerSets ? net._layerSets[net._layerSets.length - 1] : []; expect(layers.map((l: ILayer) => l.constructor.name)).toEqual([ 'Input', 'Random', 'Multiply', 'Random', 'RecurrentZeros', 'Multiply', 'Add', 'Zeros', 'Add', 'Relu', 'Random', 'Multiply', 'Random', 'Add', 'Target', ]); }); }); describe('.runInput()', () => { test('forward propagates', () => { const net = new Recurrent({ inputLayer: () => input({ width: 1 }), hiddenLayers: [ (inputLayer: ILayer, recurrentInput: IRecurrentInput) => { if (recurrentInput.setDimensions) { recurrentInput.setDimensions(1, 1); } return multiply( multiply(random({ width: 1, height: 1 }), inputLayer), recurrentInput ); }, ], outputLayer: (inputLayer: ILayer) => output({ width: 1, height: 1 }, inputLayer), }); net.initialize(); net.initializeDeep(); const spySets = net._layerSets ? net._layerSets.map((layerSet: ILayer[]) => layerSet.map((l) => jest.spyOn(l, 'predict')) ) : []; net.runInputs([Float32Array.from([0, 1])]); if (!net._model) fail(); expect(net._model.length).toEqual(3); for (let i = 0; i < spySets.length; i++) { for (let j = 0; j < spySets[i].length; j++) { expect(spySets[i][j]).toHaveBeenCalled(); } } }); }); describe('.calculateDeltas()', () => { test('back propagates values through deltas', () => { const net = new Recurrent({ inputLayer: () => input({ height: 1 }), hiddenLayers: [ (inputLayer: ILayer, recurrentInput: IRecurrentInput) => { if (recurrentInput.setDimensions) { recurrentInput.setDimensions(1, 3); } return add( multiply(random({ height: 3 }), inputLayer), recurrentInput ); }, ], outputLayer: (inputLayer: ILayer) => output({ height: 1 }, inputLayer), }); net.initialize(); net.initializeDeep(); net.runInputs([Float32Array.from([1, 1])]); if (!net._model) fail(); if (!net._layerSets) fail(); expect(net._model.length).toEqual(3); expect(net._layerSets.length).toEqual(2); for (let i = 0; i < net._layerSets.length; i++) { for (let j = 0; j < net._layerSets[i].length; j++) { expect( (net._layerSets[i][j].deltas as number[][]).every((row) => row.every((delta) => delta === 0) ) ).toBeTruthy(); } } const spySets = net._layerSets.map((layerSet: ILayer[]) => layerSet.map((l) => jest.spyOn(l, 'compare')) ); net._calculateDeltas([ [1, 1], [1, 1], ]); // The last layer propagates delta from target, the last layer propagates zero // TODO: fix // for (let i = 0; i < net._layerSets[0].length; i++) { // expect( // net._layerSets[0][i].deltas.every((row: number[]) => // row.every((delta) => delta !== 0) // ) // ).toBeTruthy(); // } for (let i = 0; i < spySets.length; i++) { for (let j = 0; j < spySets[i].length; j++) { // reuse the last matrix if (i === spySets.length - 1 && (j = spySets[i].length - 1)) { expect(spySets[i][j]).not.toHaveBeenCalled(); } else { expect(spySets[i][j]).toHaveBeenCalled(); } } } }); }); describe('.adjustWeights()', () => { test('back propagates values through weights', () => { const net = new Recurrent({ inputLayer: () => input({ height: 1 }), hiddenLayers: [ (inputLayer: ILayer, recurrentInput: IRecurrentInput) => { if (recurrentInput.setDimensions) { recurrentInput.setDimensions(1, 3); } return add( multiply(random({ height: 3 }), inputLayer), recurrentInput ); }, ], outputLayer: (inputLayer: ILayer) => output({ height: 1 }, inputLayer), }); net.initialize(); net.initializeDeep(); net.runInputs([Float32Array.from([1, 1])]); if (!net._model) fail(); if (!net._layerSets) fail(); expect(net._model.length).toEqual(3); expect(net._layerSets[0].length).toEqual(10); const weightSets = net._model.map((l: ILayer) => copy2D(l.weights as number[][]) ); const spys = net._model.map((l: ILayer) => jest.spyOn(l, 'learn')); net._calculateDeltas([[1, 1]]); net.adjustWeights(); for (let i = 0; i < spys.length; i++) { expect(spys[i]).toHaveBeenCalled(); } // weights are adjusted in model only via .learn() for (let i = 0; i < weightSets.length; i++) { const weights = weightSets[i]; for (let row = 0; row < weights.length; row++) { for (let col = 0; col < weights[row].length; col++) { expect(weights[row][col]).not.toEqual( (net._model[i].weights as number[][])[row][col] ); } } } }); }); describe('._trainPattern()', () => { test('steps back through values correctly', () => { class SuperLayer extends Filter { inputLayer: ILayer; errors?: number[][]; constructor(inputLayer: ILayer) { super({}, inputLayer); this.inputLayer = inputLayer; this.settings.width = 1; this.settings.height = 1; // this.width = 1; // this.height = 1; this.errors = [[0]]; } setupKernels() {} reuseKernels() {} predict(inputs: KernelOutput) { this.weights = inputs; } compare() { this.errors = [[5]]; } learn() {} } const net = new Recurrent({ inputLayer: () => input({ width: 1 }), hiddenLayers: [(inputLayer: ILayer) => new SuperLayer(inputLayer)], outputLayer: (inputLayer: ILayer) => new SuperLayer(inputLayer), }); net.initialize(); net.initializeDeep(); const runInputsSpy = jest.spyOn(net, 'runInputs'); const calculateDeltasSpy = jest.spyOn(net, '_calculateDeltas'); const adjustWeightsSpy = jest.spyOn(net, 'adjustWeights'); const inputValue = [ [0, 1], [1, 1], ]; const errorRate = net._trainPattern(inputValue, true); expect(errorRate).toEqual(Float32Array.from([2.5])); expect(runInputsSpy).toHaveBeenCalledWith(inputValue); expect(calculateDeltasSpy).toHaveBeenCalledWith(inputValue); expect(adjustWeightsSpy).toHaveBeenCalled(); }); describe('when called more than once', () => { test('continuously updates output layer', () => { const net = new Recurrent({ inputLayer: () => input({ height: 1 }), hiddenLayers: [ (inputLayer: ILayer, recurrentInput: IRecurrentInput) => rnnCell({ height: 3 }, inputLayer, recurrentInput), ], outputLayer: (inputLayer: ILayer) => output({ height: 1 }, inputLayer), }); net.initialize(); net.initializeDeep(); const lastLayerSet = net._layerSets ? net._layerSets[net._layerSets.length - 1] : []; const lastOutputLayer = lastLayerSet[lastLayerSet.length - 1]; expect(lastOutputLayer.weights).toEqual([new Float32Array([0])]); net._trainPattern([[1, 2]], false); const weights1 = lastOutputLayer.weights; expect(weights1).not.toEqual([[0]]); net._trainPattern([[3, 2]], false); const weights2 = lastOutputLayer.weights; expect(weights1).not.toEqual(weights2); net._trainPattern([[1, 1]], false); const weights3 = lastOutputLayer.weights; expect(weights2).not.toEqual(weights3); net._trainPattern([[3, 3]], false); const weights4 = lastOutputLayer.weights; expect(weights3).not.toEqual(weights4); }); }); }); describe('.toJSON', () => { it('serializes and deserializes correctly', () => { const net = new Recurrent({ inputLayer: () => input({ height: 1 }), hiddenLayers: [ (inputLayer: ILayer, recurrentInput: IRecurrentInput) => rnnCell({ height: 3 }, inputLayer, recurrentInput), ], outputLayer: (inputLayer: ILayer) => output({ height: 1 }, inputLayer), }); net.initialize(); const layers = net.layers as ILayer[]; expect(net.toJSON()).toEqual({ inputLayerIndex: 0, layers: [ layers[0].toJSON(), layers[1].toJSON(), { ...layers[2].toJSON(), inputLayer1Index: 1, inputLayer2Index: 0 }, layers[3].toJSON(), layers[4].toJSON(), { ...layers[5].toJSON(), inputLayer1Index: 3, inputLayer2Index: 4 }, { ...layers[6].toJSON(), inputLayer1Index: 2, inputLayer2Index: 5 }, layers[7].toJSON(), { ...layers[8].toJSON(), inputLayer1Index: 6, inputLayer2Index: 7 }, { ...layers[9].toJSON(), inputLayerIndex: 8 }, layers[10].toJSON(), { ...layers[11].toJSON(), inputLayer1Index: 10, inputLayer2Index: 9 }, layers[12].toJSON(), { ...layers[13].toJSON(), inputLayer1Index: 11, inputLayer2Index: 12, }, { ...layers[14].toJSON(), inputLayerIndex: 13 }, ], outputLayerIndex: 14, sizes: [1, 3, 1], type: 'Recurrent', }); }); }); describe('.fromJSON', () => { it('serializes and deserializes correctly', () => { const net = new Recurrent({ inputLayer: () => input({ height: 1 }), hiddenLayers: [ (inputLayer: ILayer, recurrentInput: IRecurrentInput) => rnnCell({ height: 3 }, inputLayer, recurrentInput), ], outputLayer: (inputLayer: ILayer) => output({ height: 1 }, inputLayer), }); net.initialize(); const json = JSON.parse(JSON.stringify(net.toJSON())); const newNet = Recurrent.fromJSON(json); const newNetJson = newNet.toJSON(); expect(newNetJson).toEqual(json); }); }); }); ================================================ FILE: src/test-utils.ts ================================================ import assert from 'assert'; import { IGPUTextureSettings, Kernel, Texture } from 'gpu.js'; import { ILayerTemplate, IPraxis, IPraxisSettings } from '../src/praxis/base-praxis'; import { BaseLayer, ILayerSettings, ILayer } from '../src/layer/base-layer'; export const xorTrainingData = [ { input: [0, 1], output: [1] }, { input: [0, 0], output: [0] }, { input: [1, 1], output: [0] }, { input: [1, 0], output: [1] }, ]; export function onePlusPlus3D(width: number, height: number, depth: number): number[][][] { const grid = []; let i = 1; for (let z = 0; z < depth; z++) { const rows = []; for (let y = 0; y < height; y++) { const columns = []; for (let x = 0; x < width; x++) { columns.push(i++); } rows.push(columns); } grid.push(rows); } return grid; } export function onePlusPlus2D(width: number, height: number): number[][] { const rows = []; let i = 1; for (let y = 0; y < height; y++) { const columns = []; for (let x = 0; x < width; x++) { columns.push(i++); } rows.push(columns); } return rows; } export function zero3D(width: number, height: number, depth: number): number[][][] { const grid = []; for (let z = 0; z < depth; z++) { const rows = []; for (let y = 0; y < height; y++) { const columns = []; for (let x = 0; x < width; x++) { columns.push(0); } rows.push(columns); } grid.push(rows); } return grid; } export function zero2D(width: number, height: number): number[][] { const rows = []; for (let y = 0; y < height; y++) { const columns = []; for (let x = 0; x < width; x++) { columns.push(0); } rows.push(columns); } return rows; } // export function allWeights(model, fn) { // fn(model.input.weights); // model.hiddenLayers.forEach((layer) => { // for (const p in layer) { // if (!layer.hasOwnProperty(p)) continue; // assert(fn(layer[p].weights)); // } // }); // fn(model.output.weights); // // model.equations.forEach((equation) => { // equation.states.forEach((state) => { // if (state.left && state.left.weights) fn(state.left.weights); // if (state.right && state.right.weights) fn(state.right.weights); // if (state.product && state.product.weights) fn(state.product.weights); // }); // }); // } export function allDeltas(model: any, fn: any): void { fn(model.input.deltas); model.hiddenLayers.forEach((layer: any) => { for (const p in layer) { if (!layer.hasOwnProperty(p)) continue; assert(fn(layer[p].deltas)); } }); fn(model.output.deltas); model.equations.forEach((equation: any) => { equation.states.forEach((state: any) => { if (state.left && state.left.deltas) fn(state.left.deltas); if (state.right && state.right.deltas) fn(state.right.deltas); if (state.product && state.product.deltas) fn(state.product.deltas); }); }); } export function allMatrices(model: any, fn: any): void { fn(model.input.weights); model.hiddenLayers.forEach((layer: any) => { for (const p in layer) { if (!layer.hasOwnProperty(p)) continue; fn(layer[p].weights); } }); fn(model.output.weights); model.equations.forEach((equation: any) => { equation.states.forEach((state: any) => { if (state.left && state.left.weights) fn(state.left.weights); if (state.right && state.right.weights) fn(state.right.weights); if (state.product && state.product.weights) fn(state.product.weights); }); }); fn(model.input.deltas); model.hiddenLayers.forEach((layer: any) => { for (const p in layer) { if (!layer.hasOwnProperty(p)) continue; fn(layer[p].deltas); } }); fn(model.output.deltas); model.equations.forEach((equation: any) => { equation.states.forEach((state: any) => { if (state.left && state.left.deltas) fn(state.left.deltas); if (state.right && state.right.deltas) fn(state.right.deltas); if (state.product && state.product.deltas) fn(state.product.deltas); }); }); } export function shave(value: Float32Array): Float32Array { const resultRow = new Float32Array(value.length); for (let x = 0; x < value.length; x++) { resultRow[x] = parseFloat((value[x]).toFixed(8)); } return resultRow; } export function shave2D(value: Float32Array[]): Float32Array[] { const resultMatrix = new Array(value.length); for (let y = 0; y < value.length; y++) { resultMatrix[y] = shave(value[y]); } return resultMatrix; } export function shave3D(value: Float32Array[][]): Float32Array[][] { const resultCube = new Array(value.length); for (let z = 0; z < value.length; z++) { resultCube[z] = shave2D(value[z]); } return resultCube; } // it was found that coverage breaks when you compare leftFunction.toString() === rightString.toString() // this does a check on the first line of the function source, which is good enough for knowing the function signature export function expectFunction(source: string, fn: Function): void { expect(source.toString().split(/\n/g)[0]).toBe(fn.toString().split(/\n/g)[0]); } export class TestLayer extends BaseLayer { get width(): number { return this.settings.width as number; } get height(): number { return this.settings.height as number; } get depth(): number { return this.settings.depth as number; } constructor(settings: ILayerSettings) { super(settings); } } export function mockLayer(settings: ILayerSettings = {}): ILayer { return new TestLayer({ id: 'MockLayer', ...settings }); } export function mockTexture(settings?: Partial): Texture { return new Texture({ ...settings, texture: {} as any, size: [1, 1], dimensions: [1, 1], output: [1, 1], context: {} as any, kernel: {} as Kernel, }); } export function mockPraxis(layerTemplate: ILayerTemplate, praxisSettings: Partial = {}): IPraxis { return { layerTemplate, settings: praxisSettings, kernel: null, get width() { return layerTemplate.width; }, get height() { return layerTemplate.height; }, get depth() { return layerTemplate.depth; }, run: () => {}, setupKernels: () => {}, toJSON: () => { return praxisSettings; }, }; } export interface IWithCompareKernel { compareKernel: jest.Mock; } export interface IWithPredictKernel { predictKernel: jest.Mock; } export interface IWithPredictKernelMap { predictKernelMap: jest.Mock; } ================================================ FILE: src/utilities/array-lookup-table.ts ================================================ export class ArrayLookupTable { length = 0; table: { [key: string]: number } = {}; constructor( data: Array<{ input: Array>; output: Array>; }>, public prop: 'input' | 'output' ) { for (let i = 0; i < data.length; i++) { const datum = data[i]; const ioValue = datum[prop]; for (let j = 0; j < ioValue.length; j++) { const value = ioValue[j]; for (const p in value) { if (!value.hasOwnProperty(p)) continue; if (this.table.hasOwnProperty(p)) continue; this.table[p] = this.length++; } } } } } ================================================ FILE: src/utilities/cast.test.ts ================================================ import * as cast from './cast'; import { LookupTable } from './lookup-table'; describe('cast', () => { describe('arraysToFloat32Arrays()', () => { it('converts regular nested array to nested Float32Arrays', () => { expect( cast.arraysToFloat32Arrays([ [1, 2, 3], [4, 5, 6], ]) ).toEqual([Float32Array.from([1, 2, 3]), Float32Array.from([4, 5, 6])]); }); }); describe('arrayToFloat32Arrays()', () => { it('converts regular array to Float32Arrays', () => { expect(cast.arrayToFloat32Arrays([1, 2, 3])).toEqual([ Float32Array.from([1]), Float32Array.from([2]), Float32Array.from([3]), ]); }); }); describe('arrayToFloat32Array()', () => { it('converts regular array to Float32Array', () => { expect(cast.arrayToFloat32Array([1, 2, 3])).toEqual( Float32Array.from([1, 2, 3]) ); }); }); describe('objectsToFloat32Arrays()', () => { it('converts array of objects to Float32Arrays', () => { const value = [{ one: 1, two: 2, three: 3 }]; const table = new LookupTable(value).table; expect(cast.objectsToFloat32Arrays(value, table, 3)).toEqual([ Float32Array.from([1, 2, 3]), ]); }); }); describe('objectToFloat32Arrays()', () => { it('converts an object to Float32Arrays', () => { const value = [{ one: 1, two: 2, three: 3 }]; expect(cast.objectToFloat32Arrays(value[0])).toEqual([ Float32Array.from([1]), Float32Array.from([2]), Float32Array.from([3]), ]); }); }); describe('objectToFloat32Array()', () => { it('converts object to Float32Array', () => { const value = [{ one: 1, two: 2, three: 3 }]; const table = new LookupTable(value).table; expect(cast.objectToFloat32Array(value[0], table, 3)).toEqual( Float32Array.from([1, 2, 3]) ); }); }); }); ================================================ FILE: src/utilities/cast.ts ================================================ export function arraysToFloat32Arrays(arrays: number[][]): Float32Array[] { const result: Float32Array[] = []; for (let i = 0; i < arrays.length; i++) { result.push(Float32Array.from(arrays[i])); } return result; } export function inputOutputArraysToFloat32Arrays( input: number[][], output: number[][] ): Float32Array[] { const result: Float32Array[] = []; for (let i = 0; i < input.length; i++) { result.push(Float32Array.from(input[i])); } for (let i = 0; i < output.length; i++) { result.push(Float32Array.from(output[i])); } return result; } export function arrayToFloat32Arrays(array: number[]): Float32Array[] { const result: Float32Array[] = []; for (let i = 0; i < array.length; i++) { result.push(Float32Array.from([array[i]])); } return result; } export function inputOutputArrayToFloat32Arrays( input: number[], output: number[] ): Float32Array[] { const result: Float32Array[] = []; for (let i = 0; i < input.length; i++) { result.push(Float32Array.from([input[i]])); } for (let i = 0; i < output.length; i++) { result.push(Float32Array.from([output[i]])); } return result; } export function arrayToFloat32Array(array: number[]): Float32Array { return Float32Array.from(array); } export function objectsToFloat32Arrays( objects: Array>, table: Record, length: number ): Float32Array[] { const results: Float32Array[] = []; for (let i = 0; i < objects.length; i++) { const object = objects[i]; const result = new Float32Array(length); for (const p in object) { if (object.hasOwnProperty(p)) { result[table[p]] = object[p]; } } results.push(result); } return results; } export function inputOutputObjectsToFloat32Arrays( input: Array>, output: Array>, inputTable: Record, outputTable: Record, inputLength: number, outputLength: number ): Float32Array[] { const results: Float32Array[] = []; for (let i = 0; i < input.length; i++) { const object = input[i]; const result = new Float32Array(inputLength); for (const p in object) { if (object.hasOwnProperty(p)) { result[inputTable[p]] = object[p]; } } results.push(result); } for (let i = 0; i < output.length; i++) { const object = output[i]; const result = new Float32Array(outputLength); for (const p in object) { if (object.hasOwnProperty(p)) { result[outputTable[p]] = object[p]; } } results.push(result); } return results; } export function objectToFloat32Arrays( object: Record ): Float32Array[] { const result: Float32Array[] = []; for (const p in object) { if (!object.hasOwnProperty(p)) continue; result.push(Float32Array.from([object[p]])); } return result; } export function inputOutputObjectToFloat32Arrays( input: Record, output: Record ): Float32Array[] { const result: Float32Array[] = []; for (const p in input) { if (!input.hasOwnProperty(p)) continue; result.push(Float32Array.from([input[p]])); } for (const p in output) { if (!output.hasOwnProperty(p)) continue; result.push(Float32Array.from([output[p]])); } return result; } export function objectToFloat32Array( object: Record, table: Record, length: number ): Float32Array { const result = new Float32Array(length); for (const p in object) { if (object.hasOwnProperty(p)) { result[table[p]] = object[p]; } } return result; } ================================================ FILE: src/utilities/data-formatter.test.ts ================================================ import { DataFormatter } from './data-formatter'; describe('DataFormatter', () => { describe('.fromJSON()', () => { const df1 = new DataFormatter('a'.split('')); const json = df1.toJSON(); const df2 = DataFormatter.fromJSON(json); it('sets .indexTable from JSON and matches ', () => { expect(df2.indexTable).toEqual({ a: 0, unrecognized: 1 }); expect(df2.indexTable).toEqual(df1.indexTable); }); it('sets .characterTable from JSON', () => { expect(df2.characterTable).toEqual({ '0': 'a', '1': null }); expect(df2.characterTable).toEqual(df1.characterTable); }); it('sets .characters from JSON', () => { expect(df2.characters).toEqual(['a', 'unrecognized']); expect(df2.characters).toEqual(df1.characters); }); it('sets .specialIndexes from JSON', () => { expect(json.specialIndexes).toEqual([1]); expect(df1.specialIndexes).toEqual(df1.specialIndexes); }); it('sets .isSetup from JSON', () => { expect(df2.isSetup).toEqual(true); expect(df2.isSetup).toEqual(df1.isSetup); }); }); test('does not have zeros', () => { const dataFormatter = new DataFormatter( 'abcdefghijklmnopqrstuvwxyz'.split('') ); const indexes = dataFormatter.toIndexes( 'abcdefghijklmnopqrstuvwxyz'.split('') ); expect(indexes[0]).toBe(0); expect(indexes[1]).toBe(1); expect(indexes[2]).toBe(2); expect(indexes[3]).toBe(3); expect(indexes[4]).toBe(4); expect(indexes[5]).toBe(5); expect(indexes[6]).toBe(6); expect(indexes[7]).toBe(7); expect(indexes[8]).toBe(8); expect(indexes[9]).toBe(9); expect(indexes[10]).toBe(10); expect(indexes[11]).toBe(11); expect(indexes[12]).toBe(12); expect(indexes[13]).toBe(13); expect(indexes[14]).toBe(14); expect(indexes[15]).toBe(15); expect(indexes[16]).toBe(16); expect(indexes[17]).toBe(17); expect(indexes[18]).toBe(18); expect(indexes[19]).toBe(19); expect(indexes[20]).toBe(20); expect(indexes[21]).toBe(21); expect(indexes[22]).toBe(22); expect(indexes[23]).toBe(23); expect(indexes[24]).toBe(24); expect(indexes[25]).toBe(25); }); test('should properly be able to reference indices of cat', () => { const dataFormatter = new DataFormatter([['cat']]); const asIndexes = [0, 1, 2]; dataFormatter.toIndexes(['cat']).forEach((v, i) => { expect(v).toBe(asIndexes[i]); }); }); test('should properly be able to reference indices of math', () => { const dataFormatter = new DataFormatter([ '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '=', '+', ]); const asIndexes = [0, 11, 8, 10, 8]; dataFormatter.toIndexes('0+8=8'.split('')).forEach((v, i) => { expect(v).toBe(asIndexes[i]); }); }); test('does not have zeros', () => { const dataFormatter = new DataFormatter( 'abcdefghijklmnopqrstuvwxyz'.split('') ); const characters = dataFormatter.toCharacters([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, ]); expect(characters[0]).toBe('a'); expect(characters[1]).toBe('b'); expect(characters[2]).toBe('c'); expect(characters[3]).toBe('d'); expect(characters[4]).toBe('e'); expect(characters[5]).toBe('f'); expect(characters[6]).toBe('g'); expect(characters[7]).toBe('h'); expect(characters[8]).toBe('i'); expect(characters[9]).toBe('j'); expect(characters[10]).toBe('k'); expect(characters[11]).toBe('l'); expect(characters[12]).toBe('m'); expect(characters[13]).toBe('n'); expect(characters[14]).toBe('o'); expect(characters[15]).toBe('p'); expect(characters[16]).toBe('q'); expect(characters[17]).toBe('r'); expect(characters[18]).toBe('s'); expect(characters[19]).toBe('t'); expect(characters[20]).toBe('u'); expect(characters[21]).toBe('v'); expect(characters[22]).toBe('w'); expect(characters[23]).toBe('x'); expect(characters[24]).toBe('y'); expect(characters[25]).toBe('z'); }); test('should properly be able to reference characters of cat', () => { const dataFormatter = new DataFormatter([['cat']]); const asIndexes = [0]; const asCharacters = 'cat'; expect(dataFormatter.toCharacters(asIndexes)).toEqual([asCharacters]); }); test('can handle strings', () => { const dataFormatter = new DataFormatter('a big string'.split('')); const indices = dataFormatter.toIndexes('a big string'); indices.forEach((value) => expect(value >= 0)); expect(dataFormatter.toCharacters(indices).join('')).toBe('a big string'); }); test('can handle array of strings', () => { const dataFormatter = new DataFormatter('a big string'.split('')); const indices = dataFormatter.toIndexes('a big string'.split('')); indices.forEach((value) => expect(value >= 0)); expect(dataFormatter.toCharacters(indices)).toEqual( 'a big string'.split('') ); }); test('can handle array of array of strings', () => { const dataFormatter = new DataFormatter([ 'a big string'.split(''), 'batman was here'.split(''), ]); let indices = dataFormatter.toIndexes('a big string'.split('')); indices.forEach((value) => expect(value >= 0)); expect(dataFormatter.toCharacters(indices)).toEqual( 'a big string'.split('') ); indices = dataFormatter.toIndexes('batman was here'.split('')); indices.forEach((value) => expect(value >= 0)); expect(dataFormatter.toCharacters(indices)).toEqual( 'batman was here'.split('') ); }); test('can handle array of numbers', () => { const dataFormatter = new DataFormatter([1, 2, 3]); const indices = dataFormatter.toIndexes(['1', '2', '3']); indices.forEach((value) => expect(Number(value) >= 0)); expect(dataFormatter.toCharacters(indices)).toEqual(['1', '2', '3']); }); test('can handle array of array of numbers', () => { const dataFormatter = new DataFormatter([ ['1', '2', '3'], ['4', '5', '6'], ]); let indices = dataFormatter.toIndexes(['1', '2', '3']); indices.forEach((value) => expect(Number(value) >= 0)); expect(dataFormatter.toCharacters(indices)).toEqual(['1', '2', '3']); indices = dataFormatter.toIndexes(['4', '5', '6']); indices.forEach((value) => expect(Number(value) >= 3)); expect(dataFormatter.toCharacters(indices)).toEqual(['4', '5', '6']); }); test('can handle array of booleans', () => { const dataFormatter = new DataFormatter([['true', 'false']]); const indices = dataFormatter.toIndexes(['true', 'false', 'true', 'false']); indices.forEach((value) => expect(value >= 0)); expect(dataFormatter.toCharacters(indices)).toEqual([ 'true', 'false', 'true', 'false', ]); }); test('can handle array of array of booleans', () => { const dataFormatter = new DataFormatter([['true'], ['false']]); const indices = dataFormatter.toIndexes(['true', 'false']); indices.forEach((value) => expect(value >= 0)); expect(dataFormatter.toCharacters(indices)).toEqual(['true', 'false']); }); test('when splitting values to input/output', () => { const dataFormatter = DataFormatter.fromArrayInputOutput([ { input: [1, 2, 3, 4, 5, 6, 7, 8, 9, 0], output: [0], }, ]); const indices = dataFormatter.toIndexesInputOutput( ['1', '2', '3', '4', '5'], ['1', '2', '3', '4', '5'] ); expect(dataFormatter.toCharacters(indices)).toEqual([ '1', '2', '3', '4', '5', '1', '2', '3', '4', '5', ]); }); describe('.format()', () => { test('handles data.input & data.output of string', () => { const dataFormatter = new DataFormatter(); const formatDataInSpy = jest.spyOn(dataFormatter, 'formatDataIn'); dataFormatter.format([{ input: '1', output: '2' }]); expect(formatDataInSpy).toBeCalledWith('1', '2'); }); test('handles data.input & data.output of number', () => { const dataFormatter = new DataFormatter(); const formatDataInSpy = jest.spyOn(dataFormatter, 'formatDataIn'); dataFormatter.format([{ input: 1, output: 2 }]); expect(formatDataInSpy).toBeCalledWith('1', '2'); }); test('handles data.input & data.output of string[]', () => { const dataFormatter = new DataFormatter(); const formatDataInSpy = jest.spyOn(dataFormatter, 'formatDataIn'); dataFormatter.format([{ input: ['1', '2'], output: ['3', '4'] }]); expect(formatDataInSpy).toBeCalledWith(['1', '2'], ['3', '4']); }); test('handles data.input & data.output of number[]', () => { const dataFormatter = new DataFormatter(); const formatDataInSpy = jest.spyOn(dataFormatter, 'formatDataIn'); dataFormatter.format([{ input: [1, 2], output: [3, 4] }]); expect(formatDataInSpy).toBeCalledWith(['1', '2'], ['3', '4']); }); describe('when working with array of strings', () => { it('creates an appropriate DataFormatter', () => { const dataFormatter = new DataFormatter(); const result = dataFormatter.format(['foo', 'bar', 'baz']); expect(result).toEqual([ [0, 1, 1], [2, 3, 4], [2, 3, 5], ]); expect(dataFormatter.characters).toEqual([ 'f', 'o', 'b', 'a', 'r', 'z', 'unrecognized', ]); }); }); describe('when working with array of input & output strings', () => { it('creates an appropriate DataFormatter', () => { const dataFormatter = new DataFormatter(); const result = dataFormatter.format([ { input: 'foo', output: 'bar' }, { input: 'bar', output: 'baz' }, { input: 'baz', output: 'foo' }, ]); expect(result).toEqual([ [0, 1, 1, 6, 7, 2, 3, 4], [2, 3, 4, 6, 7, 2, 3, 5], [2, 3, 5, 6, 7, 0, 1, 1], ]); expect(dataFormatter.characters).toEqual([ 'f', 'o', 'b', 'a', 'r', 'z', 'stop-input', 'start-output', 'unrecognized', ]); }); }); describe('when working with array of tokens', () => { it('creates an appropraite DataFormatter', () => { const dataFormatter = new DataFormatter(); const result = dataFormatter.format([ ['foo', 'bar', 'baz'], ['bar', 'baz', 'foo'], ['baz', 'foo', 'bar'], ]); expect(result).toEqual([ [0, 1, 2], [1, 2, 0], [2, 0, 1], ]); expect(dataFormatter.characters).toEqual([ 'foo', 'bar', 'baz', 'unrecognized', ]); }); }); describe('when working with array of input & output tokens', () => { it('creates an appropriate DataFormatter', () => { const dataFormatter = new DataFormatter(); const result = dataFormatter.format([ { input: ['foo', 'bar'], output: ['baz'] }, { input: ['bar', 'baz'], output: ['foo'] }, { input: ['baz', 'foo'], output: ['bar'] }, ]); expect(result).toEqual([ [0, 1, 3, 4, 2], [1, 2, 3, 4, 0], [2, 0, 3, 4, 1], ]); expect(dataFormatter.characters).toEqual([ 'foo', 'bar', 'baz', 'stop-input', 'start-output', 'unrecognized', ]); }); }); }); }); ================================================ FILE: src/utilities/data-formatter.ts ================================================ import { Value, IRNNDatum } from '../recurrent/rnn-data-types'; export interface IDataFormatter { indexTable: { [value: string]: number }; toIndexesInputOutput: (input: Value, output?: Value) => number[]; toIndexes: (input: string) => number[]; toCharacters: (output: number[]) => string[]; characters: Array; specialIndexes: number[]; toFunctionString: () => string; formatDataIn: (input?: Value, output?: Value) => number[]; formatDataOut: (input: number[], output: number[]) => string; format: (data: Array) => number[][]; isSetup: boolean; toJSON: () => IDataFormatterJSON; } export class DataFormatter implements IDataFormatter { indexTable: { [key: string]: number; [key: number]: number } = {}; characterTable: { [key: number]: string | number | null } = {}; characters: Array = []; specialIndexes: number[] = []; isSetup = false; constructor(private values?: Array, maxThreshold = 0) { if (values === undefined) return; this.setup(values, maxThreshold); } setup(values: Array, maxThreshold = 0): void { if (this.isSetup) throw new Error('DataFormatter is already setup'); this.values = values; // go over all characters and keep track of all unique ones seen // count up all characters this.buildCharactersFromIterable(values); this.buildTables(maxThreshold); if ((values[0] as IRNNDatum).input) { this.addInputOutput(); } this.addUnrecognized(); this.isSetup = true; } buildCharactersFromIterable(values: Array): void { const tempCharactersTable: { [character: string]: boolean } = {}; for ( let dataFormatterIndex = 0, dataFormatterLength = values.length; dataFormatterIndex < dataFormatterLength; dataFormatterIndex++ ) { const characters = values[dataFormatterIndex]; // if (typeof characters === 'string') { // const character = characters; // if (tempCharactersTable.hasOwnProperty(character)) continue; // tempCharactersTable[character] = true; // this.characters.push(character); if (characters.hasOwnProperty('length')) { const iteratable = characters as string[] | string; for ( let characterIndex = 0, charactersLength = iteratable.length; characterIndex < charactersLength; characterIndex++ ) { const character = iteratable[characterIndex]; if (tempCharactersTable.hasOwnProperty(character)) continue; tempCharactersTable[character] = true; this.characters.push(character); } } else if (typeof characters === 'number') { if (tempCharactersTable.hasOwnProperty(characters)) continue; tempCharactersTable[characters] = true; this.characters.push(characters); } else if (typeof characters === 'boolean') { const character = characters.toString(); if (tempCharactersTable.hasOwnProperty(character)) continue; tempCharactersTable[character] = true; this.characters.push(character); } else if ( Array.isArray(characters) && typeof characters[0] === 'string' ) { for (let i = 0; i < characters.length; i++) { const character = characters[i] as string; if (tempCharactersTable.hasOwnProperty(character)) continue; tempCharactersTable[character] = true; this.characters.push(character); } } else if ( Array.isArray(characters) && (typeof characters[0] === 'number' || typeof characters[0] === 'boolean') ) { for (let i = 0; i < characters.length; i++) { const character = characters[i].toString(); if (tempCharactersTable.hasOwnProperty(dataFormatterIndex)) continue; tempCharactersTable[character] = true; this.characters.push(character); } } else if ( characters.hasOwnProperty('input') && characters.hasOwnProperty('output') ) { const { input, output } = (characters as unknown) as IRNNDatum; if (Array.isArray(input)) { this.addCharacters(input, tempCharactersTable); } else { this.addCharacters(input.toString(), tempCharactersTable); } if (Array.isArray(output)) { this.addCharacters(output, tempCharactersTable); } else { this.addCharacters(output.toString(), tempCharactersTable); } } else { throw new Error('Unhandled value'); } } } addCharacters( characters: string | string[] | boolean[] | number[], charactersTable: { [character: string]: boolean } ): void { for (let i = 0; i < characters.length; i++) { const character = characters[i].toString(); if (charactersTable.hasOwnProperty(character)) continue; charactersTable[character] = true; this.characters.push(character); } } buildTables(maxThreshold: number): void { // filter by count threshold and create pointers const charactersLength = this.characters.length; for ( let characterIndex = 0; characterIndex < charactersLength; characterIndex++ ) { const character = this.characters[characterIndex]; if (characterIndex >= maxThreshold) { // add character to dataFormatter this.indexTable[character] = characterIndex; this.characterTable[characterIndex] = character; } } } toIndexes(value: Value, maxThreshold = 0): number[] { const result = []; const { indexTable } = this; switch (typeof value) { case 'number': case 'boolean': value = value.toString(); } for (let i = 0, max = value.length; i < max; i++) { const character = value[i].toString(); let index = indexTable[character]; if (index === undefined) { if (indexTable.unrecognized) { index = indexTable.unrecognized; } else { throw new Error(`unrecognized character "${character}"`); } } if (index < maxThreshold) continue; result.push(index); } return result; } toIndexesInputOutput( input: Value, output?: Value, maxThreshold = 0 ): number[] { const result: number[] = this.toIndexesValue(input, maxThreshold, true); if (typeof output === 'undefined') return result; return result.concat(this.toIndexesValue(output, maxThreshold, false)); } toIndexesValue( value: Value, maxThreshold: number, isInput: boolean ): number[] { if (typeof value === 'string') { value = value.split(''); } else if (typeof value === 'number' || typeof value === 'boolean') { value = value.toString().split(''); } else if ( Array.isArray(value) && (typeof (value as number[])[0] === 'number' || typeof (value as boolean[])[0] === 'boolean' || typeof (value as string[])[0] === 'string') ) { value = (value as string[]).map((v) => v.toString()); } else { throw new Error('unrecognized value'); } if (isInput) { value = value.concat(['stop-input', 'start-output']); } return this.toIndexes(value, maxThreshold); } toCharacters(indices: number[], maxThreshold = 0): string[] { const result: string[] = []; const { indexTable, characterTable } = this; for (let i = 0, max = indices.length; i < max; i++) { const index = indices[i]; if (index < maxThreshold) continue; let character = characterTable[index]; if (character === undefined) { if (indexTable.unrecognized) { character = characterTable[indexTable.unrecognized]; } else { throw new Error(`unrecognized index "${index}"`); } } else if (character !== null) { result.push(character.toString()); } } return result; } toString(indices: number[], maxThreshold: number): string { return this.toCharacters(indices, maxThreshold).join(''); } addInputOutput(): void { this.addSpecial('stop-input'); this.addSpecial('start-output'); } addUnrecognized(): void { this.addSpecial('unrecognized'); } static fromAllPrintable( maxThreshold: number, values = ['\n'] ): DataFormatter { for (let i = 32; i <= 126; i++) { values.push(String.fromCharCode(i)); } return new DataFormatter(values, maxThreshold); } static fromAllPrintableInputOutput( maxThreshold: number, values = ['\n'] ): DataFormatter { const dataFormatter = DataFormatter.fromAllPrintable(maxThreshold, values); dataFormatter.addInputOutput(); dataFormatter.addUnrecognized(); return dataFormatter; } static fromStringInputOutput( string: string, maxThreshold: number ): DataFormatter { const values = Array.from(new Set(string)).join(''); const dataFormatter = new DataFormatter(values.split(''), maxThreshold); dataFormatter.addInputOutput(); dataFormatter.addUnrecognized(); dataFormatter.isSetup = true; return dataFormatter; } static fromArrayInputOutput( data: IRNNDatum[], maxThreshold?: number ): DataFormatter { const values: Array = []; for (let i = 0; i < data.length; i++) { const datum = data[i]; values.push(validateAndCast(datum.input), validateAndCast(datum.output)); } const flatArray: string[] = Array.isArray(values) ? (values as string[][]).flat() : values; const dataFormatter = new DataFormatter( Array.from(new Set(flatArray)), maxThreshold ); dataFormatter.addInputOutput(); dataFormatter.addUnrecognized(); dataFormatter.isSetup = true; return dataFormatter; } static fromString(string: string, maxThreshold = 0): DataFormatter { const values = Array.from(new Set(string)).join(''); return new DataFormatter(values.split(''), maxThreshold); } toJSON(): IDataFormatterJSON { return { indexTable: this.indexTable, characterTable: this.characterTable, values: this.values as Value[], characters: this.characters, specialIndexes: this.specialIndexes, }; } /** TODO: Type better, The type of json is not "string that is a valid JSON", it is a POJO in the shape of DataFormatter. * this method re-hydrates the the data as an instance of DataFormatter. */ static fromJSON(json: IDataFormatterJSON): DataFormatter { const dataFormatter = new DataFormatter(); dataFormatter.indexTable = json.indexTable; dataFormatter.characterTable = json.characterTable; dataFormatter.values = json.values; dataFormatter.characters = json.characters; dataFormatter.specialIndexes = json.specialIndexes; dataFormatter.isSetup = true; return dataFormatter; } addSpecial(special: string | number, character = null): void { const specialIndex = (this.indexTable[special] = this.characters.length); this.characterTable[specialIndex] = character; this.specialIndexes.push(this.characters.length); this.characters.push(special); } toFunctionString(): string { return ` var characterTable = ${JSON.stringify(this.characterTable)}; var indexTable = ${JSON.stringify(this.indexTable)}; var characters = ${JSON.stringify(this.characters)}; var dataFormatter = { toIndexes: function ${this.toIndexes.toString()}, toIndexesInputOutput: function ${this.toIndexesInputOutput.toString()}, toCharacters: function ${this.toCharacters.toString()}, toIndexesValue: function ${this.toIndexesValue.toString()}, };`; } formatDataIn(input?: Value, output?: Value): number[] { if (input === undefined) return []; if (Array.isArray(input) && typeof input[0] === 'number') { return input as number[]; } if (this.indexTable?.hasOwnProperty('stop-input')) { return this.toIndexesInputOutput(input, output); } return this.toIndexes(input); } formatDataOut(input: number[], output: number[]): string { return this.toCharacters(output).join(''); } format(data: Array): number[][] { if ( typeof data[0] === 'number' && !Array.isArray(data[0]) && (!data[0].hasOwnProperty('input') || !data[0].hasOwnProperty('output')) ) { return data as number[][]; } const result: number[][] = []; if ( typeof data[0] === 'string' || typeof data[0] === 'number' || Array.isArray(data[0]) ) { if (!this.isSetup) { this.setup(data); for (let i = 0; i < data.length; i++) { result.push(this.formatDataIn(validateAndCast(data[i] as Value))); } } else { for (let i = 0, max = data.length; i < max; i++) { result.push(this.formatDataIn(data[i] as Value)); } } } else if ((data[0] as IRNNDatum).input && (data[0] as IRNNDatum).output) { if (!this.isSetup) { this.setup(data); } for (let i = 0, max = data.length; i < max; i++) { result.push( this.formatDataIn( validateAndCast((data[i] as IRNNDatum).input), validateAndCast((data[i] as IRNNDatum).output) ) ); } } else { throw new Error('unrecognized data'); } return result; } } function validateAndCast(value: Value): string | string[] { if (typeof value === 'string') return value; if (typeof value === 'number') return value.toString(); if (typeof value === 'boolean') return value.toString(); if (Array.isArray(value) && typeof value[0] === 'string') return value as string[]; if (typeof value[0] === 'boolean') { return (value as boolean[]).map((v: boolean) => v.toString()); } if (typeof value[0] === 'number') { return (value as number[]).map((v: number) => v.toString()); } throw new Error( 'unrecognized value, expected string[], string, number[], number, boolean[], or boolean' ); } export interface IDataFormatterJSON { indexTable: { [key: string]: number; [key: number]: number }; characterTable: { [key: number]: string | number | null }; values: Value[]; characters: Array; specialIndexes: number[]; } ================================================ FILE: src/utilities/flatten-layers.test.ts ================================================ import { flattenLayers } from './flatten-layers'; import { mockLayer } from '../test-utils'; describe('flattenLayers', () => { it('shallow clones the original array of layers', () => { const layers = [mockLayer({ width: 1, height: 1 })]; expect(flattenLayers(layers)).not.toBe(layers); }); it('flattens nested layers from inputLayer property', () => { const layer1 = mockLayer({ width: 1, height: 1, id: 'layer1' }); const layer2 = mockLayer({ width: 1, height: 1, id: 'layer2' }); const layer3 = mockLayer({ width: 1, height: 1, id: 'layer3' }); const layer4 = mockLayer({ width: 1, height: 1, id: 'layer4' }); layer4.inputLayer = layer3; layer3.inputLayer = layer2; layer2.inputLayer = layer1; expect(flattenLayers([layer4])).toEqual([layer1, layer2, layer3, layer4]); }); it('flattens nested layers from inputLayer1 and inputLayer2 properties', () => { const layer1 = mockLayer({ width: 1, height: 1, id: 'layer1' }); const layer2 = mockLayer({ width: 1, height: 1, id: 'layer2' }); const layer3 = mockLayer({ width: 1, height: 1, id: 'layer3' }); const layer4 = mockLayer({ width: 1, height: 1, id: 'layer4' }); layer4.inputLayer1 = layer2; layer4.inputLayer2 = layer3; layer2.inputLayer = layer1; expect(flattenLayers([layer4])).toEqual([layer1, layer2, layer3, layer4]); }); }); ================================================ FILE: src/utilities/flatten-layers.ts ================================================ import { ILayer } from '../layer/base-layer'; import { traverseLayersFrom } from './traverse-layers-from'; export function flattenLayers(layers: ILayer[]): ILayer[] { const result = layers.slice(0); for (let i = 0; i < result.length; i++) { let offset = 0; traverseLayersFrom(result[i], (layer: ILayer) => { if (!result.includes(layer)) { result.splice(i + offset, 0, layer); offset++; } }); } return result; } ================================================ FILE: src/utilities/kernel.ts ================================================ import { GPU, IConstantsThis, IGPUKernelSettings, IKernelMapRunShortcut, IKernelRunShortcut, Input, ISubKernelObject, KernelFunction, KernelOutput, OutputDimensions, Texture, ThreadFunction, ThreadKernelVariable, } from 'gpu.js'; let gpuInstance: GPU | null = null; /** * Sets up the gpu.js instance */ export function setup(value: GPU): void { gpuInstance = value; } /** * Destroys any existing gpu.js instance */ export function teardown(): void { if (gpuInstance !== null) { gpuInstance.destroy().catch(console.log); } gpuInstance = null; } export function makeKernel< ArgTypes extends ThreadKernelVariable[] = ThreadKernelVariable[], ConstantsTypes extends IConstantsThis = IConstantsThis >( fn: KernelFunction, settings: IGPUKernelSettings & { constants?: ConstantsTypes } ): IKernelRunShortcut { let _gpuInstance: GPU = gpuInstance as GPU; if (_gpuInstance === null) { _gpuInstance = new GPU({ mode: 'gpu' }); setup(_gpuInstance); } return _gpuInstance .createKernel(fn, settings) .setPipeline(true); } export function makeKernelMap< ArgTypes extends ThreadKernelVariable[], ConstantsTypes extends IConstantsThis >( map: ISubKernelObject, fn: ThreadFunction, settings: IGPUKernelSettings & { constants?: ConstantsTypes } ): IKernelMapRunShortcut { let _gpuInstance: GPU = gpuInstance as GPU; if (_gpuInstance === null) { _gpuInstance = new GPU({ mode: 'gpu' }); setup(_gpuInstance); } return _gpuInstance .createKernelMap(map, fn, settings) .setPipeline(true); } /** * Compiles a function into a gpu.js dev mode kernel */ // export function makeDevKernel( // fn: ThreadFunction, // settings: makeKernelSettings // ): IKernelRunShortcut { // if ('map' in settings) { // throw new Error('map kernels are not supported by dev kernels'); // } // const gpu = new GPU({ mode: 'dev' }); // return gpu.createKernel(fn, settings); // } export function kernelInput(value: number[], size: OutputDimensions): Input { return new Input(value, size); } /** * Deletes a gpu.js texture and frees VRAM */ export function release(possibleTexture: KernelOutput | Input): void { if (possibleTexture instanceof Texture) { possibleTexture.delete(); } } /** * Cleans ie sets all elements to 0 of a Texture or a js array */ export function clear(value: KernelOutput): void { if (value instanceof Texture) { value.clear(); return; } // array if (Array.isArray(value)) { if (typeof value[0] === 'number') { (value as number[]).fill(0); } else if (typeof value[0][0] === 'number') { for (let x = 0; x < value.length; x++) { (value[x] as number[]).fill(0); } return; } else if (typeof value[0][0][0] === 'number') { // cube for (let y = 0; y < value.length; y++) { const row: number[][] = value[y] as number[][]; for (let x = 0; x < row.length; x++) { row[x].fill(0); } } return; } } if (value instanceof Float32Array) { value.fill(0); return; } throw new Error('unhandled value'); } /** * Clones a value */ export function clone(value: KernelOutput): KernelOutput { if (value instanceof Texture) { return value.clone(); } if (value instanceof Float32Array) { return value.slice(0); } if (Array.isArray(value)) { if (typeof value[0] === 'number') { return value.slice(0); } else if (typeof value[0][0] === 'number') { const matrix = new Array(value.length); for (let x = 0; x < value.length; x++) { matrix[x] = (value[x] as Float32Array).slice(0); } return matrix; } else if (typeof value[0][0][0] === 'number') { const cube = new Array(value.length); for (let y = 0; y < value.length; y++) { const row = value[y] as number[][]; const matrix = new Array(row.length); for (let x = 0; x < row.length; x++) { matrix[x] = row[x].slice(0); } } return cube; } } throw new Error('unhandled value'); } ================================================ FILE: src/utilities/layer-from-json.test.ts ================================================ import { Add, Convolution, ILayerJSON, RecurrentZeros, Sigmoid, Target, } from '../layer'; import { layerFromJSON } from './layer-from-json'; import { mockLayer } from '../test-utils'; describe('layerFromJSON', () => { it('should return null when type is specified in a wrong way', () => { const jsonLayer: ILayerJSON = { width: 5, height: 5, depth: 2, weights: null, type: 'WrongType', praxisOpts: null, }; expect(layerFromJSON(jsonLayer)).toBe(null); }); describe('when used with a Filter layer json', () => { const jsonLayer = { width: 1, height: 1, depth: 1, type: 'Convolution', praxisOpts: null, stride: 1, filters: [[[1]]], weights: [[[2]]], biasDeltas: [[[3]]], }; it('fails if inputLayer falsey', () => { expect(() => layerFromJSON(jsonLayer)).toThrow('inputLayer missing'); }); it('should return that type instantiated', () => { const inputLayer = mockLayer({ width: 1, height: 1, depth: 1, }); expect(layerFromJSON(jsonLayer, inputLayer)).toEqual( new Convolution(jsonLayer, inputLayer) ); }); }); describe('when used with a Activation layer json', () => { const jsonLayer: ILayerJSON = { width: 5, height: 5, weights: null, type: 'Sigmoid', praxisOpts: null, }; it('fails if inputLayer1 falsey', () => { expect(() => layerFromJSON(jsonLayer)).toThrow('inputLayer missing'); }); it('should return that type instantiated', () => { const inputLayer1 = mockLayer(); expect(layerFromJSON(jsonLayer, inputLayer1)).toEqual( new Sigmoid(inputLayer1, jsonLayer) ); }); }); describe('when used with a Operator layer json', () => { const jsonLayer: ILayerJSON = { width: 5, height: 5, weights: null, type: 'RecurrentZeros', praxisOpts: null, }; it('should return that type instantiated', () => { expect(layerFromJSON(jsonLayer)).toEqual(new RecurrentZeros(jsonLayer)); }); }); describe('when used with a Operator layer json', () => { const jsonLayer: ILayerJSON = { width: 5, height: 5, weights: null, type: 'Add', praxisOpts: null, }; it('fails if inputLayer1 falsey', () => { expect(() => layerFromJSON(jsonLayer)).toThrow('inputLayer1 missing'); }); it('fails if inputLayer2 falsey', () => { const inputLayer1 = mockLayer(); expect(() => layerFromJSON(jsonLayer, inputLayer1)).toThrow( 'inputLayer2 missing' ); }); it('should return that type instantiated', () => { const inputLayer1 = mockLayer(); const inputLayer2 = mockLayer(); expect(layerFromJSON(jsonLayer, inputLayer1, inputLayer2)).toEqual( new Add(inputLayer1, inputLayer2, jsonLayer) ); }); }); describe('when used with a TargetType layer json', () => { const jsonLayer: ILayerJSON = { width: 5, height: 5, weights: null, type: 'Target', praxisOpts: null, }; it('fails if inputLayer falsey', () => { expect(() => layerFromJSON(jsonLayer)).toThrow('inputLayer missing'); }); it('should return that type instantiated', () => { const inputLayer = mockLayer(); expect(layerFromJSON(jsonLayer, inputLayer)).toEqual( new Target(jsonLayer, inputLayer) ); }); }); }); ================================================ FILE: src/utilities/layer-from-json.ts ================================================ import * as layer from '../layer'; import { layerTypes, ILayerJSON, ILayer, Target } from '../layer'; import { ActivationType } from '../layer/activation'; import { FilterType } from '../layer/filter'; import { InternalType } from '../layer/internal'; import { ModifierType } from '../layer/modifier'; import { OperatorType } from '../layer/operator'; import { BaseLayerType } from '../layer/base-layer'; import { TargetType } from '../layer/target'; const layerNameTypes = Object.keys(layer); export function layerFromJSON( jsonLayer: ILayerJSON, inputLayer1?: ILayer, inputLayer2?: ILayer ): ILayer | null { if ( !layerNameTypes.find((layerNameType) => layerNameType === jsonLayer.type) ) { return null; } const Layer = ((layer as unknown) as { [layerType: string]: | TargetType | ActivationType | FilterType | InternalType | ModifierType | OperatorType; })[jsonLayer.type]; if (Layer.prototype instanceof layerTypes.Filter) { if (!inputLayer1) throw new Error('inputLayer missing'); return new (Layer as FilterType)(jsonLayer, inputLayer1); } else if ( Layer.prototype instanceof layerTypes.Activation || Layer.prototype instanceof layerTypes.Modifier ) { if (!inputLayer1) throw new Error('inputLayer missing'); return new (Layer as ActivationType)(inputLayer1, jsonLayer); } else if (Layer.prototype instanceof layerTypes.Internal) { return new (Layer as InternalType)(jsonLayer); } else if (Layer.prototype instanceof layerTypes.Operator) { if (!inputLayer1) throw new Error('inputLayer1 missing'); if (!inputLayer2) throw new Error('inputLayer2 missing'); return new (Layer as OperatorType)(inputLayer1, inputLayer2, jsonLayer); } else if ( Layer.prototype instanceof layerTypes.InternalModel || Layer.prototype instanceof layerTypes.EntryPoint || Layer.prototype instanceof layerTypes.Model ) { return new (Layer as BaseLayerType)(jsonLayer); } else if (Layer === Target) { if (!inputLayer1) throw new Error('inputLayer missing'); return new (Layer as TargetType)(jsonLayer, inputLayer1); } return null; } ================================================ FILE: src/utilities/layer-setup.ts ================================================ import { IConvolutionSettingsBase } from '../layer/convolution'; export interface IStride { strideX: number; strideY: number; } export function getStride( settings: IConvolutionSettingsBase, defaults: IConvolutionSettingsBase ): IStride { if (typeof settings.stride === 'number') { return { strideX: settings.stride, strideY: settings.stride }; } else { let strideX: number = defaults.stride as number; let strideY: number = defaults.stride as number; if (typeof settings.strideX === 'number') { strideX = settings.strideX; } if (typeof settings.strideY === 'number') { strideY = settings.strideY; } return { strideX, strideY }; } } export interface IPadding { paddingX: number; paddingY: number; } export function getPadding( settings: IConvolutionSettingsBase, defaults: IConvolutionSettingsBase ): IPadding { if (typeof settings.padding === 'number') { return { paddingX: settings.padding, paddingY: settings.padding }; } else { let paddingX: number = defaults.padding as number; let paddingY: number = defaults.padding as number; if (typeof settings.paddingX === 'number') { paddingX = settings.paddingX; } if (typeof settings.paddingY === 'number') { paddingY = settings.paddingY; } return { paddingX, paddingY }; } } ================================================ FILE: src/utilities/layer-size.test.ts ================================================ import { BaseLayer, ILayer } from '../layer'; import { checkSameSize } from './layer-size'; let layer: ILayer; let layerToCompare: ILayer; describe('LayerSize', () => { describe('checkSameSize', () => { describe('if layer1.width !== layer2.width', () => { it('throws', () => { layer = new BaseLayer({ width: 10 }); layerToCompare = new BaseLayer({ width: 10.1 }); expect(() => { checkSameSize(layer, layerToCompare); }).toThrow(); }); }); describe('if layer1.width === layer2.width', () => { it('throws', () => { expect(() => { layer = new BaseLayer({ width: 10 }); layerToCompare = new BaseLayer({ width: 10 }); checkSameSize(layer, layerToCompare); }).not.toThrow(); }); }); describe('if layer1.height !== layer2.height', () => { it('throws', () => { expect(() => { layer = new BaseLayer({ height: 10 }); layerToCompare = new BaseLayer({ height: 10.1 }); checkSameSize(layer, layerToCompare); }).toThrow(); }); }); describe('if layer1.height === layer2.height', () => { it('throws', () => { expect(() => { layer = new BaseLayer({ height: 10 }); layerToCompare = new BaseLayer({ height: 10 }); checkSameSize(layer, layerToCompare); }).not.toThrow(); }); }); }); }); ================================================ FILE: src/utilities/layer-size.ts ================================================ import { ILayer } from '../layer/base-layer'; export function checkSameSize(layer1: ILayer, layer2: ILayer): void { if (layer1.width !== layer2.width) { throw new Error( `Layer width mismatch of ${layer1.width} and ${layer2.width}` ); } if (layer1.height !== layer2.height) { throw new Error( `Layer height mismatch of ${layer1.height} and ${layer2.height}` ); } } ================================================ FILE: src/utilities/lookup-table.ts ================================================ import { InputOutputValue, INumberHash, ITrainingDatum } from '../lookup'; export type LookupTableProp = 'input' | 'output'; export class LookupTable { length: number; prop: LookupTableProp | null = null; table: INumberHash = {}; constructor( data: ITrainingDatum[] | InputOutputValue[] | InputOutputValue[][], prop?: LookupTableProp ) { this.length = 0; const table = this.table; if (prop) { this.prop = prop; for (let i = 0; i < data.length; i++) { const datum = (data as ITrainingDatum[])[i]; const object = datum[prop] as INumberHash; for (const p in object) { if (!object.hasOwnProperty(p)) continue; if (table.hasOwnProperty(p)) continue; table[p] = this.length++; } } } else if (Array.isArray(data) && Array.isArray(data[0])) { for (let i = 0; i < data.length; i++) { const array = (data as InputOutputValue[][])[i]; for (let j = 0; j < array.length; j++) { const object = array[j]; for (const p in object) { if (!object.hasOwnProperty(p)) continue; if (table.hasOwnProperty(p)) continue; table[p] = this.length++; } } } } else { for (let i = 0; i < data.length; i++) { const object = (data as INumberHash[])[i]; for (const p in object) { if (!object.hasOwnProperty(p)) continue; if (table.hasOwnProperty(p)) continue; table[p] = this.length++; } } } } } ================================================ FILE: src/utilities/max.test.ts ================================================ import { max } from './max'; describe('max', () => { test('should find max in object', () => { const obj = { a: 1, b: 5, c: 10, d: 0 }; expect(max(obj)).toBe(10); }); }); ================================================ FILE: src/utilities/max.ts ================================================ export function max( values: | Float32Array | { [key: string]: number; } ): number { if (Array.isArray(values) || values instanceof Float32Array) { return Math.max(...values); } else { return Math.max(...Object.values(values)); } } ================================================ FILE: src/utilities/mse.test.ts ================================================ import { toArray } from './to-array'; import { zeros } from './zeros'; describe('mse', () => { test('should return the same array if an array are passed', () => { const collection = zeros(10); const temp = toArray(collection); expect(collection.constructor).toBe(temp.constructor); }); test('should return an array if object is passed', () => { const collection = { name: 0, // 'Steve Jobs', alive: 1, // false, }; const temp = toArray(collection); expect(temp.constructor).toBe(Float32Array); expect(temp.length).toBe(Object.keys(collection).length); }); }); ================================================ FILE: src/utilities/mse.ts ================================================ export function mse(errors: Float32Array): number { // mean squared error let sum = 0; for (let i = 0; i < errors.length; i++) { sum += errors[i] ** 2; } return sum / errors.length; } ================================================ FILE: src/utilities/ones.test.ts ================================================ import { ones, ones2D } from './ones'; describe('ones', () => { test('should return an array with all ones', () => { expect(ones(2)).toEqual(Float32Array.from([1, 1])); }); }); describe('ones2D', () => { test('should return an array with all ones', () => { expect(ones2D(2, 3)).toEqual([ Float32Array.from([1, 1]), Float32Array.from([1, 1]), Float32Array.from([1, 1]), ]); }); }); ================================================ FILE: src/utilities/ones.ts ================================================ export function ones(size: number): Float32Array { return new Float32Array(size).fill(1); } export function ones2D(width: number, height: number): Float32Array[] { const result = new Array(height); for (let y = 0; y < height; y++) { result[y] = ones(width); } return result; } ================================================ FILE: src/utilities/random-weight.test.ts ================================================ import { randomWeight } from './random-weight'; describe('randomWeight', () => { test('weight', () => { expect(typeof randomWeight()).toBe('number'); }); }); ================================================ FILE: src/utilities/random-weight.ts ================================================ export function randomWeight(): number { return Math.random() * 0.4 - 0.2; } ================================================ FILE: src/utilities/random.test.ts ================================================ import { randomFloat, randomInteger, randomN } from './random'; describe('random', () => { test('randomF', () => { const val = randomFloat(0, 10); expect(typeof val).toBe('number'); expect(val).toBeGreaterThan(0); expect(val).toBeLessThan(11); }); test('randomI', () => { const val = randomInteger(0, 10); expect(typeof val).toBe('number'); expect(val).toBeGreaterThanOrEqual(0); expect(val).toBeLessThan(11); }); test('randomN', () => { const val = randomN(10, 5); expect(typeof val).toBe('number'); }); }); ================================================ FILE: src/utilities/random.ts ================================================ /** * Returns a random float between given min and max bounds (inclusive) * @param min Minimum value of the ranfom float * @param max Maximum value of the random float */ export function randomFloat(min: number, max: number): number { return Math.random() * (max - min) + min; } /** * Complicated math. All you need to know is that it returns a random number. * More info: https://en.wikipedia.org/wiki/Normal_distribution */ export function gaussRandom(): number { if (gaussRandom.returnV) { gaussRandom.returnV = false; return gaussRandom.vVal; } const u = 2 * Math.random() - 1; const v = 2 * Math.random() - 1; const r = u * u + v * v; if (r === 0 || r > 1) { return gaussRandom(); } const c = Math.sqrt((-2 * Math.log(r)) / r); gaussRandom.vVal = v * c; // cache this gaussRandom.returnV = true; return u * c; } /** * Returns a random integer between given min and max bounds * @param min Minimum value of the random integer * @param max Maximum value of the random integer */ export function randomInteger(min: number, max: number): number { return Math.floor(Math.random() * (max - min) + min); } /** * If you know what this is: https://en.wikipedia.org/wiki/Normal_distribution * @param mu * @param std */ export function randomN(mu: number, std: number): number { return mu + gaussRandom() * std; } gaussRandom.returnV = false; gaussRandom.vVal = 0; ================================================ FILE: src/utilities/randos.test.ts ================================================ import { randos } from './randos'; describe('randos', () => { test('should return an array of finite random weights', () => { const temp: Float32Array = randos(10); const tempCheck: Float32Array = temp.filter((el) => Number.isFinite(el)); expect(temp.length).toBe(tempCheck.length); }); }); ================================================ FILE: src/utilities/randos.ts ================================================ import { randomWeight } from './random-weight'; import { randomFloat } from './random'; /** * Returns an array of given size, full of randomness */ export function randos(size: number, std: number | null = null): Float32Array { const array: Float32Array = new Float32Array(size); if (std === null) { for (let i = 0; i < size; i++) { array[i] = randomWeight(); } } else { for (let i = 0; i < size; i++) { array[i] = randomFloat(-std, std); } } return array; } /** * Returns a 2D matrix of given size, full of randomness */ export function randos2D( width: number, height: number, std?: number | null ): Float32Array[] { const result: Float32Array[] = new Array(height); for (let y = 0; y < height; y++) { result[y] = randos(width, std); } return result; } /** * Returns a 3D tensor of given size, full of randomness */ export function randos3D( width: number, height: number, depth: number, std?: number | null ): Float32Array[][] { const result: Float32Array[][] = new Array(depth); for (let z = 0; z < depth; z++) { result[z] = randos2D(width, height, std); } return result; } ================================================ FILE: src/utilities/range.test.ts ================================================ import { range } from './range'; describe('range', () => { test('should return range from start & end', () => { expect(range(0, 1)).toBeInstanceOf(Array); expect(range(5, 10)).toEqual([5, 6, 7, 8, 9]); }); }); ================================================ FILE: src/utilities/range.ts ================================================ /** * * @param start * @param end * @returns {Array} */ export function range(start: number, end: number): number[] { const result: number[] = []; for (; start < end; start++) { result.push(start); } return result; } ================================================ FILE: src/utilities/to-array.test.ts ================================================ import { toArray } from './to-array'; describe('to-array', () => { it('should convert object to array', () => { const obj = { a: 1, b: 5, c: 10, d: 0 }; const array = toArray(obj); expect(array).toBeInstanceOf(Float32Array); expect(array.join('')).toBe([1, 5, 10, 0].join('')); expect(array.length).toBe(4); }); }); ================================================ FILE: src/utilities/to-array.ts ================================================ export function toArray( values: number[] | Float32Array | { [key: string]: number } ): Float32Array { if (Array.isArray(values)) { return Float32Array.from(values); } return Float32Array.from(Object.values(values)); } ================================================ FILE: src/utilities/to-svg.test.ts ================================================ import parser from 'fast-xml-parser'; import { FeedForward } from '../feed-forward'; import { feedForward, input, target } from '../layer'; import { NeuralNetwork } from '../neural-network'; import { Recurrent } from '../recurrent'; import { RNN } from '../recurrent/rnn'; import { RNNTimeStep } from '../recurrent/rnn-time-step'; import { toSVG } from './to-svg'; describe('toSvg', () => { const options = { height: 200, width: 300, radius: 4, line: { width: 0.5, color: 'black', className: 'test-connection', }, inputs: { color: 'rgba(0, 128, 0, 0.5)', labels: null, className: 'test-input', }, hidden: { color: 'rgba(255, 127, 80, 0.5)', className: 'test-hidden-neuron', }, outputs: { color: 'rgba(100, 149, 237, 0.5)', className: 'test-output', }, fontSize: '11px', fontClassName: 'test-label', }; describe('`NeuralNetwork` input', () => { it('should throw if net is invalid', () => { expect(() => { const empty = {}; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error toSVG(empty, options); }).toThrow(); }); it('should return valid xml', () => { const net = new NeuralNetwork({ inputSize: 2, hiddenLayers: [3], outputSize: 1, }); expect(parser.validate(toSVG(net, options))).toBe(true); }); it('should have proper numbers of neurons', () => { const net = new NeuralNetwork({ inputSize: 2, hiddenLayers: [3], outputSize: 1, }); const svg = toSVG(net, options); const json = parser.parse(svg, { ignoreAttributes: false, attributeNamePrefix: '', }); expect(json.svg.rect.length).toBe(2); for (let i = 0; i < 2; i++) { expect(json.svg.rect[i].class).toBe('test-input'); } expect(json.svg.rect[1].class).toBe('test-input'); expect(json.svg.line.length).toBe(12); for (let i = 0; i < 12; i++) { expect(json.svg.line[i].class).toBe('test-connection'); } expect(json.svg.circle.length).toBe(4); // hidden neurons first for (let i = 0; i < 3; i++) { expect(json.svg.circle[i].class).toBe('test-hidden-neuron'); } expect(json.svg.circle[3].class).toBe('test-output'); }); it('throws if inputs.labels are defined as array, but do not match the input count', () => { const net = new NeuralNetwork({ inputSize: 2, hiddenLayers: [3], outputSize: 1, }); expect(() => { toSVG(net, { ...options, inputs: { ...options.inputs, labels: ['first'], }, }); }).toThrow(); }); it('can have labels', () => { const net = new NeuralNetwork({ inputSize: 2, hiddenLayers: [3], outputSize: 1, }); const svg = toSVG(net, { ...options, inputs: { ...options.inputs, labels: ['first', 'second'], }, }); const json = parser.parse(svg, { ignoreAttributes: false, attributeNamePrefix: '', }); expect(json.svg.text.length).toBe(2); expect(json.svg.text[0].class).toBe('test-label'); expect(json.svg.text[0]['#text']).toBe('first'); expect(json.svg.text[1].class).toBe('test-label'); expect(json.svg.text[1]['#text']).toBe('second'); }); }); describe('`json` NeuralNetwork input', () => { it('should throw when empty net object provided', () => { const empty = {}; expect(() => { // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error toSVG(empty, options); }).toThrow(); }); it('should return valid xml', () => { const net = { inputSize: 4, hiddenLayers: [3], outputSize: 2, }; expect(parser.validate(toSVG(net, options))).toBe(true); }); }); describe('`RNN` input', () => { it('should throw when empty net object provided', () => { const empty = new RNN({ inputSize: 0, hiddenLayers: [], outputSize: 0, }); expect(() => { toSVG(empty, options); }).toThrow(); }); it('should return valid xml', () => { const net = new RNN({ inputSize: 2, hiddenLayers: [3], outputSize: 1, }); expect(parser.validate(toSVG(net, options))).toBe(true); }); }); describe('`RNN` json input', () => { it('should return valid xml', () => { const net = new RNN({ inputSize: 2, hiddenLayers: [3], outputSize: 1, }); net.initialize(); expect(parser.validate(toSVG(net.toJSON(), options))).toBe(true); }); }); describe('`RNNTimeStep` input', () => { it('should throw when empty net object provided', () => { const empty = new RNNTimeStep(); empty.options = { ...empty.options, inputSize: 0, hiddenLayers: [], outputSize: 0, }; expect(() => { toSVG(empty, options); }).toThrow(); }); it('should return valid xml', () => { const net = new RNNTimeStep({ inputSize: 2, hiddenLayers: [3], outputSize: 1, }); expect(parser.validate(toSVG(net, options))).toBe(true); }); }); describe('`RNNTimeStep` json input', () => { it('should return valid xml', () => { const net = new RNNTimeStep({ inputSize: 2, hiddenLayers: [3], outputSize: 1, }); net.initialize(); expect(parser.validate(toSVG(net.toJSON(), options))).toBe(true); }); }); describe('`FeedForward` input', () => { it('should throw when empty net object provided', () => { const empty = new FeedForward(); expect(() => { toSVG(empty, options); }).toThrow(); }); it('should return valid xml', () => { const net = new FeedForward({ inputLayer: () => input({ height: 2 }), hiddenLayers: [ (inputLayer) => feedForward({ height: 3 }, inputLayer), (inputLayer) => feedForward({ height: 1 }, inputLayer), ], outputLayer: (inputLayer) => target({ height: 1 }, inputLayer), }); expect(parser.validate(toSVG(net, options))).toBe(true); }); }); describe('`Recurrent` input', () => { it('should throw when empty net object provided', () => { const empty = new Recurrent(); expect(() => { toSVG(empty, options); }).toThrow(); }); it('should return valid xml', () => { const net = new Recurrent({ inputLayer: () => input({ height: 2 }), hiddenLayers: [ (inputLayer) => feedForward({ height: 3 }, inputLayer), (inputLayer) => feedForward({ height: 1 }, inputLayer), ], outputLayer: (inputLayer) => target({ height: 1 }, inputLayer), }); expect(parser.validate(toSVG(net, options))).toBe(true); }); }); describe('just using sizes', () => { it('should throw when empty net object provided', () => { const empty = { sizes: null }; expect(() => { // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error toSVG(empty, options); }).toThrow(); }); it('should return valid xml', () => { const net = { sizes: [2, 3, 1] }; expect(parser.validate(toSVG(net, options))).toBe(true); }); }); }); ================================================ FILE: src/utilities/to-svg.ts ================================================ import { FeedForward, IFeedForwardJSON } from '../feed-forward'; import { recurrentZeros } from '../layer/recurrent-zeros'; import { Recurrent } from '../recurrent'; import { IRNNJSON, RNN } from '../recurrent/rnn'; import { INeuralNetworkData, INeuralNetworkJSON, NeuralNetwork, } from '../neural-network'; import { GRU } from '../recurrent/gru'; import { LSTM } from '../recurrent/lstm'; import { NeuralNetworkGPU } from '../neural-network-gpu'; import { IRNNTimeStepJSON, RNNTimeStep } from '../recurrent/rnn-time-step'; import { LSTMTimeStep } from '../recurrent/lstm-time-step'; import { GRUTimeStep } from '../recurrent/gru-time-step'; import { ILayer } from '../layer'; interface LineDrawInfo { className: string; color: string; width: number; } interface NodeDrawInfo { className: string; color: string; } interface BaseDrawArgs { pixelX: number; pixelY: number; radius: number; row: number; column: number; } interface InputDrawArgs extends BaseDrawArgs { line: LineDrawInfo; inputs: NodeDrawInfo & { labels?: string[] | null }; fontSize: string; fontClassName: string; } export function drawInput({ pixelX, pixelY, radius, inputs, row, line, fontSize, fontClassName, }: InputDrawArgs): string { let svg = ` `; if (inputs.labels) { svg += `${inputs.labels[row]}`; } return svg; } export interface NeuronDrawArgs extends BaseDrawArgs { column: number; hidden: NodeDrawInfo; } export function drawNeuron({ pixelX, pixelY, row, column, radius, hidden, }: NeuronDrawArgs): string { return ``; } export interface OutputDrawArgs extends BaseDrawArgs { column: number; line: LineDrawInfo; outputs: NodeDrawInfo; } export function drawOutput({ pixelX, pixelY, row, column, line, outputs, radius, }: OutputDrawArgs): string { return ` `; } export interface BackwardConnectionsDrawArgs extends BaseDrawArgs { column: number; lineY: number; previousConnectionIndex: number; line: LineDrawInfo; } export function drawBackwardConnections({ pixelX, pixelY, row, column, radius, lineY, line, previousConnectionIndex, }: BackwardConnectionsDrawArgs): string { return ``; } export interface NeuralNetworkDrawOptions { sizes: number[]; height: number; width: number; radius: number; line: LineDrawInfo; inputs: NodeDrawInfo & { labels?: string[] | null }; hidden: NodeDrawInfo; outputs: NodeDrawInfo; fontSize: string; fontClassName: string; } export function neuralNetworkToInnerSVG( options: NeuralNetworkDrawOptions ): string { const { sizes, height, width } = options; let svg = ''; const pixelX = width / sizes.length; for (let column = 0; column < sizes.length; column++) { const size = sizes[column]; const pixelY = height / size; for (let row = 0; row < size; row++) { if (column === 0) { svg += drawInput({ pixelX, pixelY, row, column, ...options }); } else { if (column === sizes.length - 1) { svg += drawOutput({ pixelX, pixelY, row, column, ...options }); } else { svg += drawNeuron({ pixelX, pixelY, row, column, ...options }); } const previousSize = sizes[column - 1]; const lineY = height / previousSize; for ( let previousConnectionIndex = 0; previousConnectionIndex < previousSize; previousConnectionIndex++ ) { svg += drawBackwardConnections({ pixelX, pixelY, row, column, lineY, previousConnectionIndex, ...options, }); } } } } return svg; } export interface RecurrentConnectionsDrawArgs extends BaseDrawArgs { column: number; recurrentLine: LineDrawInfo; } export function drawRecurrentConnections({ pixelX, pixelY, row, column, radius, recurrentLine, }: RecurrentConnectionsDrawArgs): string { const moveX = pixelX / 2 + column * pixelX + radius + 1; const moveY = pixelY / 2 + row * pixelY; const x = moveX - radius * 2 - 2; const y = moveY; const x1 = x + 100; const y1 = y + 50; const x2 = moveX - 100; const y2 = moveY + 50; return ``; } export interface RecurrentNeuralNetworkDrawOptions extends NeuralNetworkDrawOptions { recurrentLine: LineDrawInfo; } export function rnnToInnerSVG( options: RecurrentNeuralNetworkDrawOptions ): string { const { width, height, recurrentLine, sizes, radius } = options; const pixelX = width / sizes.length; let svg = ` `; svg += neuralNetworkToInnerSVG(options); for (let column = 1; column < sizes.length; column++) { const size = sizes[column]; const pixelY = height / size; for (let row = 0; row < size; row++) { svg += drawRecurrentConnections({ pixelX, pixelY, row, column, radius, recurrentLine, }); } } return svg; } export function getFeedForwardLayers(network: FeedForward): ISimpleNet { const { options } = network; if (!options) { throw new Error('options not defined'); } if (!options.inputLayer) { throw new Error('options.inputLater not defined'); } if (!options.hiddenLayers) { throw new Error('options.hiddenLayers not defined'); } if (options.hiddenLayers.length < 1) { throw new Error('options.hiddenLayers is empty'); } if (!options.outputLayer) { throw new Error('options.outputLayer not defined'); } const inputLayer = options.inputLayer(); const hiddenLayers = []; hiddenLayers.push(options.hiddenLayers[0](inputLayer, 0)); for (let i = 1; i < options.hiddenLayers.length; i++) { hiddenLayers.push(options.hiddenLayers[i](hiddenLayers[i - 1], i)); } const outputLayer = options.outputLayer( hiddenLayers[hiddenLayers.length - 1], hiddenLayers.length ); return { inputSize: inputLayer.height, hiddenLayers: hiddenLayers.map((hiddenLayer: ILayer) => hiddenLayer.height), outputSize: outputLayer.height, }; } export function getRecurrentLayers(network: Recurrent): ISimpleNet { const hiddenLayers: ILayer[] = []; const { options } = network; if (!options.inputLayer) { throw new Error('inputLayer not defined'); } if (!options.outputLayer) { throw new Error('outputLayer not defined'); } const inputLayer = options.inputLayer(); hiddenLayers.push(options.hiddenLayers[0](inputLayer, recurrentZeros(), 0)); for (let i = 1; i < options.hiddenLayers.length; i++) { hiddenLayers.push( options.hiddenLayers[i](hiddenLayers[i - 1], recurrentZeros(), i) ); } const outputLayer = options.outputLayer( hiddenLayers[hiddenLayers.length - 1], -1 ); return { inputSize: inputLayer.height, hiddenLayers: hiddenLayers.map((hiddenLayer: ILayer) => hiddenLayer.height), outputSize: outputLayer.height, }; } export function wrapOuterSVG( svgBody: string, width: number, height: number ): string { // language=html return `${svgBody}`; } export function getNeuralNetworkJSONSizes(json: INeuralNetworkJSON): number[] { return json.sizes; } export function getNeuralNetworkSizes< InputType extends INeuralNetworkData, OutputType extends INeuralNetworkData >( net: | NeuralNetwork | NeuralNetworkGPU ): number[] { const { options, sizes } = net; const { inputSize, outputSize, hiddenLayers } = options; if (!sizes) { if (typeof inputSize === 'number' && inputSize < 1) { throw new Error('inputSize not set'); } if (typeof outputSize === 'number' && outputSize < 1) { throw new Error('outputSize not set'); } if (hiddenLayers?.some((v) => v < 1)) { throw new Error('hiddenLayers not set'); } } return typeof inputSize === 'number' && Array.isArray(hiddenLayers) && typeof outputSize === 'number' ? [inputSize].concat(hiddenLayers).concat([outputSize]) : sizes; } export function getRNNSizes( net: RNN | LSTM | GRU | RNNTimeStep | LSTMTimeStep | GRUTimeStep | IRNNJSON ): number[] { const { options } = net; const { inputSize, outputSize, hiddenLayers } = options; return [inputSize].concat(hiddenLayers).concat([outputSize]); } export function defaultOptions(): RecurrentNeuralNetworkDrawOptions { return { line: { width: 0.5, color: 'black', className: 'connection', }, recurrentLine: { width: 1, color: 'red', className: 'recurrence', }, inputs: { color: 'rgba(0, 128, 0, 0.5)', labels: null, className: 'input', }, outputs: { color: 'rgba(100, 149, 237, 0.5)', className: 'output', }, hidden: { color: 'rgba(255, 127, 80, 0.5)', className: 'hidden-neuron', }, fontSize: '14px', fontClassName: 'label', radius: 8, width: 400, height: 250, sizes: [], }; } export interface ISimpleNet { inputSize: number; hiddenLayers: number[]; outputSize: number; } export interface ISizes { sizes: number[]; } export function toSVG< T extends | ISimpleNet | ISizes | Recurrent | FeedForward | IFeedForwardJSON | RNNTimeStep | IRNNTimeStepJSON | LSTMTimeStep | GRUTimeStep | RNN | IRNNJSON | GRU | LSTM | NeuralNetwork | INeuralNetworkJSON | NeuralNetworkGPU, InputType extends INeuralNetworkData, OutputType extends INeuralNetworkData >( net: T, options?: | Partial | Partial ): string { const mergedOptions = { ...defaultOptions(), ...options }; const { width, height, inputs } = mergedOptions; // Get network size array for NeuralNetwork or NeuralNetworkGPU let sizes: number[] = []; if (net instanceof NeuralNetwork || net instanceof NeuralNetworkGPU) { sizes = getNeuralNetworkSizes(net); } // get network size for Recurrent else if (net instanceof Recurrent) { const { inputSize, hiddenLayers, outputSize } = getRecurrentLayers(net); sizes = [inputSize].concat(hiddenLayers).concat([outputSize]); } // get network size for FeedForward else if (net instanceof FeedForward) { const { inputSize, hiddenLayers, outputSize } = getFeedForwardLayers(net); sizes = [inputSize].concat(hiddenLayers).concat([outputSize]); } // handle json, recurrent first else if ( net instanceof RNN || net instanceof LSTM || net instanceof GRU || net instanceof RNNTimeStep || net instanceof LSTMTimeStep || net instanceof GRUTimeStep ) { return wrapOuterSVG( rnnToInnerSVG({ ...mergedOptions, sizes: checkSizes( getRNNSizes( (net as unknown) as | RNN | LSTM | GRU | RNNTimeStep | LSTMTimeStep | GRUTimeStep ), inputs.labels ), }), width, height ); } // handle json, NeuralNetwork else if (net.hasOwnProperty('type')) { switch ((net as INeuralNetworkJSON).type) { case 'NeuralNetwork': case 'NeuralNetworkGPU': return wrapOuterSVG( neuralNetworkToInnerSVG({ ...mergedOptions, sizes: checkSizes( getNeuralNetworkJSONSizes(net as INeuralNetworkJSON), inputs.labels ), }), width, height ); case 'RNN': case 'GRU': case 'LSTM': case 'RNNTimeStep': case 'GRUTimeStep': case 'LSTMTimeStep': return wrapOuterSVG( rnnToInnerSVG({ ...mergedOptions, sizes: checkSizes(getRNNSizes(net as IRNNJSON), inputs.labels), }), width, height ); default: throw new Error('unrecognized network'); } } else if ( net.hasOwnProperty('inputSize') && net.hasOwnProperty('hiddenLayers') && net.hasOwnProperty('outputSize') ) { const { inputSize, hiddenLayers, outputSize } = net as ISimpleNet; sizes = [inputSize, ...hiddenLayers, outputSize]; } else if (net.hasOwnProperty('sizes')) { sizes = (net as ISizes).sizes; } else { throw new Error('unrecognized network'); } return wrapOuterSVG( neuralNetworkToInnerSVG({ ...mergedOptions, sizes: checkSizes(sizes, inputs.labels), }), width, height ); } export function checkSizes( sizes: number[], labels: string[] | null | undefined ): number[] { if (!sizes) { throw new Error('sizes not set'); } if (sizes.some((size: number) => size < 1)) { throw new Error('sizes not set correctly'); } if (labels && labels.length !== sizes[0]) { throw new Error('not enough labels for inputs'); } return sizes; } ================================================ FILE: src/utilities/traverse-layers-excluding-from.ts ================================================ import { ILayer } from '../layer/base-layer'; export default function traverseLayersExcludingFrom( layer: ILayer, inputLayer: ILayer, recurrentLayer: ILayer, cb: (layer: ILayer) => void ): void { if (layer === inputLayer || layer === recurrentLayer) return; if (layer.hasOwnProperty('inputLayer')) { traverseLayersExcludingFrom( (layer as ILayer & { inputLayer: ILayer }).inputLayer, inputLayer, recurrentLayer, cb ); } else { if (layer.hasOwnProperty('inputLayer1')) { traverseLayersExcludingFrom( (layer as ILayer & { inputLayer1: ILayer }).inputLayer1, inputLayer, recurrentLayer, cb ); } if (layer.hasOwnProperty('inputLayer2')) { traverseLayersExcludingFrom( (layer as ILayer & { inputLayer2: ILayer }).inputLayer2, inputLayer, recurrentLayer, cb ); } } cb(layer); } ================================================ FILE: src/utilities/traverse-layers-from.ts ================================================ import { ILayer } from '../layer/base-layer'; export function traverseLayersFrom( layer: ILayer, cb: (layer: ILayer) => void ): void { if (layer.hasOwnProperty('inputLayer')) { traverseLayersFrom( (layer as ILayer & { inputLayer: ILayer }).inputLayer, cb ); } else { if (layer.hasOwnProperty('inputLayer1')) { traverseLayersFrom( (layer as ILayer & { inputLayer1: ILayer }).inputLayer1, cb ); } if (layer.hasOwnProperty('inputLayer2')) { traverseLayersFrom( (layer as ILayer & { inputLayer2: ILayer }).inputLayer2, cb ); } } cb(layer); } ================================================ FILE: src/utilities/values-2d.ts ================================================ import { values } from './values'; /** * Returns a matrix of given width and height with each element filled with the same value */ export function values2D( width: number, height: number, value: number ): Float32Array[] { const result: Float32Array[] = new Array(height); for (let y = 0; y < height; y++) { result[y] = values(width, value); } return result; } ================================================ FILE: src/utilities/values-3d.ts ================================================ import { values2D } from './values-2d'; /** * Returns a 3D tensor of given width, height and depth with each element equal to the given value */ export function values3D( width: number, height: number, depth: number, value: number ): Float32Array[][] { const result: Float32Array[][] = new Array(depth); for (let z = 0; z < depth; z++) { result[z] = values2D(width, height, value); } return result; } ================================================ FILE: src/utilities/values.ts ================================================ /** * Returns an array of a given size with each element filled with a single value */ export function values(size: number, value: number): Float32Array { return new Float32Array(size).fill(value); } ================================================ FILE: src/utilities/zeros-2d.ts ================================================ import { zeros } from './zeros'; /** * Returns a 2D tensor(matrix) of zeros */ export function zeros2D(width: number, height: number): Float32Array[] { const result: Float32Array[] = new Array(height); for (let y = 0; y < height; y++) { result[y] = zeros(width); } return result; } ================================================ FILE: src/utilities/zeros-3d.ts ================================================ import { zeros2D } from './zeros-2d'; /** * Returns a 3D tensor of arrays */ export function zeros3D( width: number, height: number, depth: number ): Float32Array[][] { const result: Float32Array[][] = new Array(depth); for (let z = 0; z < depth; z++) { result[z] = zeros2D(width, height); } return result; } ================================================ FILE: src/utilities/zeros.test.ts ================================================ import { zeros } from './zeros'; describe('zeros', () => { test('should return an array with all zeros', () => { const temp = zeros(10); const tempCheck = temp.filter((el) => el === 0); expect(temp.length).toBe(tempCheck.length); }); }); ================================================ FILE: src/utilities/zeros.ts ================================================ /** * Returns an array of zeros */ export function zeros(size: number): Float32Array { return new Float32Array(size); } ================================================ FILE: tsconfig.json ================================================ { "compilerOptions": { "allowJs": true, "allowSyntheticDefaultImports": true, "checkJs": false, "esModuleInterop": true, // "declaration": true, // "emitDeclarationOnly": true, // "declarationMap": true, "module": "esnext", "moduleResolution": "node", // "noEmit": true, "noEmitOnError": true, "outDir": "./dist", "rootDir": "./src", "skipLibCheck": true, "strict": true, "target": "es2018", // To match node 10: https://github.com/microsoft/TypeScript/wiki/Node-Target-Mapping#node-10 "lib": ["es2019", "es2019.Array", "dom", "esnext"] }, "include": [ "src/" ], "exclude": [ "node_modules", "**/*.json", "**/*.d.ts", "dist", "examples", "__coverage__" ] }