Repository: SemkoDev/nelson.cli Branch: locked Commit: db1f7e347e7b Files: 106 Total size: 392.8 KB Directory structure: gitextract__2nyh35j/ ├── .babelrc ├── .dockerignore ├── .editorconfig ├── .gitignore ├── .gitlab-ci.yml ├── CHANGELOG.md ├── CONTRIBUTING.md ├── Dockerfile ├── ENTRYNODES ├── ISSUE_TEMPLATE.md ├── LICENSE.md ├── README.md ├── config.ini.example ├── contrib/ │ └── ansible-playbook/ │ ├── .gitignore │ ├── README.md │ ├── group_vars/ │ │ └── all/ │ │ ├── common.yml │ │ ├── iri.yml │ │ └── nelson.yml │ ├── inventory │ ├── roles/ │ │ ├── common/ │ │ │ ├── handlers/ │ │ │ │ └── main.yml │ │ │ └── tasks/ │ │ │ ├── firewalld.yml │ │ │ ├── install.yml │ │ │ ├── main.yml │ │ │ ├── role.yml │ │ │ ├── setup_apt.yml │ │ │ ├── setup_pip.yml │ │ │ ├── setup_yum.yml │ │ │ └── ufw.yml │ │ ├── iri/ │ │ │ ├── files/ │ │ │ │ ├── iri.service │ │ │ │ ├── nbctl │ │ │ │ ├── ps_mem │ │ │ │ └── reattach │ │ │ ├── handlers/ │ │ │ │ └── main.yml │ │ │ ├── tasks/ │ │ │ │ ├── firewalld.yml │ │ │ │ ├── iri.yml │ │ │ │ ├── main.yml │ │ │ │ ├── role.yml │ │ │ │ └── ufw.yml │ │ │ └── templates/ │ │ │ └── iri.ini │ │ └── nelson/ │ │ ├── files/ │ │ │ └── nelson.service │ │ ├── handlers/ │ │ │ └── main.yml │ │ ├── tasks/ │ │ │ ├── firewall.yml │ │ │ ├── main.yml │ │ │ ├── nelson.yml │ │ │ └── role.yml │ │ └── templates/ │ │ └── config.ini.j2 │ └── site.yml ├── dist/ │ ├── api/ │ │ ├── index.js │ │ ├── node.js │ │ ├── peer.js │ │ ├── utils.js │ │ └── webhooks.js │ ├── index.js │ ├── nelson.js │ ├── node/ │ │ ├── __mocks__/ │ │ │ ├── iri.js │ │ │ └── node.js │ │ ├── base.js │ │ ├── guard.js │ │ ├── heart.js │ │ ├── index.js │ │ ├── iri.js │ │ ├── node.js │ │ ├── peer-list.js │ │ ├── peer.js │ │ └── tools/ │ │ ├── terminal.js │ │ └── utils.js │ └── simulation/ │ ├── bin/ │ │ ├── nelson.js │ │ └── network.js │ ├── index.js │ ├── network.js │ └── node.js ├── package.json └── src/ ├── api/ │ ├── __tests__/ │ │ ├── api-test.js │ │ ├── node-test.js │ │ ├── peer-test.js │ │ └── webhooks-test.js │ ├── index.js │ ├── node.js │ ├── peer.js │ ├── utils.js │ └── webhooks.js ├── index.js ├── nelson.js ├── node/ │ ├── __mocks__/ │ │ ├── iri.js │ │ └── node.js │ ├── __tests__/ │ │ ├── guard-test.js │ │ ├── heart-test.js │ │ ├── node-test.js │ │ ├── peer-list-test.js │ │ └── peer-test.js │ ├── base.js │ ├── guard.js │ ├── heart.js │ ├── index.js │ ├── iri.js │ ├── node.js │ ├── peer-list.js │ ├── peer.js │ └── tools/ │ ├── terminal.js │ └── utils.js └── simulation/ ├── __tests__/ │ └── node-network-integration-test.js ├── bin/ │ ├── nelson.js │ └── network.js ├── index.js ├── network.js └── node.js ================================================ FILE CONTENTS ================================================ ================================================ FILE: .babelrc ================================================ { "presets": ["es2015", "stage-2"] } ================================================ FILE: .dockerignore ================================================ .idea/ node_modules/ temp/ data/ builds/ config.ini *.iml .git ================================================ FILE: .editorconfig ================================================ root = true [*] indent_style = space indent_size = 4 end_of_line = lf charset = utf-8 trim_trailing_whitespace = true insert_final_newline = true ================================================ FILE: .gitignore ================================================ .idea/ node_modules/ temp/ data/ builds/ config.ini *.iml .history/ ================================================ FILE: .gitlab-ci.yml ================================================ image: docker:latest services: - docker:dind stages: - test - release cache: paths: - node_modules/ before_script: - apk update - apk add jq yarn - yarn install test: stage: test script: - yarn run test release:npm: stage: release only: - /v\d*\.\d*\.\d*/ except: - branches script: # build the package - yarn run make # publish on npm - echo '//registry.npmjs.org/:_authToken=${NPM_TOKEN}'>.npmrc - npm publish release:docker.io: stage: release only: - /v\d*\.\d*\.\d*/ except: - branches script: # build the package - yarn run make # export the current version from package.json into a variable - export PACKAGE_VERSION=$(cat package.json | jq -r .version) # login to official docker registry - docker login -u "$DOCKER_REGISTRY_USER" -p "$DOCKER_REGISTRY_PASSWORD" # push tagged with version - docker build --pull -t "$DOCKER_REGISTRY_IMAGE:$PACKAGE_VERSION" . - docker push "$DOCKER_REGISTRY_IMAGE:$PACKAGE_VERSION" # push tagged with latest - docker build --pull -t "$DOCKER_REGISTRY_IMAGE" . - docker push "$DOCKER_REGISTRY_IMAGE" release:gitlab: stage: release only: - /v\d*\.\d*\.\d*/ except: - branches script: # build the package - yarn run make # export the current version from package.json into a variable - export PACKAGE_VERSION=$(cat package.json | jq -r .version) # publish on the official docker hub registry - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY # push tagged with version - docker build --pull -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" . - docker push "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" # push tagged with latest - docker build --pull -t "$CI_REGISTRY_IMAGE" . - docker push "$CI_REGISTRY_IMAGE" ================================================ FILE: CHANGELOG.md ================================================ # Changelog All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). ## [Unreleased] ## [0.4.0] - 2018-02-03 ### Added - API basic HTTP auth ### Changed - Fixed orphaned neighbors check. - Fixed API security bug. ## [0.3.22] - 2018-01-29 ### Changed - Fixed tests on some systems that were failing. - Upgraded IOTA IRI JS Library to 0.4.7 ## [0.3.21] - 2018-01-24 ### Added - Additional entry nodes - Possible fix for #45 ECONNRESET error - Interval-compression of the neighbors database - IRI cleanup of neighbors. Possible fix for #50 - Additional Peer and PeerList tests. Fixes #43 - Guard tests - Basic node tests - Node network integration tests - Basic node network simulation package - Parts of the node simulation package for integration tests ### Changed - Upgraded minimal node version to 8.9.4 - Cleanup nelson on uncaught exception. Possible fix for #50 - Upgrades WebSockets to 4.0.0. Possible fix for #45 - Fixed docker to copy faster, ignoring unneeded files - Made docker run the tests while building ## [0.3.16] - 2018-01-09 ### Changed - Fixed IRI TCP negotiation bug #5 ## [0.3.15] - 2018-01-09 ### Changed - Fixed IRI TCP negotiation bug #4 - Removed binaries from the versioning - Fixed terminal display ## [0.3.12] - 2018-01-06 ### Changed - Fixed IRI TCP negotiation bug #3 ## [0.3.11] - 2018-01-06 ### Changed - Fixed IRI TCP negotiation bug #2 ## [0.3.9] - 2018-01-06 ### Changed - Fixed IRI TCP negotiation bug ## [0.3.8] - 2018-01-06 ### Added - IRI protocol negotiation between nodes ### Changed - Fixed ECONNRESET bug. ## [0.3.5] - 2018-01-02 ### Changed - Fixes IPv6 check ## [0.3.4] - 2018-01-02 ### Changed - Fixed removed static neighbors on exit. - Fixed possible neighbor leak in IRI. - Fixes IPv6 URIs. - Updated Dockerfile to make the build faster. ## [0.3.1] - 2018-01-02 ### Added - TCP switch for IRI ### Changed - Improved neighbor weighting algorithm. Fixed a few minor bugs. - Smarter neighbor quality algorithm. - Random peer dropping inversely-weighted by peer quality now. - Improved incoming new/top peer rules. - Restructured and cleaned up the README. - Increased default minimal neighbors back to 5+6 (11) for stronger security. ## [0.3.0] - 2017-12-27 ### Added - IRI info to the API. - Webhooks. - Dynamic IP support. - Node naming. - Temporarily penalizing lazy/broken neighbors. ### Changed - Access to the whole peer list only from local requests. - Fixes trust updating issues. ## [0.2.5] - 2017-12-21 ### Added - Request throttling guard. - Made incoming/outgoing limits public. - Warnings when setting too low incoming/outgoing limits. ### Changed - Updated iota.lib.js - Fixes hard limits for nodes. - Lowers the amount of minimum nodes to 9 - Limited the amount of recommended/shared nodes. - Allowed cross-origin requests to API. ## [0.2.4] - 2017-12-19 ### Added - Readme info on pm2 manager and docker volume mounting. ### Changes - Makes Nelson ignore static neighbors completely, even if they run Nelson as well. ## [0.2.3] - 2017-12-19 ### Added - Ansible playbook for Nelson ### Changed - README docker ports for IRI - Terminal: prevent box overlapping ## [0.2.2] - 2017-12-18 ### Adds - peer-stats to API - Checking of NELSON_CONFIG env var for configuration path. ## [0.2.1] - 2017-12-18 ### Changed - Fixes getNeighbors when used in config.ini ## [0.2.0] - 2017-12-18 ### Added - Automatic entry nodes list downloading - IRI healthchecks on startup without throwing an error. - Actively remove peers, if the limit is trespassed at any point for any reason. - Improved Dockerfile. ## [0.1.11] - 2017-12-16 ### Changed - Fixes IRI neighbors removal ## [0.1.10] - 2017-12-16 ### Changed - Replacing only incoming nodes with trusted nodes (possible limit breaker) ## [0.1.9] - 2017-12-16 ### Changed - Switched IRI to run in UDP mode due to TCP bugs in IRI. https://github.com/iotaledger/iri/issues/345 ## [0.1.8] - 2017-12-16 ### Added - Delayed retry of unavailable peers. ### Changed - Default IRI API port: 14265 - Epoch time to 15 minutes - Delayed neighbors remove from IRI (prevent orphans) ### Removed - Removed instant drops after handshake due to oft reconnects (moved into handshake) ## [0.1.7] - 2017-12-15 ### Changed - DNS resolve hostnames provided by IRI in health checks. ## [0.1.6] - 2017-12-15 ### Changed - Improved logs - Fixed orphaned IRI neighbors ## [0.1.5] - 2017-12-14 ### Changed - Improved connection strategy to minimize reconnects. - Improved incoming connection strategy to minimize dead nodes. ## [0.1.4] - 2017-12-13 ### Added - Fixed IRI health checks ## [0.1.3] - 2017-12-13 ### Added - Terminal GUI for Nelson - IRI health checks ### Changed - Cleaned up logs (double-removals of peers) - Minor bugfixes. ## [0.1.1] - 2017-12-13 ### Added - Option for setting nelson api listening hostname. ### Changed - Cleaned Docker README section. - Cleaned up logs (double-removals of peers) ## [0.1.0] - 2017-12-13 ### Added - setting of IRI's hostname ### Changed - Dockerfile to use specific nelson version - Readme about docker ## [0.0.7] - 2017-12-13 ### Added - Adds API versioning: drop connections from other major versions ### Changed - Fixes neighbors default port setting ## [0.0.6] - 2017-12-12 - improve console log visualization - added Dockerfile ## [0.0.5] - 2017-12-12 ### Changed - Dynamic openness in function with node's maturity. - Sharing of opinion about neighbours. - Implemented improved weighting from tri-tests. - Decreased the average number of connected nodes to 8 (+/-4). ## [0.0.4] - 2017-12-09 ### Added - Contributing message ### Changed - forgotten dist and bin updates for 0.0.3 ## [0.0.3] - 2017-12-09 ### Added - Command line params for incoming/outgoing slots count. ### Changed - How master nodes recycle peers (all) and treat outgoing connections. ## [0.0.2] - 2017-12-08 ### Added - Changelog - Nelson API server for status updates incl README part ### Changed - Nelson API default port to 18600 ## [0.0.1] - 2017-12-06 Initial version ================================================ FILE: CONTRIBUTING.md ================================================ # Contributing When contributing to this repository, please first discuss the change you wish to make via issue, email, or any other method with the owners of this repository before making a change. Please note we have a code of conduct, please follow it in all your interactions with the project. ## Pull Request Process 1. Ensure any install or build dependencies are removed before the end of the layer when doing a build. 2. Update the README.md with details of changes to the interface, this includes new environment variables, exposed ports, useful file locations and container parameters. 3. Increase the version numbers in any examples files and the README.md to the new version that this Pull Request would represent. The versioning scheme we use is [SemVer](http://semver.org/). 4. You may merge the Pull Request in once you have the sign-off of two other developers, or if you do not have permission to do that, you may request the second reviewer to merge it for you. ## Code of Conduct ### Our Pledge In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. ### Our Standards Examples of behavior that contributes to creating a positive environment include: * Using welcoming and inclusive language * Being respectful of differing viewpoints and experiences * Gracefully accepting constructive criticism * Focusing on what is best for the community * Showing empathy towards other community members Examples of unacceptable behavior by participants include: * The use of sexualized language or imagery and unwelcome sexual attention or advances * Trolling, insulting/derogatory comments, and personal or political attacks * Public or private harassment * Publishing others' private information, such as a physical or electronic address, without explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting ### Our Responsibilities Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. ### Scope This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. ### Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at roman@deviota.com. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. ### Attribution This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] [homepage]: http://contributor-covenant.org [version]: http://contributor-covenant.org/version/1/4/ ================================================ FILE: Dockerfile ================================================ FROM node:8.9.4-alpine as builder COPY . /usr/src/nelson WORKDIR /usr/src/nelson RUN npm install -g EXPOSE 16600 EXPOSE 18600 CMD ["/usr/local/bin/nelson"] ENTRYPOINT ["/usr/local/bin/nelson"] ================================================ FILE: ENTRYNODES ================================================ mainnet.deviota.com/16600 mainnet2.deviota.com/16600 iotairi.tt-tec.net/16600 voss-hosting.de/16600 136.243.73.66/16600 iotanode.party/16600 nelson.vanityfive.de/16600 tangle.vanityfive.de/16600 tanglenode.de/16600 nelson.iota.fm/16000 node.io7a.com/16600 us1.tangleno.de/16600 eu1.tangleno.de/16600 iota.bluemx.de/16600 nelson.iotacore.de/16600 ================================================ FILE: ISSUE_TEMPLATE.md ================================================ ### Expected behaviour ### Actual behaviour ### Steps to reproduce ### Basic Info * Operating System: * Node (npm) Version: * IRI Version: * Nelson version: ### Nelson Info * Epoch: * Cycle: * Connected peers: ================================================ FILE: LICENSE.md ================================================ Copyright (c) 2017, Roman Semko - SemkoDev GbR Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ================================================ FILE: README.md ================================================ # Nelson Nelson is a tool meant to be used with IOTA's IRI Node. It automatically manages neighbors of your full node, negotiating connections, finding new neighbors and protecting against bad actors. ## Table of contents * [Getting Started](#getting-started) * [Prerequisites](#prerequisites) * [Installing](#installing) * [Upgrading](#upgrading) * [Running as a service](#running-as-a-service) * [Docker](#docker) * [Building Locally](#building-locally) * [Configuration](#configuration) * [config.ini](#config.ini) * [Command line options](#command-line-options) * [Options description](#options-description) * [Automated Scripts](#automated-scripts) * [Amazon CloudFormation](#amazon-cloudformation) * [Running Nelson](#running-nelson) * [Initial nodes](#initial-nodes) * [Epochs and Cycles](#epochs-and-cycles) * [Monitor](#monitor) * [API](#api) * [Webhooks](#webhooks) * [FAQ](#faq) * [Contributing](#contributing) * [Donations](#donations) * [Running your own entry node](#running-your-own-entry-node) * [Authors](#authors) * [License](#license) ## Getting Started These instructions will get you a copy of the project up and running on your local machine. ### Prerequisites It is expected that you have already installed Java, downloaded the IRI jar file and know how to start it. The local IRI instance must have api enabled and allowing to add/remove neighbors. Nelson is running on Node.js You will have to install **node (at least version LTS 8.9.4)** and *npm* (node package manager) on your system. Alternatively to npm you can (and should) use yarn package manager. #### Port Forwarding If you are trying to run a Nelson node at home, you may need to open some ports (port forwarding) in your NAT Router: * **UDP 14600** * **TCP 15600** * **TCP 16600** Please refer to your Router's manual on how to do that. Furthermore, please be aware that apart of firewall and port-forwarding in router, your Internet provider may also be an issue. Some providers (like Vodafone in Germany) do not have enough IPv4 addresses for homes and thus use something called "**IPv4 over DS Lite**". In those cases the **traffic will not come through** over the ports mentioned above. Unfortunately, there is no quick fix for this issue (maybe changing providers). There is some hope with the upcoming PCP-protocol, this will not happen this year (2018) for most providers, though. #### WARNING FOR UBUNTU Ubuntu 16.04 apt comes with an **outdated Node version (4.X)**. You need to install the latest version separately: https://nodejs.org/en/download/package-manager/ ### Installing Globally install Nelson ``` npm install -g nelson.cli ``` And run it ``` nelson --gui --getNeighbors ``` The ```--getNeighbors``` option is used to download an entry set of trusted Nelson peers for new Nelson instances. As your Nelson stays online and gets to know its neighbors, it will rely less and less on the initial entry points. The ```--gui``` option is used to provide a simple GUI interface in the console. Below is the list of all possible options. ### Upgrading To upgrade your Nelson to version X.X.X, simply run: ``` npm install -g nelson.cli@x.x.x ``` **Please check where npm installs your global packages**! It happens very often that the first installed binary is put into ```/usr/local/bin``` and the updated into ```/usr/bin```. Run ```nelson --version``` after the upgrade to make sure you are using the most recent one. Update your scripts and/or services to point to the right binary! ### Running as a service You can use the [node process manager](http://pm2.keymetrics.io/) to run Nelson as a service. Just do the following: ``` # Install the process manager: npm install pm2 -g # Make pm2 start at startup: pm2 startup # Start the Nelson as service # If you created a nelson config somewhere on your system, provide the path to the config: pm2 start nelson -- --config /path/to/nelson-config.ini # Otherwise you can just do: pm2 start nelson # Save current processes runing with pm2 to startup on boot: pm2 save # Get Nelson logs: pm2 monit # or pm2 log ``` ## Docker Provided you have docker installed, Nelson can be started as follows: ``` docker run romansemko/nelson.cli ``` Hence, running IRI with Nelson can be done with two simple commands: ``` docker run -d --net host -p 14265:14265 --name iri iotaledger/iri docker run -d --net host -p 18600:18600 --name nelson romansemko/nelson.cli -r localhost -i 14265 -u 14777 -t 15777 --neighbors "mainnet.deviota.com/16600 mainnet2.deviota.com/16600 mainnet3.deviota.com/16600 iotairi.tt-tec.net/16600" ``` The options passed to Nelson's docker (```-r localhost -i 14265 -u 14600 -t 15600 --neighbors ...```) set IRI's hostname and ports (api, TCP, UDP) and the initial neighbors (You could also have used ```--getNeighbors```). Please refer below for more info on options. To keep Nelson's peer database outside of the container, so that you do not lose your collected neighbor's data, you can mount a volume bound to a host's folder: ``` docker run -d --net host -p 18600:18600 --name nelson -v /path/to/nelson/data/directory:/data romansemko/nelson.cli ``` ## Building Locally If you are a developer you may want to build the project locally and play around with the sources. Otherwise, ignore this section. Make sure you have [yarn](https://yarnpkg.com) package manager installed. Checkout the project: ``` git clone https://github.com/SemkoDev/nelson.cli.git cd nelson.cli ``` Install dependencies: ``` yarn install --pure-lockfile ``` Run tests and make binaries: ``` yarn make ``` Try to run Nelson: ``` node ./dist/nelson.js --gui --getNeighbors ``` ## Configuration You are free to either use command line options or an ```.ini``` file to configure Nelson. If you use a config file, it has precedence and all command line options are ignored. ### config.ini To use a configuration file, run Nelson with ```--config``` option: ``` nelson --config ./config.ini # Alternatively, set an environment variable: NELSON_CONFIG= ./config.ini nelson ``` You can provide one or more of the following options in your ini file. Example: ``` [nelson] name = My Nelson Node cycleInterval = 60 epochInterval = 300 apiPort = 18600 apiHostname = 127.0.0.1 port = 16600 IRIHostname = localhost IRIProtocol = any IRIPort = 14265 TCPPort = 15600 UDPPort = 14600 dataPath = data/neighbors.db ; maximal incoming connections. Please do not set below this limit: incomingMax = 5 ; maximal outgoing connections. Only set below this limit, if you have trusted, manual neighbors: outgoingMax = 4 isMaster = false silent = false gui = false getNeighbors = https://raw.githubusercontent.com/SemkoDev/nelson.cli/master/ENTRYNODES ; add as many initial Nelson neighbors, as you like neighbors[] = mainnet.deviota.com/16600 neighbors[] = mainnet2.deviota.com/16600 neighbors[] = mainnet3.deviota.com/16600 neighbors[] = iotairi.tt-tec.net/16600 ; Protect API with basic auth [nelson.apiAuth] username=user password=pass ``` #### WARNING ON NEIGHBORS: These are **NOT IRI neighbor** addresses, but the **Nelson** addresses. If you have used them erroneously as Nelson addresses in the past, chances are that Nelson will think these "static" neighbors are his and will keep removing them from IRI. To Fix this, just delete data/neighbors.db and start Nelson fresh with just ```--getNeighbors``` ### Command line options Command line options are named the same as INI options. Some have additional short versions. ### Options description | Option | Description | Default | | ----------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------- | | --name | Name your node. This identifier will appear in API/webhooks and for your neighbors | | --neighbors, -n | space-separated list of entry Nelson neighbors | | --getNeighbors | Downloads a list of entry Nelson neighbors. If no URL is provided, will use a default URL (https://raw.githubusercontent.com/SemkoDev/nelson.cli/master/ENTRYNODES). If this option is not set, no neighbors will be downloaded. This option can be used together with ````--neighbors`` | false | | --apiAuth | Add basic HTTP auth to API. On the command line, please provide username and password in `user:pass` format. If you use config file, you will have to create a new `[nelson.apiAuth]` section with `username` and `password` See the example above. | | --apiPort, -a | Nelson API port to request current node status data | 18600 | | --apiHostname, -o | Nelson API hostname to request current node status data. Default value will only listen to local connections | 127.0.0.1 | | --port, -p | TCP port, on which to start your Nelson instance | 16600 | | --webhooks, -w | List of URLS to regularly call back with the current node status data | | --webhookInterval | Interval in seconds between each webhook call | 30 | | --IRIHostname, -r | IRI API hostname of the running IRI node instance | localhost | | --IRIPort, -i | IRI API port of the running IRI node instance | 14265 | | --TCPPort, -t | IRI TCP Port | 15600 | | --UDPPort, -u | IRI UDP Port | 14600 | | --IRIProtocol | Protocol to use for connecting neighbors. Possible values **'any'**, **'preferudp'**, **'prefertcp'**, **'udp'**, **'tcp'**. **WARNING**: Please only use with IRI v.1.4.1.6 and do not set to **udp** or **tcp** unless you are 100% sure that you cannot accept other protocol connections in no circumstances. Otherwise, setting **udp** will categorically deny connections from **tcp**-only hosts and vice-versa. **Durung the upgrade phase** setting to **tcp** will probably make your node unreachable as all of the older Nelson version nodes will be running **udp** only! Preferably set **preferudp** or **prefertcp**. "**any**" is always the best choice. | any | | --dataPath, -d | path to the file, that will be used as neighbor storage | data/neighbors.db | | --silent, -s | Run the node without any output | | --gui, -g | Run the node in console-gui mode | | --cycleInterval | Interval between Nelson cycles | 60 | | --epochInterval | Interval between Nelson epochs | 300 | | --isMaster | Whether you are intending to run a master node | | --incomingMax | How many incoming connections to accept. Please do not set below the default value! | 5 | | --outgoingMax | How many active/outgoing connections to establish. Please do not set below the default value, if you do not have any static/manual neighbors! | 4 | | --lazyLimit | After how many seconds a new Neighbors without new transactions should be dropped | 300 | | --lazyTimesLimit | After how many consecutive connections from a consistently lazy neighbor, should it be penalized | 3 | ## Automated Scripts ### Amazon CloudFormation Thanks to [iotFab](https://github.com/iotFab) for creating the [cloudformation script](https://github.com/iotFab/iota-aws-full-node) to easily launch IRI+Nelson! If You have an AWS account, you can launch a new full node in a matter of few clicks: [![alt text](https://s3.amazonaws.com/cloudformation-examples/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-west-1#/stacks/new?stackName=IotaAwsFullNode&templateURL=https://s3-eu-west-1.amazonaws.com/nelson-iri/cloudformation.yml) 1. Make sure "Specify an Amazon S3 template URL" is checked and continue. 2. Click continue. You can leave all config with default values. 3. If you want to be able to access your instance, you will need to provide a keypair. This is not required, though. 4. Wait about 10 for the instance to launch. 5. Done! ## Running Nelson ### Initial nodes The neighbors you provide in the beginning are treated as trusted neighbors. This means that Nelson will be more inclined to accept contact requests from these neighbors and also to recommend them to other neighbors. They are also used as initial contact for a young Nelson. They provide him with other neighbors' addresses. ### Epochs and Cycles Nelson grows. And with each new age (epoch), he treats his neighbors differently. A neighbor that he didn't like in the past, might become his best friend in the new epoch. The epoch option defines the interval in seconds between each epoch change. Do not change it, unless you know, what you are doing. Nelson checks upon its neighbors from time to time to make sure they are okay. Sometimes the neighbors die without saying a word or maybe move somewhere else. Nelson wants to know, with whom he should keep in contact. Each cycle Nelson pings the neighbors, to make sure they are okay. You can control the cycle interval with the ```cycleInterval``` option. ### Monitor There is a simple [Nelson server/monitor](https://github.com/SemkoDev/nelson.gui) available at: https://github.com/SemkoDev/nelson.gui This is work in progress, so please bear with the simplicity. You might need to run your nelson.cli with ```--apiHostname 0.0.0.0``` so that the monitor web-app has access to the Nelson API server. ### API Nelson comes with a simple API to get its current status: ``` # Replace the port, if you changed it when starting Nelson: curl http://localhost:18600 # Answer: { "ready": true, "totalPeers": 200, "connectedPeers": [ { "hostname": "xxxxxxxxxxxxxxx", "ip": "xxxxxxxxxxxxxxxx", "port": 16600, "TCPPort": 15600, "UDPPort": 14600, "seen": 1, "connected": 50, "tried": 0, "weight": 0.75, "dateTried": "2017-12-18T07:58:10.614Z", "dateLastConnected": "2017-12-18T07:58:10.705Z", "dateCreated": "2017-12-17T00:07:16.787Z", "isTrusted": false, "_id": "pOsnVKeGtWufM6AI", "nelsonID": "544a0355" }, ... ], "config": { "cycleInterval": 60, "epochInterval": 900, "beatInterval": 10, "dataPath": "/data/neighbors.db", "port": 16600, "apiPort": 18600, "IRIPort": 14265, "TCPPort": 15777, "UDPPort": 14777, "isMaster": false, "temporary": false }, "heart": { "lastCycle": "2017-12-18T08:10:07.806Z", "lastEpoch": "2017-12-18T08:01:02.967Z", "personality": { "id": "d856113128efbb33d313f7a5bd2c6befa40923544a5ae478613e4ac4c0cd0314341f1b4c6fcc30fd5cfe08a1db709a2f", "publicId": "d8561131", "feature": "e" }, "currentCycle": 1944, "currentEpoch": 130, "startDate": "2017-12-16T23:40:04.615Z" } ``` You can also get the full list of known peers: ``` curl http://localhost:18600/peers ``` Or just the short stats about your known peers: ``` curl http://localhost:18600/peer-stats #Output: { "newNodes": { "hourAgo": 43, "fourAgo": 275, "twelveAgo": 733, "dayAgo": 1825, "weekAgo": 2466 }, "activeNodes": { "hourAgo": 133, "fourAgo": 463, "twelveAgo": 950, "dayAgo": 2133, "weekAgo": 2257 } } ``` if you use `apiAuth` option to protect your API, you will need to provide the authentication details in your requests: ``` curl -u username:password http://localhost:18600 ``` ### Webhooks You can provide Nelson a list of webhook URLs that have to be regularly called back with all the node stats data. It basically provides the same data as calling ```curl http://localhost:18600/``` API. All webhook requests are POST requests. To add a webhook to nelson, start it with ```--webhooks``` option: ``` nelson --webhooks "http://webhook.one/ http://webhook.two/" ``` ## FAQ ### Help! Nelson isn't connecting to neighbors! Depending on Nelson's age/epoch he might or might not like a certain neighbor. That's okay. Just wait for the neighbor to mature and he might accept you into his circle. This is more acute for new nodes without any neighbors at all. You might need to wait for quite some time to be accepted into the network. The same happens to your own Nelson instance. It might deny contact from new neighbors or those he doesn't know well. The less trusted and less known a neighbor is, the less likely your Nelson will contact him. This is a security measure to slowly structure the network and give more weight to old, trusted neighborhood. You can read more about it in the Nelson's release article: https://semkodev.com/nelson-in-a-nutshell/ ### Nelson is still not connecting! Make sure that Nelson's port (default: 16600) is not firewalled. ### Nelson connects to the neighbors, but I am not getting any transactions Make sure that you provided the correct TCP/UDP IRI ports to Nelson. If your ports differ from the defaults (TCP: 15600 and UDP: 14600) you have to provide them! ### Nelson constantly connects/disconnects Nelson generates a lot of log output. Each handshake try and fail generates at least 3 lines of logs: - Connecting - Closing connection - Removing neighbor from IRI (although non has been added, yet). This is Normal. ### I have too many neighbors Nelson adds up to 10/11 additional neighbors. If you have a lot of "manual" neighbors, this might be too much. ### I am getting an error: ``` usr/bin/env: »node“ Unknown command... ``` Make sure you have node v.8.9.4 or higher installed on your machine. ### I am getting an error: ``` module.exports = (externalConfig = {}) => { ^ SyntaxError: Unexpected token = at exports.runInThisContext (vm.js:53:16) at Module._compile (module.js:374:25) at Object.Module._extensions..js (module.js:417:10) at Module.load (module.js:344:32) at Function.Module._load (module.js:301:12) at Module.require (module.js:354:17) at require (internal/module.js:12:17) at Object. (/usr/local/lib/node_modules/nelson.cli/node_modules/external-ip/index.js:2:18) at Module._compile (module.js:410:26) at Object.Module._extensions..js (module.js:417:10) ``` Your node version is outdated. Make sure you have node v.6.9.1 or higher installed on your machine. ### I upgraded nelson, but it's still the old version! Please refer to [upgrading](#upgrading) for a possible reason. ## Contributing ### Running your own entry node As the network grows, we will need more entry nodes. These "master" nodes serve as gates to the network for new nodes. They accept slightly more connections and do not actively connect to others. The entry nodes only share info about the nodes that have contacted them sometime in the past. You can run a master node by adding these options to Nelson: ``` --isMaster --epochInterval 180 --incomingMax 9 ``` The first value tells Nelson to run in "master" mode. The second decreases the epoch time so that the connected nodes are rotated faster, giving space to new nodes. The third increases the amount of accepted connections (since master nodes do not have active connections, the outgoingMax for masters does not do anything). You can contact the maintainer of this repo (http://www.twitter.com/RomanSemko) to get your node included here. An initiative for donations to entry nodes is under way. ## Authors * **Roman Semko** - *SemkoDev* - (https://github.com/romansemko) * **Vitaly Semko** - *SemkoDev* - (https://github.com/witwit) ## License This project is licensed under the ICS License - see the [LICENSE.md](LICENSE.md) file for details ================================================ FILE: config.ini.example ================================================ [nelson] cycleInterval = 60 epochInterval = 300 apiPort = 18600 port = 16600 IRIPort = 14265 TCPPort = 15600 UDPPort = 14600 dataPath = data/neighbors.db isMaster = false silent = false ; use automatic service to download latest initial nodes getNeighbors = https://raw.githubusercontent.com/SemkoDev/nelson.cli/master/ENTRYNODES ; or/and add as many initial nelson neighbors, as you like neighbors[] = mainnet.deviota.com/14600 neighbors[] = mainnet2.deviota.com/14600 neighbors[] = mainnet3.deviota.com/14600 ================================================ FILE: contrib/ansible-playbook/.gitignore ================================================ site.retry .*.swp ================================================ FILE: contrib/ansible-playbook/README.md ================================================ # IOTA Nelson (IRI) Fullnode Ansible Playbook This playbook will install IRI and Nelson As Docker containers. ## Requirements ### Operating System This playbook has been tested on: * Ubuntu 16.04 and 17.04 * CentOS 7.4 ### Software Dependencies **Note** Docker CE will be installed by the playbook, it is not strictly required to install it before running the playbook. * Docker CE For Ubuntu: https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/ For CentOS: https://docs.docker.com/engine/installation/linux/docker-ce/centos/ * Ansible >= 2.4 To install Ansible: **Ubuntu**: ```sh apt-get upgrade -y && apt-get clean && apt-get update -y && apt-get install software-properties-common -y && apt-add-repository ppa:ansible/ansible -y && apt-get update -y && apt-get install ansible -y ``` **CentOS**: ```sh yum install ansible -y ``` #### Consideration Consider to run the playbook within a screen session. Should the SSH connection drop, the playbook's session will remain active. Ensure `screen` is installed: **Ubuntu**: ```sh apt-get install screen -y ``` **CentOS**: ```sh yum install screen -y ``` Then use `screen -S nelson` to create a session and run the next commands within. To detach from the session, press `CTRL-A` and `d`. To reattach to a session `screen -r nelson` or `screen -D -r nelson` if the screen is still attached. Use `exit` or `CTRL-D` within the session to end the session. ## Configuration If you want to configure values before running the playbook you will find the variables in the files under: ```sh group_vars/all/*.yml ``` ## Installation Run: ```sh ansible-playbook -i inventory -v site.yml ``` Specifc roles and or tasks can be run individually or skipped using `--tags=tag_name_a,tag_name_b` or `--skip-tags=tag_name`. ## Controls To start, stop or view status of either `nelson` or `iri` run: ```sh systemctl status iri ``` Replace the service name or command as required. ## Logs To view the logs of either `nelson` or `iri` run: ```sh journalctl -u iri ``` Use `shift-g` to scroll to the bottom. Alterntively, to avoid using the pager: ```sh journalctl -u nelson --no-pager -n50 ``` This command will display the last 50 lines of the log. You can use `-f` to follow the tail of the log. ## File Locations * Nelson's configuration is at `/etc/nelson/config.ini` * IRI config is at `/etc/iri/iri.ini` * Nelson's data directory is at `/var/lib/nelson/` * IRI's database is at `/var/lib/iri/` ================================================ FILE: contrib/ansible-playbook/group_vars/all/common.yml ================================================ # source: https://github.com/geerlingguy/ansible-role-docker/blob/dd0c6e0f8ee3aa5a9638d5318593fc80f2eaacfb/defaults/main.yml # Edition can be'ce' (Community Edition) or 'ee' (Enterprise Edition). docker_edition: 'ce' docker_package: "docker-{{ docker_edition }}" docker_package_state: present # Used only for Debian/Ubuntu. Switch 'stable' to 'edge' if needed. docker_apt_release_channel: stable docker_apt_repository: "deb https://download.docker.com/linux/{{ ansible_distribution|lower }} {{ ansible_distribution_release }} {{ docker_apt_release_channel }}" # Used only for RedHat/CentOS. docker_yum_repo_url: https://download.docker.com/linux/centos/docker-{{ docker_edition }}.repo docker_yum_repo_enable_edge: 0 docker_yum_repo_enable_test: 0 # Globals ssh_port: 22 ================================================ FILE: contrib/ansible-playbook/group_vars/all/iri.yml ================================================ # Unprivileged user to run iri with iri_username: iri # Base directory where iri is installed and runs from iri_basedir: /var/lib/iri # IRI configuration dir iri_configdir: /etc/iri # IRI docker image iri_image: nuriel77/iri-image # IRI Docker image tag iri_tag: latest # The TCP port on which IRI listens for API calls and allows for light wallets to connect to iri_api_port: 14265 # The UDP neighbor peering port iri_udp_port: 14600 # The TCP neighbor peering port iri_tcp_port: 14600 # Limit IRI memory usage iri_java_mem: 4096m # Initial memory usage iri_init_java_mem: 768m # Automatically configure memory limits # Overrides above `iri_java_mem` value memory_autoset: true # Let the iri_api_port bind to all interfaces (0.0.0.0). # `true` is necessary in order to allow external wallets/APIs to connect without tunneling. # If set to `false` it will only bind to localhost (127.0.0.1) # If setting to `true`, make sure you use the `iri_remote_limit_api` to limit what users can do. api_port_remote: false ================================================ FILE: contrib/ansible-playbook/group_vars/all/nelson.yml ================================================ # Nelson docker image tag to run nelson_tag: latest # Nelson image name nelson_image: romansemko/nelson # User name under which to run nelson nelson_username: nelson # Nelson configuration directory nelson_configdir: /etc/nelson # Nelson data dir nelson_datadir: /var/lib/nelson # IRI host on which to bind to IRI nelson_iri_host: 127.0.0.1 # Nelson bind API to this address nelson_bind_address: 127.0.0.1 # Nelson API port nelson_api_port: 18600 # Nelson communication TCP port nelson_tcp_port: 16600 ================================================ FILE: contrib/ansible-playbook/inventory ================================================ [fullnode] # Here the host on which to run the playbook on. # If using a remote host use either IP or a resolvable name. # Also, remove the ansible_connection=local if a remote host. localhost ansible_connection=local ================================================ FILE: contrib/ansible-playbook/roles/common/handlers/main.yml ================================================ - name: reload systemd sudo: yes command: systemctl daemon-reload ================================================ FILE: contrib/ansible-playbook/roles/common/tasks/firewalld.yml ================================================ - name: ensure firewalld started and enabled systemd: name: firewalld state: started enabled: yes - name: ensure selinux enabled selinux: policy: targeted state: enforcing register: selinux_enabled - name: check selinux not disabled shell: "getenforce" changed_when: false register: getenforce - name: exit and notify reboot required if selinux got enabled block: - name: exit and notify reboot required to get selinux enabled debug: msg: "** NOTE *** Selinux was disabled on this host. It has now been enabled. Please reboot this host `shutdown -r now` and re-run this playbook." - meta: end_play when: selinux_enabled is defined and selinux_enabled.changed - name: exit and notify selinux not enabled block: - name: exit and notify selinux not enabled debug: msg: > ** NOTE ** The system might require a reboot to get selinux enabled. Check /etc/sysconfig/selinux if selinux is `enforcing`. If it is, the host needs to be rebooted `shutdown -r now`. Refusing to continue. - meta: end_play when: "getenforce is defined and 'stdout' in getenforce and 'Disabled' in getenforce.stdout" ================================================ FILE: contrib/ansible-playbook/roles/common/tasks/install.yml ================================================ - name: Install Docker package: name: "{{ docker_package }}" state: "{{ docker_package_state }}" - name: Ensure Docker is started and enabled service: name: docker state: started enabled: yes ================================================ FILE: contrib/ansible-playbook/roles/common/tasks/main.yml ================================================ - import_tasks: role.yml tags: - common_role ================================================ FILE: contrib/ansible-playbook/roles/common/tasks/role.yml ================================================ - import_tasks: setup_apt.yml tags: - common_setup_apt when: ansible_distribution == 'Ubuntu' - import_tasks: setup_yum.yml tags: - common_setup_yum when: ansible_distribution == 'CentOS' - import_tasks: install.yml tags: - common_install - import_tasks: setup_pip.yml tags: - common_setup_pip - import_tasks: firewalld.yml tags: - common_firewalld when: ansible_distribution == 'CentOS' - import_tasks: ufw.yml tags: - common_ufw when: ansible_distribution == 'Ubuntu' ================================================ FILE: contrib/ansible-playbook/roles/common/tasks/setup_apt.yml ================================================ # Source: https://github.com/geerlingguy/ansible-role-docker/blob/master/tasks/setup-Debian.yml - name: Ensure depdencies are installed apt: name: "{{ item }}" state: present with_items: - apt-transport-https - ca-certificates - jq - ufw - wget - lsof - curl - pv - python-pip - name: Add Docker apt key apt_key: url: https://download.docker.com/linux/ubuntu/gpg id: 9DC858229FC7DD38854AE2D88D81803C0EBFCD88 state: present register: add_repository_key ignore_errors: true - name: Ensure curl is present (on older systems without SNI). package: name=curl state=present when: add_repository_key|failed - name: Add Docker apt key (alternative for older systems without SNI). shell: "curl -sSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -" args: warn: no when: add_repository_key|failed - name: Add Docker repository. apt_repository: repo: "{{ docker_apt_repository }}" state: present update_cache: yes ================================================ FILE: contrib/ansible-playbook/roles/common/tasks/setup_pip.yml ================================================ - name: install python deps via pip pip: name: docker-py ================================================ FILE: contrib/ansible-playbook/roles/common/tasks/setup_yum.yml ================================================ # Required - name: Install epel-release yum: state=latest name=epel-release - name: Install some packages yum: state=latest name={{ item }} with_items: - policycoreutils-python - firewalld - curl - wget - screen - lsof - jq - pv - python-pip - name: Add Docker GPG key rpm_key: key: https://download.docker.com/linux/centos/gpg state: present - name: Add Docker repository. get_url: url: "{{ docker_yum_repo_url }}" dest: '/etc/yum.repos.d/docker-{{ docker_edition }}.repo' owner: root group: root mode: 0644 - name: Configure Docker Edge repo. ini_file: dest: '/etc/yum.repos.d/docker-{{ docker_edition }}.repo' section: 'docker-{{ docker_edition }}-edge' option: enabled value: '{{ docker_yum_repo_enable_edge }}' - name: Configure Docker Test repo. ini_file: dest: '/etc/yum.repos.d/docker-{{ docker_edition }}.repo' section: 'docker-{{ docker_edition }}-test' option: enabled value: '{{ docker_yum_repo_enable_test }}' ================================================ FILE: contrib/ansible-playbook/roles/common/tasks/ufw.yml ================================================ - name: allow ssh port firewall ufw: rule: allow direction: in proto: tcp port: "{{ ssh_port }}" - name: ufw default outgoing policy allowed ufw: direction: outgoing policy: allow - name: ensure ufw started and default incoming policy denied ufw: state: enabled direction: incoming policy: deny ================================================ FILE: contrib/ansible-playbook/roles/iri/files/iri.service ================================================ [Unit] Description=IRI Fullnode Docker Container Requires=docker.service After=docker.service [Service] Restart=on-failure RestartSec=10 ExecStart=/usr/bin/docker start -a %p ExecStop=-/usr/bin/docker stop -t 2 %p [Install] WantedBy=multi-user.target ================================================ FILE: contrib/ansible-playbook/roles/iri/files/nbctl ================================================ #!/usr/bin/env python import argparse import urllib2 import json import sys """Script to add or remove neighbors from IRI API. source: https://github.com/nuriel77/iri-playbook """ def parse_args(): parser = argparse.ArgumentParser( description='Add or remove full node neighbors.', epilog='Example: nbctl -a -n' ' udp://1.2.3.4:12345 -n tcp://4.3.2.1:4321') parser.add_argument('--neighbors', '-n', action='append', required=True, help='Neighbors to process. Can be specified' ' multiple times.') parser.add_argument('--remove', '-r', action='store_true', help='Removes neighbors,') parser.add_argument('--add', '-a', action='store_true', help='Add neighbors') parser.add_argument('--host', '-i', default='http://localhost:14265', help='IRI API endpoint. Default: %(default)s') parser.add_argument('--api-version', '-x', default='1.4', help='IRI API Version. Default: %(default)s') return parser.parse_args() def run(): try: args = parse_args() except Exception as e: sys.stderr.write("Error parsing arguments: %s\n" % e) sys.exit(1) if args.add and args.remove: sys.stderr.write("You can either select `--add` or `--remove`" ", not both.\n") sys.exit(1) elif not args.add and not args.remove: sys.stderr.write("You must select either `--add` or `--remove`\n") sys.exit(1) command = 'addNeighbors' if args.add else 'removeNeighbors' headers = { 'content-type': 'application/json', 'X-IOTA-API-Version': args.api_version } to_send = json.dumps({ 'command': command, 'uris': args.neighbors }) request = urllib2.Request(url=args.host, data=to_send, headers=headers) return_data = urllib2.urlopen(request).read() json_data = json.loads(return_data) print(json_data) if __name__ == "__main__": run() ================================================ FILE: contrib/ansible-playbook/roles/iri/files/ps_mem ================================================ #!/usr/bin/env python # Try to determine how much RAM is currently being used per program. # Note per _program_, not per process. So for example this script # will report RAM used by all httpd process together. In detail it reports: # sum(private RAM for program processes) + sum(Shared RAM for program processes) # The shared RAM is problematic to calculate, and this script automatically # selects the most accurate method available for your kernel. # Licence: LGPLv2 # Author: P@draigBrady.com # Source: http://www.pixelbeat.org/scripts/ps_mem.py # V1.0 06 Jul 2005 Initial release # V1.1 11 Aug 2006 root permission required for accuracy # V1.2 08 Nov 2006 Add total to output # Use KiB,MiB,... for units rather than K,M,... # V1.3 22 Nov 2006 Ignore shared col from /proc/$pid/statm for # 2.6 kernels up to and including 2.6.9. # There it represented the total file backed extent # V1.4 23 Nov 2006 Remove total from output as it's meaningless # (the shared values overlap with other programs). # Display the shared column. This extra info is # useful, especially as it overlaps between programs. # V1.5 26 Mar 2007 Remove redundant recursion from human() # V1.6 05 Jun 2007 Also report number of processes with a given name. # Patch from riccardo.murri@gmail.com # V1.7 20 Sep 2007 Use PSS from /proc/$pid/smaps if available, which # fixes some over-estimation and allows totalling. # Enumerate the PIDs directly rather than using ps, # which fixes the possible race between reading # RSS with ps, and shared memory with this program. # Also we can show non truncated command names. # V1.8 28 Sep 2007 More accurate matching for stats in /proc/$pid/smaps # as otherwise could match libraries causing a crash. # Patch from patrice.bouchand.fedora@gmail.com # V1.9 20 Feb 2008 Fix invalid values reported when PSS is available. # Reported by Andrey Borzenkov # V3.9 07 Mar 2017 # http://github.com/pixelb/scripts/commits/master/scripts/ps_mem.py # Notes: # # All interpreted programs where the interpreter is started # by the shell or with env, will be merged to the interpreter # (as that's what's given to exec). For e.g. all python programs # starting with "#!/usr/bin/env python" will be grouped under python. # You can change this by using the full command line but that will # have the undesirable affect of splitting up programs started with # differing parameters (for e.g. mingetty tty[1-6]). # # For 2.6 kernels up to and including 2.6.13 and later 2.4 redhat kernels # (rmap vm without smaps) it can not be accurately determined how many pages # are shared between processes in general or within a program in our case: # http://lkml.org/lkml/2005/7/6/250 # A warning is printed if overestimation is possible. # In addition for 2.6 kernels up to 2.6.9 inclusive, the shared # value in /proc/$pid/statm is the total file-backed extent of a process. # We ignore that, introducing more overestimation, again printing a warning. # Since kernel 2.6.23-rc8-mm1 PSS is available in smaps, which allows # us to calculate a more accurate value for the total RAM used by programs. # # Programs that use CLONE_VM without CLONE_THREAD are discounted by assuming # they're the only programs that have the same /proc/$PID/smaps file for # each instance. This will fail if there are multiple real instances of a # program that then use CLONE_VM without CLONE_THREAD, or if a clone changes # its memory map while we're checksumming each /proc/$PID/smaps. # # I don't take account of memory allocated for a program # by other programs. For e.g. memory used in the X server for # a program could be determined, but is not. # # FreeBSD is supported if linprocfs is mounted at /compat/linux/proc/ # FreeBSD 8.0 supports up to a level of Linux 2.6.16 import getopt import time import errno import os import sys # The following exits cleanly on Ctrl-C or EPIPE # while treating other exceptions as before. def std_exceptions(etype, value, tb): sys.excepthook = sys.__excepthook__ if issubclass(etype, KeyboardInterrupt): pass elif issubclass(etype, IOError) and value.errno == errno.EPIPE: pass else: sys.__excepthook__(etype, value, tb) sys.excepthook = std_exceptions # # Define some global variables # PAGESIZE = os.sysconf("SC_PAGE_SIZE") / 1024 #KiB our_pid = os.getpid() have_pss = 0 have_swap_pss = 0 class Proc: def __init__(self): uname = os.uname() if uname[0] == "FreeBSD": self.proc = '/compat/linux/proc' else: self.proc = '/proc' def path(self, *args): return os.path.join(self.proc, *(str(a) for a in args)) def open(self, *args): try: if sys.version_info < (3,): return open(self.path(*args)) else: return open(self.path(*args), errors='ignore') except (IOError, OSError): val = sys.exc_info()[1] if (val.errno == errno.ENOENT or # kernel thread or process gone val.errno == errno.EPERM or val.errno == errno.EACCES): raise LookupError raise proc = Proc() # # Functions # def parse_options(): try: long_options = [ 'split-args', 'help', 'total', 'discriminate-by-pid', 'swap' ] opts, args = getopt.getopt(sys.argv[1:], "shtdSp:w:", long_options) except getopt.GetoptError: sys.stderr.write(help()) sys.exit(3) if len(args): sys.stderr.write("Extraneous arguments: %s\n" % args) sys.exit(3) # ps_mem.py options split_args = False pids_to_show = None discriminate_by_pid = False show_swap = False watch = None only_total = False for o, a in opts: if o in ('-s', '--split-args'): split_args = True if o in ('-t', '--total'): only_total = True if o in ('-d', '--discriminate-by-pid'): discriminate_by_pid = True if o in ('-S', '--swap'): show_swap = True if o in ('-h', '--help'): sys.stdout.write(help()) sys.exit(0) if o in ('-p',): try: pids_to_show = [int(x) for x in a.split(',')] except: sys.stderr.write(help()) sys.exit(3) if o in ('-w',): try: watch = int(a) except: sys.stderr.write(help()) sys.exit(3) return ( split_args, pids_to_show, watch, only_total, discriminate_by_pid, show_swap ) def help(): help_msg = 'Usage: ps_mem [OPTION]...\n' \ 'Show program core memory usage\n' \ '\n' \ ' -h, -help Show this help\n' \ ' -p [,pid2,...pidN] Only show memory usage PIDs in the '\ 'specified list\n' \ ' -s, --split-args Show and separate by, all command line'\ ' arguments\n' \ ' -t, --total Show only the total value\n' \ ' -d, --discriminate-by-pid Show by process rather than by program\n' \ ' -S, --swap Show swap information\n' \ ' -w Measure and show process memory every'\ ' N seconds\n' return help_msg # (major,minor,release) def kernel_ver(): kv = proc.open('sys/kernel/osrelease').readline().split(".")[:3] last = len(kv) if last == 2: kv.append('0') last -= 1 while last > 0: for char in "-_": kv[last] = kv[last].split(char)[0] try: int(kv[last]) except: kv[last] = 0 last -= 1 return (int(kv[0]), int(kv[1]), int(kv[2])) #return Private,Shared #Note shared is always a subset of rss (trs is not always) def getMemStats(pid): global have_pss global have_swap_pss mem_id = pid #unique Private_lines = [] Shared_lines = [] Pss_lines = [] Rss = (int(proc.open(pid, 'statm').readline().split()[1]) * PAGESIZE) Swap_lines = [] Swap_pss_lines = [] Swap = 0 Swap_pss = 0 if os.path.exists(proc.path(pid, 'smaps')): # stat lines = proc.open(pid, 'smaps').readlines() # open # Note we checksum smaps as maps is usually but # not always different for separate processes. mem_id = hash(''.join(lines)) for line in lines: if line.startswith("Shared"): Shared_lines.append(line) elif line.startswith("Private"): Private_lines.append(line) elif line.startswith("Pss"): have_pss = 1 Pss_lines.append(line) elif line.startswith("Swap:"): Swap_lines.append(line) elif line.startswith("SwapPss:"): have_swap_pss = 1 Swap_pss_lines.append(line) Shared = sum([int(line.split()[1]) for line in Shared_lines]) Private = sum([int(line.split()[1]) for line in Private_lines]) #Note Shared + Private = Rss above #The Rss in smaps includes video card mem etc. if have_pss: pss_adjust = 0.5 # add 0.5KiB as this avg error due to truncation Pss = sum([float(line.split()[1])+pss_adjust for line in Pss_lines]) Shared = Pss - Private # Note that Swap = Private swap + Shared swap. Swap = sum([int(line.split()[1]) for line in Swap_lines]) if have_swap_pss: # The kernel supports SwapPss, that shows proportional swap share. # Note that Swap - SwapPss is not Private Swap. Swap_pss = sum([int(line.split()[1]) for line in Swap_pss_lines]) elif (2,6,1) <= kernel_ver() <= (2,6,9): Shared = 0 #lots of overestimation, but what can we do? Private = Rss else: Shared = int(proc.open(pid, 'statm').readline().split()[2]) Shared *= PAGESIZE Private = Rss - Shared return (Private, Shared, mem_id, Swap, Swap_pss) def getCmdName(pid, split_args, discriminate_by_pid): cmdline = proc.open(pid, 'cmdline').read().split("\0") if cmdline[-1] == '' and len(cmdline) > 1: cmdline = cmdline[:-1] path = proc.path(pid, 'exe') try: path = os.readlink(path) # Some symlink targets were seen to contain NULs on RHEL 5 at least # https://github.com/pixelb/scripts/pull/10, so take string up to NUL path = path.split('\0')[0] except OSError: val = sys.exc_info()[1] if (val.errno == errno.ENOENT or # either kernel thread or process gone val.errno == errno.EPERM or val.errno == errno.EACCES): raise LookupError raise if split_args: return ' '.join(cmdline).replace('\n', ' ') if path.endswith(" (deleted)"): path = path[:-10] if os.path.exists(path): path += " [updated]" else: #The path could be have prelink stuff so try cmdline #which might have the full path present. This helped for: #/usr/libexec/notification-area-applet.#prelink#.fX7LCT (deleted) if os.path.exists(cmdline[0]): path = cmdline[0] + " [updated]" else: path += " [deleted]" exe = os.path.basename(path) cmd = proc.open(pid, 'status').readline()[6:-1] if exe.startswith(cmd): cmd = exe #show non truncated version #Note because we show the non truncated name #one can have separated programs as follows: #584.0 KiB + 1.0 MiB = 1.6 MiB mozilla-thunder (exe -> bash) # 56.0 MiB + 22.2 MiB = 78.2 MiB mozilla-thunderbird-bin if sys.version_info >= (3,): cmd = cmd.encode(errors='replace').decode() if discriminate_by_pid: cmd = '%s [%d]' % (cmd, pid) return cmd #The following matches "du -h" output #see also human.py def human(num, power="Ki", units=None): if units is None: powers = ["Ki", "Mi", "Gi", "Ti"] while num >= 1000: #4 digits num /= 1024.0 power = powers[powers.index(power)+1] return "%.1f %sB" % (num, power) else: return "%.f" % ((num * 1024) / units) def cmd_with_count(cmd, count): if count > 1: return "%s (%u)" % (cmd, count) else: return cmd #Warn of possible inaccuracies #2 = accurate & can total #1 = accurate only considering each process in isolation #0 = some shared mem not reported #-1= all shared mem not reported def shared_val_accuracy(): """http://wiki.apache.org/spamassassin/TopSharedMemoryBug""" kv = kernel_ver() pid = os.getpid() if kv[:2] == (2,4): if proc.open('meminfo').read().find("Inact_") == -1: return 1 return 0 elif kv[:2] == (2,6): if os.path.exists(proc.path(pid, 'smaps')): if proc.open(pid, 'smaps').read().find("Pss:")!=-1: return 2 else: return 1 if (2,6,1) <= kv <= (2,6,9): return -1 return 0 elif kv[0] > 2 and os.path.exists(proc.path(pid, 'smaps')): return 2 else: return 1 def show_shared_val_accuracy( possible_inacc, only_total=False ): level = ("Warning","Error")[only_total] if possible_inacc == -1: sys.stderr.write( "%s: Shared memory is not reported by this system.\n" % level ) sys.stderr.write( "Values reported will be too large, and totals are not reported\n" ) elif possible_inacc == 0: sys.stderr.write( "%s: Shared memory is not reported accurately by this system.\n" % level ) sys.stderr.write( "Values reported could be too large, and totals are not reported\n" ) elif possible_inacc == 1: sys.stderr.write( "%s: Shared memory is slightly over-estimated by this system\n" "for each program, so totals are not reported.\n" % level ) sys.stderr.close() if only_total and possible_inacc != 2: sys.exit(1) def get_memory_usage(pids_to_show, split_args, discriminate_by_pid, include_self=False, only_self=False): cmds = {} shareds = {} mem_ids = {} count = {} swaps = {} shared_swaps = {} for pid in os.listdir(proc.path('')): if not pid.isdigit(): continue pid = int(pid) # Some filters if only_self and pid != our_pid: continue if pid == our_pid and not include_self: continue if pids_to_show is not None and pid not in pids_to_show: continue try: cmd = getCmdName(pid, split_args, discriminate_by_pid) except LookupError: #operation not permitted #kernel threads don't have exe links or #process gone continue try: private, shared, mem_id, swap, swap_pss = getMemStats(pid) except RuntimeError: continue #process gone if shareds.get(cmd): if have_pss: #add shared portion of PSS together shareds[cmd] += shared elif shareds[cmd] < shared: #just take largest shared val shareds[cmd] = shared else: shareds[cmd] = shared cmds[cmd] = cmds.setdefault(cmd, 0) + private if cmd in count: count[cmd] += 1 else: count[cmd] = 1 mem_ids.setdefault(cmd, {}).update({mem_id: None}) # Swap (overcounting for now...) swaps[cmd] = swaps.setdefault(cmd, 0) + swap if have_swap_pss: shared_swaps[cmd] = shared_swaps.setdefault(cmd, 0) + swap_pss else: shared_swaps[cmd] = 0 # Total swaped mem for each program total_swap = 0 # Total swaped shared mem for each program total_shared_swap = 0 # Add shared mem for each program total = 0 for cmd in cmds: cmd_count = count[cmd] if len(mem_ids[cmd]) == 1 and cmd_count > 1: # Assume this program is using CLONE_VM without CLONE_THREAD # so only account for one of the processes cmds[cmd] /= cmd_count if have_pss: shareds[cmd] /= cmd_count cmds[cmd] = cmds[cmd] + shareds[cmd] total += cmds[cmd] # valid if PSS available total_swap += swaps[cmd] if have_swap_pss: total_shared_swap += shared_swaps[cmd] sorted_cmds = sorted(cmds.items(), key=lambda x:x[1]) sorted_cmds = [x for x in sorted_cmds if x[1]] return sorted_cmds, shareds, count, total, swaps, shared_swaps, \ total_swap, total_shared_swap def print_header(show_swap, discriminate_by_pid): output_string = " Private + Shared = RAM used" if show_swap: if have_swap_pss: output_string += " " * 5 + "Shared Swap" output_string += " Swap used" output_string += "\tProgram" if discriminate_by_pid: output_string += "[pid]" output_string += "\n\n" sys.stdout.write(output_string) def print_memory_usage(sorted_cmds, shareds, count, total, swaps, total_swap, shared_swaps, total_shared_swap, show_swap): for cmd in sorted_cmds: output_string = "%9s + %9s = %9s" output_data = (human(cmd[1]-shareds[cmd[0]]), human(shareds[cmd[0]]), human(cmd[1])) if show_swap: if have_swap_pss: output_string += "\t%9s" output_data += (human(shared_swaps[cmd[0]]),) output_string += " %9s" output_data += (human(swaps[cmd[0]]),) output_string += "\t%s\n" output_data += (cmd_with_count(cmd[0], count[cmd[0]]),) sys.stdout.write(output_string % output_data) if have_pss: if show_swap: if have_swap_pss: sys.stdout.write("%s\n%s%9s%s%9s%s%9s\n%s\n" % ("-" * 61, " " * 24, human(total), " " * 7, human(total_shared_swap), " " * 3, human(total_swap), "=" * 61)) else: sys.stdout.write("%s\n%s%9s%s%9s\n%s\n" % ("-" * 45, " " * 24, human(total), " " * 3, human(total_swap), "=" * 45)) else: sys.stdout.write("%s\n%s%9s\n%s\n" % ("-" * 33, " " * 24, human(total), "=" * 33)) def verify_environment(pids_to_show): if os.geteuid() != 0 and not pids_to_show: sys.stderr.write("Sorry, root permission required, or specify pids with -p\n") sys.stderr.close() sys.exit(1) try: kernel_ver() except (IOError, OSError): val = sys.exc_info()[1] if val.errno == errno.ENOENT: sys.stderr.write( "Couldn't access " + proc.path('') + "\n" "Only GNU/Linux and FreeBSD (with linprocfs) are supported\n") sys.exit(2) else: raise def main(): split_args, pids_to_show, watch, only_total, discriminate_by_pid, \ show_swap = parse_options() verify_environment(pids_to_show) if not only_total: print_header(show_swap, discriminate_by_pid) if watch is not None: try: sorted_cmds = True while sorted_cmds: sorted_cmds, shareds, count, total, swaps, shared_swaps, \ total_swap, total_shared_swap = \ get_memory_usage(pids_to_show, split_args, discriminate_by_pid) if only_total and have_pss: sys.stdout.write(human(total, units=1)+'\n') elif not only_total: print_memory_usage(sorted_cmds, shareds, count, total, swaps, total_swap, shared_swaps, total_shared_swap, show_swap) sys.stdout.flush() time.sleep(watch) else: sys.stdout.write('Process does not exist anymore.\n') except KeyboardInterrupt: pass else: # This is the default behavior sorted_cmds, shareds, count, total, swaps, shared_swaps, total_swap, \ total_shared_swap = get_memory_usage(pids_to_show, split_args, discriminate_by_pid) if only_total and have_pss: sys.stdout.write(human(total, units=1)+'\n') elif not only_total: print_memory_usage(sorted_cmds, shareds, count, total, swaps, total_swap, shared_swaps, total_shared_swap, show_swap) # We must close explicitly, so that any EPIPE exception # is handled by our excepthook, rather than the default # one which is reenabled after this script finishes. sys.stdout.close() vm_accuracy = shared_val_accuracy() show_shared_val_accuracy( vm_accuracy, only_total ) if __name__ == '__main__': main() ================================================ FILE: contrib/ansible-playbook/roles/iri/files/reattach ================================================ #!/usr/bin/env python # coding=utf-8 import sys import argparse from iota import * """Script to reattach a transaction to tangle source: https://github.com/nuriel77/iri-playbook """ def parse_args(): parser = argparse.ArgumentParser( description='Reattach a transaction.', epilog='Example: ./reattach -x TXHASH' ' -i http://localhost:14265 -m 14 -d 2') parser.add_argument('--txhash', '-x', type=str, required=True, help='Transaction Hash to reattach') parser.add_argument('--depth', '-d', type=int, default=2, help='Depth. Default: %(default)s') parser.add_argument('--magnitute', '-m', type=int, default=14, help='Minimum Weight Magnitute') parser.add_argument('--host', '-i', type=str, default='http://localhost:14265', help='IRI API endpoint. Default: %(default)s') return parser.parse_args() def run(): try: args = parse_args() except Exception as e: sys.stderr.write("Error parsing arguments: %s\n" % e) sys.exit(1) tx_hash = args.txhash min_mag = args.magnitute depth = args.depth api_uri = args.host # Create the API instance. api = Iota(api_uri) replayed = api.replay_bundle(tx_hash, depth, min_mag) print(replayed) if __name__ == "__main__": run() ================================================ FILE: contrib/ansible-playbook/roles/iri/handlers/main.yml ================================================ - name: restart iri systemd: name: iri.service state: restarted enabled: yes ================================================ FILE: contrib/ansible-playbook/roles/iri/tasks/firewalld.yml ================================================ - name: allow iri tcp port in firewall firewalld: port: "{{ iri_tcp_port }}/tcp" permanent: true state: enabled immediate: yes - name: allow iri udp port in firewall firewalld: port: "{{ iri_udp_port }}/udp" permanent: true state: enabled immediate: yes - name: allow iri api port in firewall firewalld: port: "{{ iri_api_port }}/tcp" permanent: true state: enabled immediate: yes when: api_port_remote is defined and api_port_remote ================================================ FILE: contrib/ansible-playbook/roles/iri/tasks/iri.yml ================================================ - name: set variables centos/redhat set_fact: systemd_dir: /usr/lib/systemd/system config_dir: /etc/sysconfig when: ansible_distribution == 'CentOS' - name: set variables debian/ubuntu set_fact: systemd_dir: /lib/systemd/system config_dir: /etc/default when: ansible_distribution == 'Ubuntu' # TODO: set 0.8 to be configured in iri.yml variable file - name: set memory limit for java-iri set_fact: iri_java_mem: "{{ (ansible_memtotal_mb * 0.8)|int|abs }}m" when: memory_autoset is defined and memory_autoset tags: - mem_override - name: add user to run iri as user: name: "{{ iri_username }}" shell: /sbin/nologin createhome: no home: "{{ iri_basedir }}" tags: - iri_user - name: get iri user uid shell: "echo -n $(id -u {{ iri_username }})" changed_when: false register: iri_uid - name: delete iri data basedir block: - name: stop iri systemd: name: iri.service state: stopped false_when: false - name: remove basedir file: path: "{{ iri_basedir }}" state: absent when: remove_iri_basedir is defined and remove_iri_basedir - name: ensure iri basedir ownership and permissions file: path: "{{ iri_basedir }}" state: directory mode: 0700 owner: "{{ iri_username }}" group: "{{ iri_username }}" - name: ensure iri basedir ownership and permissions file: path: "{{ iri_basedir }}" state: directory mode: 0700 owner: "{{ iri_username }}" group: "{{ iri_username }}" - name: ensure iri config dir exists file: path: "{{ iri_configdir }}" state: directory mode: 0700 owner: "{{ iri_username }}" group: "{{ iri_username }}" - name: copy utility scripts copy: src: "files/{{ item }}" dest: "/usr/bin/{{ item }}" mode: 0755 with_items: - nbctl - reattach - ps_mem tags: - scripts - name: copy iri systemd file template: src: files/iri.service dest: "{{ systemd_dir }}/iri.service" notify: - reload systemd - name: copy iri config file template: src: templates/iri.ini dest: "{{ iri_configdir }}/iri.ini" owner: "{{ iri_username }}" group: "{{ iri_username }}" force: no notify: - restart iri - name: Create iri container docker_container: name: iri state: present user: "{{ iri_uid.stdout }}" restart_policy: unless-stopped network_mode: host hostname: iri image: "{{ iri_image }}:{{ iri_tag }}" ports: - "{{ iri_api_port }}:{{ iri_api_port }}" - "{{ iri_tcp_port }}:{{ iri_tcp_port }}" - "{{ iri_udp_port }}:{{ iri_udp_port }}/udp" volumes: - "{{ iri_basedir }}:/iri/target:Z" - "{{ iri_configdir }}:/iri/conf:ro,Z" env: IRI_CONFIG: "/iri/conf/iri.ini" INIT_MEMORY: "{{ iri_init_java_mem }}" MAX_MEMORY: "{{ iri_java_mem }}" tags: - iri_create_container notify: - restart iri - name: flush handlers meta: flush_handlers - name: ensure iri started and enabled systemd: daemon_reload: true name: iri.service state: started enabled: true ================================================ FILE: contrib/ansible-playbook/roles/iri/tasks/main.yml ================================================ - import_tasks: role.yml tags: - iri_role ================================================ FILE: contrib/ansible-playbook/roles/iri/tasks/role.yml ================================================ - import_tasks: ufw.yml tags: - iri_ufw when: ansible_distribution == 'Ubuntu' - import_tasks: firewalld.yml tags: - iri_firewalld when: ansible_distribution == 'CentOS' - import_tasks: iri.yml tags: - iri_install ================================================ FILE: contrib/ansible-playbook/roles/iri/tasks/ufw.yml ================================================ - name: allow iri tcp port in firewall ufw: rule: allow direction: in proto: tcp port: "{{ iri_tcp_port }}" - name: allow iri udp port in firewall ufw: rule: allow direction: in proto: udp port: "{{ iri_udp_port }}" - name: allow iri api port in firewall ufw: rule: allow direction: in proto: tcp port: "{{ iri_api_port }}" when: api_port_remote is defined and api_port_remote ================================================ FILE: contrib/ansible-playbook/roles/iri/templates/iri.ini ================================================ [IRI] PORT = {{ iri_api_port }} UDP_RECEIVER_PORT = {{ iri_udp_port }} TCP_RECEIVER_PORT = {{ iri_tcp_port }} IXI_DIR = ixi HEADLESS = true DEBUG = false DB_PATH = /iri/target NEIGHBORS = udp://my.favorite.com:15600 REMOTE_LIMIT_API = "removeNeighbors, addNeighbors, interruptAttachingToTangle, attachToTangle, getNeighbors, setApiRateLimit" {% if api_port_remote is defined and api_port_remote %}API_HOST = 0.0.0.0{% endif %} # Uncommend this line and set user and password # in the format: `user:password` to password protect # the IRI API. Change requires restart of iri. # If enabled, the API will have to be called using # basic auth. For example, with curl: # curl http://user:password@localhost:14265 ... #REMOTE_AUTH = iota:password # set max requests value #MAX_REQUESTS_LIST = 9999 # set max find transactions value #MAX_FIND_TRANSACTIONS = 100000 ================================================ FILE: contrib/ansible-playbook/roles/nelson/files/nelson.service ================================================ [Unit] Description=Nelson Docker Container Requires=docker.service After=docker.service [Service] Restart=on-failure RestartSec=10 ExecStart=/usr/bin/docker start -a %p ExecStop=-/usr/bin/docker stop -t 2 %p [Install] WantedBy=multi-user.target ================================================ FILE: contrib/ansible-playbook/roles/nelson/handlers/main.yml ================================================ - name: restart nelson systemd: name: nelson.service enabled: true state: restarted ================================================ FILE: contrib/ansible-playbook/roles/nelson/tasks/firewall.yml ================================================ - name: allow nelson tcp port in firewall firewalld: port: "{{ nelson_tcp_port }}/tcp" permanent: true state: enabled immediate: yes when: ansible_distribution == 'CentOS' - name: allow nelson tcp port in firewall ufw: rule: allow direction: in proto: tcp port: "{{ nelson_tcp_port }}" when: ansible_distribution == 'Ubuntu' ================================================ FILE: contrib/ansible-playbook/roles/nelson/tasks/main.yml ================================================ - import_tasks: role.yml tags: - nelson_role ================================================ FILE: contrib/ansible-playbook/roles/nelson/tasks/nelson.yml ================================================ - name: set variables centos/redhat set_fact: systemd_dir: /usr/lib/systemd/system config_dir: /etc/sysconfig when: ansible_distribution == 'CentOS' - name: set variables debian/ubuntu set_fact: systemd_dir: /lib/systemd/system config_dir: /etc/default when: ansible_distribution == 'Ubuntu' - name: add nelson user user: name: "{{ nelson_username }}" shell: /sbin/nologin createhome: no home: "{{ nelson_datadir }}" tags: - nelson_user - name: get nelson user uid shell: "echo -n $(id -u {{ nelson_username }})" changed_when: false register: nelson_uid - name: create nelson data and config directory file: path: "{{ item }}" state: directory owner: "{{ nelson_username }}" group: "{{ nelson_username }}" mode: 0700 with_items: - "{{ nelson_configdir }}" - "{{ nelson_datadir }}" - name: copy nelson config template: src: "templates/config.ini.j2" dest: "{{ nelson_configdir }}/config.ini" owner: "{{ nelson_username }}" group: "{{ nelson_username }}" mode: 0600 force: no notify: - restart nelson - name: copy systemd service file template: src: files/nelson.service dest: "{{ systemd_dir }}/nelson.service" notify: - reload systemd - name: Create nelson container docker_container: name: nelson state: present user: "{{ nelson_uid.stdout }}" restart_policy: unless-stopped network_mode: host hostname: nelson image: "{{ nelson_image }}:{{ nelson_tag }}" ports: - "{{ nelson_api_port }}:{{ nelson_api_port }}" - "{{ nelson_tcp_port }}:{{ nelson_tcp_port }}" volumes: - "{{ nelson_datadir }}:/var/lib/nelson/data:Z" - "{{ nelson_configdir }}:/etc/nelson:ro,Z" env: NELSON_CONFIG: "{{ nelson_configdir }}/config.ini" tags: - nelson_create_container notify: - restart nelson - name: ensure nelson started and enabled systemd: name: nelson.service state: started enabled: yes ================================================ FILE: contrib/ansible-playbook/roles/nelson/tasks/role.yml ================================================ - import_tasks: firewall.yml tags: - nelson_firewall - import_tasks: nelson.yml tags: - nelson_install ================================================ FILE: contrib/ansible-playbook/roles/nelson/templates/config.ini.j2 ================================================ [nelson] cycleInterval = 60 epochInterval = 300 apiPort = {{ nelson_api_port }} apiHostname = {{ nelson_iri_host }} port = {{ nelson_tcp_port }} IRIHostname = {{ nelson_iri_host }} IRIProtocol = any IRIPort = {{ iri_api_port }} TCPPort = {{ iri_tcp_port }} UDPPort = {{ iri_udp_port }} dataPath = {{ nelson_datadir }}/data/neighbors.db ; maximal incoming connections. Please do not set below this limit: incomingMax = 5 ; maximal outgoing connections. Only set below this limit, if you have trusted, manual neighbors: outgoingMax = 4 isMaster = false silent = false gui = false ; use automatic service to download latest initial nodes getNeighbors = https://raw.githubusercontent.com/SemkoDev/nelson.cli/master/ENTRYNODES ; add as many initial nelson neighbors, as you like neighbors[] = mainnet.deviota.com/16600 neighbors[] = mainnet2.deviota.com/16600 neighbors[] = mainnet3.deviota.com/16600 neighbors[] = iotairi.tt-tec.net/16600 ================================================ FILE: contrib/ansible-playbook/site.yml ================================================ --- - hosts: fullnode gather_facts: true become: true become_method: sudo roles: - common - iri - nelson ================================================ FILE: dist/api/index.js ================================================ 'use strict'; var _extends = Object.assign || function (target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) { if (Object.prototype.hasOwnProperty.call(source, key)) { target[key] = source[key]; } } } return target; }; function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; } var express = require('express'); var helmet = require('helmet'); var bodyParser = require('body-parser'); var basicAuth = require('express-basic-auth'); var _require = require('./node'), getNodeStats = _require.getNodeStats, getSummary = _require.getSummary; var _require2 = require('./peer'), getPeerStats = _require2.getPeerStats; var _require3 = require('./webhooks'), startWebhooks = _require3.startWebhooks; var DEFAULT_OPTIONS = { node: null, webhooks: [], webhookInterval: 30, username: null, password: null, apiPort: 18600, apiHostname: '127.0.0.1' }; /** * Creates an Express APP instance, also starts regular webhooks callbacks. * @param options * @returns {*|Function} */ function createAPI(options) { var opts = _extends({}, DEFAULT_OPTIONS, options); // Start webhook callbacks if (opts.webhooks && opts.webhooks.length) { startWebhooks(opts.node, opts.webhooks, opts.webhookInterval); } // Start API server var app = express(); app.set('node', opts.node); // Basic app protection app.use(helmet()); // Enable basic HTTP Auth if (opts.username && opts.password) { app.use(basicAuth({ users: _defineProperty({}, opts.username, opts.password) })); } // parse application/x-www-form-urlencoded app.use(bodyParser.urlencoded({ extended: false })); // parse application/json app.use(bodyParser.json()); //////////////////////// ENDPOINTS //////////////////////// app.get('/', function (req, res) { res.json(getNodeStats(opts.node)); }); app.get('/peer-stats', function (req, res) { res.json(getSummary(opts.node)); }); app.get('/peers', function (req, res) { res.json(opts.node.list.all().map(getPeerStats)); }); return app.listen(opts.apiPort, opts.apiHostname); } module.exports = { createAPI: createAPI, DEFAULT_OPTIONS: DEFAULT_OPTIONS }; ================================================ FILE: dist/api/node.js ================================================ 'use strict'; var _require = require('./peer'), getPeerStats = _require.getPeerStats; var version = require('../../package.json').version; /** * Returns summary of the node stats * @param {Node} node * @returns {{newNodes: {hourAgo, fourAgo, twelveAgo, dayAgo, weekAgo}, activeNodes: {hourAgo, fourAgo, twelveAgo, dayAgo, weekAgo}}} */ function getSummary(node) { var now = new Date(); var hour = 3600000; var hourAgo = new Date(now - hour); var fourAgo = new Date(now - hour * 4); var twelveAgo = new Date(now - hour * 12); var dayAgo = new Date(now - hour * 24); var weekAgo = new Date(now - hour * 24 * 7); return { newNodes: { hourAgo: node.list.all().filter(function (p) { return p.data.dateCreated >= hourAgo; }).length, fourAgo: node.list.all().filter(function (p) { return p.data.dateCreated >= fourAgo; }).length, twelveAgo: node.list.all().filter(function (p) { return p.data.dateCreated >= twelveAgo; }).length, dayAgo: node.list.all().filter(function (p) { return p.data.dateCreated >= dayAgo; }).length, weekAgo: node.list.all().filter(function (p) { return p.data.dateCreated >= weekAgo; }).length }, activeNodes: { hourAgo: node.list.all().filter(function (p) { return p.data.dateLastConnected >= hourAgo; }).length, fourAgo: node.list.all().filter(function (p) { return p.data.dateLastConnected >= fourAgo; }).length, twelveAgo: node.list.all().filter(function (p) { return p.data.dateLastConnected >= twelveAgo; }).length, dayAgo: node.list.all().filter(function (p) { return p.data.dateLastConnected >= dayAgo; }).length, weekAgo: node.list.all().filter(function (p) { return p.data.dateLastConnected >= weekAgo; }).length } }; } /** * Returns clean node stats to be used in the API * @param {Node} node * @returns {{name, version, ready: (boolean|*|null), isIRIHealthy: (*|boolean), iriStats: *, peerStats: {newNodes: {hourAgo, fourAgo, twelveAgo, dayAgo, weekAgo}, activeNodes: {hourAgo, fourAgo, twelveAgo, dayAgo, weekAgo}}, totalPeers, connectedPeers: Array, config: {cycleInterval: (Command.opts.cycleInterval|*), epochInterval: (Command.opts.epochInterval|*), beatInterval: (Command.opts.beatInterval|*), dataPath: (Command.opts.dataPath|*), port: (Command.opts.port|*), apiPort: (Command.opts.apiPort|*), IRIPort: (Command.opts.IRIPort|*), TCPPort: (Command.opts.TCPPort|*), UDPPort: (Command.opts.UDPPort|*), IRIProtocol: (Command.opts.IRIProtocol|*), isMaster: (Command.opts.isMaster|*), temporary: (Command.opts.temporary|*)}, heart: {lastCycle: (heart.lastCycle|Heart.lastCycle|_require2.Heart.lastCycle), lastEpoch: (heart.lastEpoch|Heart.lastEpoch|_require2.Heart.lastEpoch), personality: (heart.personality|Heart.personality|_require2.Heart.personality), currentCycle: (heart.currentCycle|Heart.currentCycle|_require2.Heart.currentCycle), currentEpoch: (heart.currentEpoch|Heart.currentEpoch|_require2.Heart.currentEpoch), startDate: (heart.startDate|Heart.startDate|_require2.Heart.startDate)}}} */ function getNodeStats(node) { var _node$opts = node.opts, cycleInterval = _node$opts.cycleInterval, epochInterval = _node$opts.epochInterval, beatInterval = _node$opts.beatInterval, dataPath = _node$opts.dataPath, port = _node$opts.port, apiPort = _node$opts.apiPort, IRIPort = _node$opts.IRIPort, TCPPort = _node$opts.TCPPort, UDPPort = _node$opts.UDPPort, isMaster = _node$opts.isMaster, IRIProtocol = _node$opts.IRIProtocol, temporary = _node$opts.temporary; var _node$heart = node.heart, lastCycle = _node$heart.lastCycle, lastEpoch = _node$heart.lastEpoch, personality = _node$heart.personality, currentCycle = _node$heart.currentCycle, currentEpoch = _node$heart.currentEpoch, startDate = _node$heart.startDate; var totalPeers = node.list.all().length; var isIRIHealthy = node.iri && node.iri.isHealthy; var iriStats = node.iri && node.iri.iriStats; var connectedPeers = Array.from(node.sockets.keys()).filter(function (p) { return node.sockets.get(p).readyState === 1; }).map(getPeerStats); return { name: node.opts.name, version: version, ready: node._ready, isIRIHealthy: isIRIHealthy, iriStats: iriStats, peerStats: getSummary(node), totalPeers: totalPeers, connectedPeers: connectedPeers, config: { cycleInterval: cycleInterval, epochInterval: epochInterval, beatInterval: beatInterval, dataPath: dataPath, port: port, apiPort: apiPort, IRIPort: IRIPort, TCPPort: TCPPort, UDPPort: UDPPort, IRIProtocol: IRIProtocol, isMaster: isMaster, temporary: temporary }, heart: { lastCycle: lastCycle, lastEpoch: lastEpoch, personality: personality, currentCycle: currentCycle, currentEpoch: currentEpoch, startDate: startDate } }; } module.exports = { getSummary: getSummary, getNodeStats: getNodeStats }; ================================================ FILE: dist/api/peer.js ================================================ "use strict"; /** * Returns a clean Peer object that can be used in the API * @param {Peer} peer * @returns {{name, hostname, ip, port, TCPPort, UDPPort, protocol, IRIProtocol, seen, connected, tried, weight, dateTried, dateLastConnected, dateCreated, isTrusted, lastConnections}} */ function getPeerStats(peer) { var _peer$data = peer.data, name = _peer$data.name, hostname = _peer$data.hostname, ip = _peer$data.ip, port = _peer$data.port, TCPPort = _peer$data.TCPPort, UDPPort = _peer$data.UDPPort, protocol = _peer$data.protocol, seen = _peer$data.seen, connected = _peer$data.connected, tried = _peer$data.tried, weight = _peer$data.weight, dateTried = _peer$data.dateTried, dateLastConnected = _peer$data.dateLastConnected, dateCreated = _peer$data.dateCreated, IRIProtocol = _peer$data.IRIProtocol, isTrusted = _peer$data.isTrusted, lastConnections = _peer$data.lastConnections; return { name: name, hostname: hostname, ip: ip, port: port, TCPPort: TCPPort, UDPPort: UDPPort, protocol: protocol, IRIProtocol: IRIProtocol, seen: seen, connected: connected, tried: tried, weight: weight, dateTried: dateTried, dateLastConnected: dateLastConnected, dateCreated: dateCreated, isTrusted: isTrusted, lastConnections: lastConnections }; } module.exports = { getPeerStats: getPeerStats }; ================================================ FILE: dist/api/utils.js ================================================ "use strict"; ================================================ FILE: dist/api/webhooks.js ================================================ 'use strict'; var request = require('request'); var _require = require('./node'), getNodeStats = _require.getNodeStats; function startWebhooks(node, webhooks, webhookInterval) { var interval = setInterval(function () { webhooks.forEach(function (uri) { return request({ uri: uri, method: 'POST', json: getNodeStats(node) }, function (err) { if (err) { node.log(('Webhook returned error: ' + uri).yellow); } }); }); }, webhookInterval * 1000); return { stop: function stop() { clearInterval(interval); } }; } module.exports = { startWebhooks: startWebhooks }; ================================================ FILE: dist/index.js ================================================ 'use strict'; var _extends = Object.assign || function (target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) { if (Object.prototype.hasOwnProperty.call(source, key)) { target[key] = source[key]; } } } return target; }; function _toConsumableArray(arr) { if (Array.isArray(arr)) { for (var i = 0, arr2 = Array(arr.length); i < arr.length; i++) { arr2[i] = arr[i]; } return arr2; } else { return Array.from(arr); } } require('colors'); var request = require('request'); var terminal = require('./node/tools/terminal'); var node = require('./node').node; var api = require('./api/index'); var utils = require('./node').utils; // Some general TODOs: // TODO: add linting // TODO: add editor config module.exports = _extends({ initNode: function initNode(opts) { var init = function init(options) { var _node = new node.Node(options); var terminate = function terminate() { return _node.end().then(function () { process.exit(0); }); }; process.on('SIGINT', terminate); process.on('SIGTERM', terminate); opts.gui && terminal.init(opts.name, utils.getVersion(), terminate); _node.start().then(function (n) { api.createAPI({ node: n, webhooks: opts.webhooks, webhookInterval: opts.webhookInterval, apiPort: opts.apiPort, apiHostname: opts.apiHostname, username: opts.apiAuth && opts.apiAuth.username, password: opts.apiAuth && opts.apiAuth.password }); terminal.ports(n.opts); n.log(('Nelson v.' + utils.getVersion() + ' initialized').green.bold); }); }; if (opts.getNeighbors) { if (typeof opts.getNeighbors === 'boolean') { opts.getNeighbors = 'https://raw.githubusercontent.com/SemkoDev/nelson.cli/master/ENTRYNODES'; } var neighbors = []; request(opts.getNeighbors, function (err, resp, body) { if (err) { throw err; } neighbors = body.split('\n').map(function (str) { if (!str || !str.length) { return null; } if (utils.validNeighbor(str)) { console.log('Downloaded entry neighbor:', str); return str; } else { console.log('Wrong entry neighbor format:', str); return null; } }).filter(function (n) { return n; }); opts.neighbors = [].concat(_toConsumableArray(opts.neighbors ? opts.neighbors : []), _toConsumableArray(neighbors)); init(opts); }); } else { init(opts); } } }, node); ================================================ FILE: dist/nelson.js ================================================ #!/usr/bin/env node 'use strict'; require('colors'); var ini = require('ini'); var fs = require('fs'); var _require = require('url'), URL = _require.URL; var program = require('commander'); var _require2 = require('./index'), initNode = _require2.initNode; var _require3 = require('./node/node'), DEFAULT_OPTIONS = _require3.DEFAULT_OPTIONS; var _require4 = require('./node/peer'), PROTOCOLS = _require4.PROTOCOLS; var _require5 = require('./node/peer-list'), DEFAULT_LIST_OPTIONS = _require5.DEFAULT_OPTIONS; var _require6 = require('./api/index'), DEFAULT_API_OPTIONS = _require6.DEFAULT_OPTIONS; var version = require('../package.json').version; var parseNeighbors = function parseNeighbors(val) { return val.split(' '); }; var parseURLs = function parseURLs(val) { return val.split(' ').map(function (v) { return new URL(v); }).map(function (u) { return u.href; }); }; var parseProtocol = function parseProtocol(val) { var lower = val.toLowerCase(); return PROTOCOLS.includes(lower) ? lower : DEFAULT_OPTIONS.IRIProtocol; }; var parseNumber = function parseNumber(v) { return parseInt(v); }; var parseAuth = function parseAuth(v) { var tokens = v.split(':'); if (!tokens.length === 2) { throw new Error('Wrong apiAuth format! Use: "username.password"'); } if (!tokens[0].length) { throw new Error('apiAuth username not provided!'); } if (!tokens[1].length) { throw new Error('apiAuth password not provided!'); } return { username: tokens[0], password: tokens[1] }; }; program.version(version).option('--name [value]', 'Name of your node instance', DEFAULT_OPTIONS.name).option('-n, --neighbors [value]', 'Trusted neighbors', parseNeighbors, []).option('--getNeighbors [url]', 'Download default set of neighbors', false).option('-c, --cycleInterval [value]', 'Cycle interval in seconds', parseNumber, DEFAULT_OPTIONS.cycleInterval).option('-e, --epochInterval [value]', 'Epoch interval in seconds', parseNumber, DEFAULT_OPTIONS.epochInterval).option('--incomingMax [value]', 'Maximal incoming connection slots', parseNumber, DEFAULT_OPTIONS.incomingMax).option('--outgoingMax [value]', 'Maximal outgoing connection slots', parseNumber, DEFAULT_OPTIONS.outgoingMax).option('--lazyLimit [value]', 'Seconds after which neighbor is dropped for not having provided any new TXs', parseNumber, DEFAULT_OPTIONS.lazyLimit).option('--lazyTimesLimit [value]', 'How many consecutive times a lazy neighbor can connect before getting penalized', parseNumber, DEFAULT_OPTIONS.lazyTimesLimit).option('--apiAuth [value]', 'Nelson API username:password', parseAuth, null).option('-a, --apiPort [value]', 'Nelson API port', parseNumber, DEFAULT_API_OPTIONS.apiPort).option('-o, --apiHostname [value]', 'Nelson API hostname', DEFAULT_API_OPTIONS.apiHostname).option('-w, --webhooks [value]', 'Nelson API webhook URLs', parseURLs, DEFAULT_API_OPTIONS.webhooks).option('--webhookInterval [value]', 'Webhooks callback interval in seconds', parseNumber, DEFAULT_API_OPTIONS.webhookInterval).option('-p, --port [value]', 'Nelson port', parseNumber, DEFAULT_OPTIONS.port).option('-r, --IRIHostname [value]', 'IRI API hostname', DEFAULT_OPTIONS.IRIHostname).option('-i, --IRIPort [value]', 'IRI API port', parseNumber, DEFAULT_OPTIONS.IRIPort).option('-t, --TCPPort [value]', 'IRI TCP port', parseNumber, DEFAULT_OPTIONS.TCPPort).option('-u, --UDPPort [value]', 'IRI UDP port', parseNumber, DEFAULT_OPTIONS.UDPPort).option('--IRIProtocol [value]', 'IRI protocol to use: udp, tcp, prefertcp, preferudp or any', parseProtocol, DEFAULT_OPTIONS.IRIProtocol).option('-d, --dataPath [value]', 'Peer database path', DEFAULT_LIST_OPTIONS.dataPath).option('-m, --isMaster [value]', 'Is a master node', false).option('-s, --silent [value]', 'Silent', false).option('-g, --gui [value]', 'GUI', false).option('--temporary [value]', 'Create a temporary node', false).option('--config [value]', 'Config file path', null).parse(process.argv); var configPath = process.env.NELSON_CONFIG || program.config; initNode(configPath ? ini.parse(fs.readFileSync(configPath, 'utf-8')).nelson : program); ================================================ FILE: dist/node/__mocks__/iri.js ================================================ 'use strict'; var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }(); function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } function _possibleConstructorReturn(self, call) { if (!self) { throw new ReferenceError("this hasn't been initialised - super() hasn't been called"); } return call && (typeof call === "object" || typeof call === "function") ? call : self; } function _inherits(subClass, superClass) { if (typeof superClass !== "function" && superClass !== null) { throw new TypeError("Super expression must either be null or a function, not " + typeof superClass); } subClass.prototype = Object.create(superClass && superClass.prototype, { constructor: { value: subClass, enumerable: false, writable: true, configurable: true } }); if (superClass) Object.setPrototypeOf ? Object.setPrototypeOf(subClass, superClass) : subClass.__proto__ = superClass; } var req = require.requireActual ? require.requireActual : require; var _req = req('../iri'), BaseIRI = _req.IRI, DEFAULT_OPTIONS = _req.DEFAULT_OPTIONS; /** * Class responsible to RUN and communicate with local IRI instance * @class */ var IRI = function (_BaseIRI) { _inherits(IRI, _BaseIRI); function IRI() { _classCallCheck(this, IRI); return _possibleConstructorReturn(this, (IRI.__proto__ || Object.getPrototypeOf(IRI)).apply(this, arguments)); } _createClass(IRI, [{ key: 'start', /** * Starts the IRI process, returning self on success. * @returns {Promise} */ value: function start() { var _this2 = this; return new Promise(function (resolve) { _this2._isStarted = true; _this2.isHealthy = true; _this2.ticker = setInterval(_this2._tick, 15000); _this2.getStats().then(function () { return resolve(_this2); }); }); } /** * Removes a list of neighbors from IRI, except static neighbors. Returns list of removed peers. * @param {Peer[]} peers * @returns {Promise} */ }, { key: 'removeNeighbors', value: function removeNeighbors(peers) { if (!this.isAvailable()) { return Promise.reject(); } return new Promise(function (resolve) { resolve(peers); }); } /** * Adds a list of peers to IRI. * @param {Peer[]} peers * @returns {Promise} */ }, { key: 'addNeighbors', value: function addNeighbors(peers) { if (!this.isAvailable()) { return Promise.reject(); } return new Promise(function (resolve) { resolve(peers); }); } /** * Cleans up any orphans from the IRI * @param {Peer[]} peers * @returns {Promise} */ }, { key: 'cleanupNeighbors', value: function cleanupNeighbors(peers) { if (!this.isAvailable()) { return Promise.reject(); } return new Promise(function (resolve) { resolve([]); }); } /** * Updates the list of neighbors at the IRI backend. Removes all neighbors, replacing them with * the newly provided neighbors. * @param {Peer[]} peers * @returns {Promise} */ }, { key: 'updateNeighbors', value: function updateNeighbors(peers) { var _this3 = this; if (!this.isAvailable()) { return Promise.reject(); } if (!peers || !peers.length) { return Promise.resolve([]); } return new Promise(function (resolve, reject) { var addNeighbors = function addNeighbors() { _this3.addNeighbors(peers).then(resolve).catch(reject); }; addNeighbors(); }); } /** * Removes all IRI neighbors, except static neighbors. * @returns {Promise} */ }, { key: 'removeAllNeighbors', value: function removeAllNeighbors() { if (!this.isAvailable()) { return Promise.reject(); } return new Promise(function (resolve) { resolve(); }); } /** * Returns IRI node info * @returns {Promise} */ }, { key: 'getStats', value: function getStats() { var _this4 = this; return new Promise(function (resolve) { _this4.iriStats = { mock: true }; resolve(_this4.iriStats); }); } }, { key: '_tick', value: function _tick() { var onHealthCheck = this.opts.onHealthCheck; this.getStats().then(function () { onHealthCheck(true, []); }); } }]); return IRI; }(BaseIRI); IRI.isMocked = true; module.exports = { IRI: IRI, DEFAULT_OPTIONS: DEFAULT_OPTIONS }; ================================================ FILE: dist/node/__mocks__/node.js ================================================ 'use strict'; var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }(); var _extends = Object.assign || function (target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) { if (Object.prototype.hasOwnProperty.call(source, key)) { target[key] = source[key]; } } } return target; }; function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } function _possibleConstructorReturn(self, call) { if (!self) { throw new ReferenceError("this hasn't been initialised - super() hasn't been called"); } return call && (typeof call === "object" || typeof call === "function") ? call : self; } function _inherits(subClass, superClass) { if (typeof superClass !== "function" && superClass !== null) { throw new TypeError("Super expression must either be null or a function, not " + typeof superClass); } subClass.prototype = Object.create(superClass && superClass.prototype, { constructor: { value: subClass, enumerable: false, writable: true, configurable: true } }); if (superClass) Object.setPrototypeOf ? Object.setPrototypeOf(subClass, superClass) : subClass.__proto__ = superClass; } var _require = require('../node'), BaseNode = _require.Node, DEFAULT_NODE_OPTIONS = _require.DEFAULT_OPTIONS; var _require2 = require('../tools/utils'), getRandomInt = _require2.getRandomInt; var _require3 = require('./iri'), IRI = _require3.IRI; var DEFAULT_OPTIONS = _extends({}, DEFAULT_NODE_OPTIONS, { localNodes: true, beatInterval: 2, cycleInterval: 3, epochInterval: 30, lazyLimit: 6, testnet: true, temporary: true }); /** * This is a mock for the "real" node. What it does are several things: * * 1. Mock away IRI backend so we do not start it. We just want to test the P2P functionality. * 2. Create a separate neighbor database for each node. * 3. Report stats to the parent process * * @class Node */ var Node = function (_BaseNode) { _inherits(Node, _BaseNode); function Node(options) { _classCallCheck(this, Node); var _this = _possibleConstructorReturn(this, (Node.__proto__ || Object.getPrototypeOf(Node)).call(this, _extends({}, DEFAULT_OPTIONS, options))); _this.sendStats = _this.sendStats.bind(_this); setInterval(_this.sendStats, 1000); return _this; } _createClass(Node, [{ key: '_getIRI', value: function _getIRI() { var _this2 = this; var _opts = this.opts, APIPort = _opts.APIPort, TCPPort = _opts.TCPPort, UDPPort = _opts.UDPPort, testnet = _opts.testnet, silent = _opts.silent, temporary = _opts.temporary; return new IRI({ APIPort: APIPort, TCPPort: TCPPort, UDPPort: UDPPort, testnet: testnet, silent: silent, temporary: temporary, logIdent: this.opts.port + '::IRI' }).start().then(function (iri) { _this2.iri = iri; return iri; }); } }, { key: '_setPublicIP', value: function _setPublicIP() { this.ipv4 = 'localhost'; return Promise.resolve(0); } }, { key: '_onIRIHealth', value: function _onIRIHealth() { Array.from(this.sockets.keys()).forEach(function (peer) { peer.updateConnection({ numberOfAllTransactions: getRandomInt(0, 1000), numberOfNewTransactions: getRandomInt(0, 150), numberOfRandomTransactionRequests: getRandomInt(0, 100), numberOfInvalidTransactions: getRandomInt(0, 10) }); }); } /////////////////////////////////// MOCK SPECIFICS /////////////////////////////////// }, { key: 'sendStats', value: function sendStats() { var _this3 = this; var sockets = Array.from(this.sockets.values()); process.send({ isMaster: this.opts.isMaster, peers: this.list ? this.list.all().map(function (p) { return p.data.port; }) : [], connections: { list: Array.from(this.sockets.keys()).filter(function (k) { return _this3.sockets.get(k).readyState === 1; }).map(function (peer) { return '' + peer.data.port; }), connecting: sockets.filter(function (s) { return s.readyState === 0; }).length, connected: sockets.filter(function (s) { return s.readyState === 1; }).length, closed: sockets.filter(function (s) { return s.readyState > 1; }).length } }); } }]); return Node; }(BaseNode); module.exports = { DEFAULT_OPTIONS: DEFAULT_OPTIONS, Node: Node }; ================================================ FILE: dist/node/base.js ================================================ 'use strict'; var _extends = Object.assign || function (target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) { if (Object.prototype.hasOwnProperty.call(source, key)) { target[key] = source[key]; } } } return target; }; var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }(); function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } require('colors'); var terminal = require('./tools/terminal'); var DEFAULT_OPTIONS = { silent: false, logIdent: 'BASE', logIdentWidth: 12 }; /** * Base class with generic functionality. * @class Base */ var Base = function () { function Base(options) { _classCallCheck(this, Base); this.opts = _extends({}, DEFAULT_OPTIONS, options); } _createClass(Base, [{ key: 'log', value: function log() { if (!this.opts || !this.opts.silent || arguments[0] === '!!') { var date = new Date(); var timeString = (date.toLocaleTimeString() + '.' + this.formatMilliseconds(date.getMilliseconds())).dim; var space = this.opts.logIdent.length > this.opts.logIdentWidth ? '\n' + ' '.repeat(this.opts.logIdentWidth) : ' '.repeat(this.opts.logIdentWidth - this.opts.logIdent.length); var logIdent = ('' + this.opts.logIdent + space).dim.bold; terminal.log.apply(terminal, [timeString + '\t' + logIdent].concat(Array.prototype.slice.call(arguments))); } } }, { key: 'formatNode', value: function formatNode(hostname, port) { return (hostname + ':' + port).cyan; } }, { key: 'formatMilliseconds', value: function formatMilliseconds(milliseconds) { var formatted = milliseconds / 1000; formatted = formatted.toFixed(3); formatted = formatted.toString(); return formatted.slice(2); } }, { key: 'start', value: function start() {} }, { key: 'end', value: function end() {} }]); return Base; }(); module.exports = { DEFAULT_OPTIONS: DEFAULT_OPTIONS, Base: Base }; ================================================ FILE: dist/node/guard.js ================================================ 'use strict'; var _extends = Object.assign || function (target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) { if (Object.prototype.hasOwnProperty.call(source, key)) { target[key] = source[key]; } } } return target; }; var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }(); function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } function _possibleConstructorReturn(self, call) { if (!self) { throw new ReferenceError("this hasn't been initialised - super() hasn't been called"); } return call && (typeof call === "object" || typeof call === "function") ? call : self; } function _inherits(subClass, superClass) { if (typeof superClass !== "function" && superClass !== null) { throw new TypeError("Super expression must either be null or a function, not " + typeof superClass); } subClass.prototype = Object.create(superClass && superClass.prototype, { constructor: { value: subClass, enumerable: false, writable: true, configurable: true } }); if (superClass) Object.setPrototypeOf ? Object.setPrototypeOf(subClass, superClass) : subClass.__proto__ = superClass; } var _require = require('./base'), Base = _require.Base; var _require2 = require('./tools/utils'), getSecondsPassed = _require2.getSecondsPassed; var DEFAULT_OPTIONS = { beatInterval: 1, throttleInterval: 2, // Minimal amount of beats to pass until a remote address is allowed again. localNodes: false, logIdent: 'GUARD' }; /** * Simple throttling system for incoming connections. * @class Heart */ var Guard = function (_Base) { _inherits(Guard, _Base); function Guard(options) { _classCallCheck(this, Guard); var _this = _possibleConstructorReturn(this, (Guard.__proto__ || Object.getPrototypeOf(Guard)).call(this, _extends({}, DEFAULT_OPTIONS, options))); _this.requests = {}; return _this; } _createClass(Guard, [{ key: 'isAllowed', value: function isAllowed(address, port) { var target = '' + (this.opts.localNodes ? port : address); if (!this.requests[target]) { this.requests[target] = new Date(); return true; } else { var allowed = getSecondsPassed(this.requests[target]) >= this.opts.beatInterval * this.opts.throttleInterval; this.requests[target] = new Date(); return allowed; } } }]); return Guard; }(Base); module.exports = { DEFAULT_OPTIONS: DEFAULT_OPTIONS, Guard: Guard }; ================================================ FILE: dist/node/heart.js ================================================ 'use strict'; var _extends = Object.assign || function (target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) { if (Object.prototype.hasOwnProperty.call(source, key)) { target[key] = source[key]; } } } return target; }; var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }(); function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } function _possibleConstructorReturn(self, call) { if (!self) { throw new ReferenceError("this hasn't been initialised - super() hasn't been called"); } return call && (typeof call === "object" || typeof call === "function") ? call : self; } function _inherits(subClass, superClass) { if (typeof superClass !== "function" && superClass !== null) { throw new TypeError("Super expression must either be null or a function, not " + typeof superClass); } subClass.prototype = Object.create(superClass && superClass.prototype, { constructor: { value: subClass, enumerable: false, writable: true, configurable: true } }); if (superClass) Object.setPrototypeOf ? Object.setPrototypeOf(subClass, superClass) : subClass.__proto__ = superClass; } var _require = require('./base'), Base = _require.Base; var _require2 = require('./tools/utils'), getSecondsPassed = _require2.getSecondsPassed, getRandomInt = _require2.getRandomInt, createIdentifier = _require2.createIdentifier; var terminal = require('./tools/terminal'); var DEFAULT_OPTIONS = { cycleInterval: 300, epochInterval: 900, beatInterval: 1, autoStart: false, logIdent: 'HEART', onEpoch: function onEpoch(currentEpoch) { return Promise.resolve(false); }, onCycle: function onCycle(currentCycle) { return Promise.resolve(false); }, onTick: function onTick(currentCycle) { return Promise.resolve(0); } }; /** * Manages epoch and cycle updates * @class Heart */ var Heart = function (_Base) { _inherits(Heart, _Base); function Heart(options) { _classCallCheck(this, Heart); var _this = _possibleConstructorReturn(this, (Heart.__proto__ || Object.getPrototypeOf(Heart)).call(this, _extends({}, DEFAULT_OPTIONS, options))); _this.id = null; _this.ticker = null; _this.lastCycle = null; _this.lastEpoch = null; _this.personality = {}; _this.currentCycle = 0; _this.currentEpoch = 0; _this.startDate = null; _this._tick = _this._tick.bind(_this); _this.opts.autoStart && _this.start(); return _this; } _createClass(Heart, [{ key: 'start', value: function start() { this.startDate = new Date(); this.startNewEpoch(); this.lastCycle = new Date(); this.log('Cycle/epoch intervals:', this.opts.cycleInterval, this.opts.epochInterval); terminal.settings({ epochInterval: this.opts.epochInterval, cycleInterval: this.opts.cycleInterval, startDate: this.startDate }); this._tick(); } }, { key: 'end', value: function end() { this.ticker && clearTimeout(this.ticker); } /** * Starts new epoch, resetting node identifiers and memorizing last epoch switch datetime. */ }, { key: 'startNewEpoch', value: function startNewEpoch() { this.setNewPersonality(); this.lastEpoch = new Date(); this.currentEpoch += 1; } /** * Sets this heart's personality: ID, feature, etc. */ }, { key: 'setNewPersonality', value: function setNewPersonality() { var id = createIdentifier(); this.personality = { id: id, publicId: id.slice(0, 8), feature: id[getRandomInt(0, id.length)] }; this.log('new personality', this.personality.feature, this.personality.id); } /** * Ticker that handles cycle and epoch changes. * @private */ }, { key: '_tick', value: function _tick() { var _this2 = this; this.opts.onTick(this.currentCycle).then(function () { var passedSecondsEpoch = getSecondsPassed(_this2.lastEpoch); var passedSecondsCycle = getSecondsPassed(_this2.lastCycle); var pctEpoch = passedSecondsEpoch / _this2.opts.epochInterval; var pctCycle = passedSecondsCycle / _this2.opts.cycleInterval; terminal.beat({ epoch: _this2.currentEpoch, cycle: _this2.currentCycle, startDate: _this2.startDate, pctEpoch: pctEpoch, pctCycle: pctCycle }); if (passedSecondsCycle > _this2.opts.cycleInterval) { _this2.opts.onCycle(_this2.currentCycle).then(function (skipABeat) { if (!skipABeat) { _this2.lastCycle = new Date(); _this2.currentCycle += 1; if (passedSecondsEpoch > _this2.opts.epochInterval) { _this2.opts.onEpoch(_this2.currentEpoch).then(function (skipAge) { !skipAge && _this2.startNewEpoch(); _this2._setTicker(); }); return; } } _this2._setTicker(); }); return; } _this2._setTicker(); }); } /** * Sets the ticker for the next beat * @private */ }, { key: '_setTicker', value: function _setTicker() { this.ticker && clearTimeout(this.ticker); this.ticker = setTimeout(this._tick, this.opts.beatInterval * 1000); } }]); return Heart; }(Base); module.exports = { DEFAULT_OPTIONS: DEFAULT_OPTIONS, Heart: Heart }; ================================================ FILE: dist/node/index.js ================================================ 'use strict'; var base = require('./base'); var heart = require('./heart'); var iri = require('./iri'); var node = require('./node'); var peer = require('./peer'); var peerList = require('./peer-list'); var utils = require('./tools/utils'); module.exports = { base: base, heart: heart, iri: iri, node: node, peer: peer, peerList: peerList, utils: utils }; ================================================ FILE: dist/node/iri.js ================================================ 'use strict'; var _extends = Object.assign || function (target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) { if (Object.prototype.hasOwnProperty.call(source, key)) { target[key] = source[key]; } } } return target; }; var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }(); function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } function _possibleConstructorReturn(self, call) { if (!self) { throw new ReferenceError("this hasn't been initialised - super() hasn't been called"); } return call && (typeof call === "object" || typeof call === "function") ? call : self; } function _inherits(subClass, superClass) { if (typeof superClass !== "function" && superClass !== null) { throw new TypeError("Super expression must either be null or a function, not " + typeof superClass); } subClass.prototype = Object.create(superClass && superClass.prototype, { constructor: { value: subClass, enumerable: false, writable: true, configurable: true } }); if (superClass) Object.setPrototypeOf ? Object.setPrototypeOf(subClass, superClass) : subClass.__proto__ = superClass; } var IOTA = require('iota.lib.js'); var _require = require('url'), URL = _require.URL; var tmp = require('tmp'); var _require2 = require('./base'), Base = _require2.Base; var _require3 = require('./tools/utils'), getIP = _require3.getIP; tmp.setGracefulCleanup(); var DEFAULT_OPTIONS = { hostname: 'localhost', port: 14265, TCPPort: 15600, UDPPort: 14600, logIdent: 'IRI', onHealthCheck: function onHealthCheck(isHealthy, neighbors) {} }; /** * Class responsible to RUN and communicate with local IRI instance * @class */ var IRI = function (_Base) { _inherits(IRI, _Base); function IRI(options) { _classCallCheck(this, IRI); var _this = _possibleConstructorReturn(this, (IRI.__proto__ || Object.getPrototypeOf(IRI)).call(this, _extends({}, DEFAULT_OPTIONS, options))); _this.api = new IOTA({ host: 'http://' + _this.opts.hostname, port: _this.opts.port }).api; _this.removeNeighbors = _this.removeNeighbors.bind(_this); _this.addNeighbors = _this.addNeighbors.bind(_this); _this.updateNeighbors = _this.updateNeighbors.bind(_this); _this._tick = _this._tick.bind(_this); _this._getIRIPeerURI = _this._getIRIPeerURI.bind(_this); _this.ticker = null; _this.isHealthy = false; _this.iriStats = {}; _this.staticNeighbors = []; return _this; } /** * Starts the IRI process, returning self on success. * @returns {Promise} */ _createClass(IRI, [{ key: 'start', value: function start() { var _this2 = this; return new Promise(function (resolve) { var getNodeInfo = function getNodeInfo() { return _this2.api.getNeighbors(function (error, neighbors) { if (!error) { var addresses = neighbors.map(function (n) { return new URL(n.connectionType + '://' + n.address).hostname; }); Promise.all(addresses.map(getIP)).then(function (ips) { _this2._isStarted = true; _this2.isHealthy = true; _this2.staticNeighbors = ips.concat(addresses); _this2.log('Static neighbors: ' + addresses); // TODO: make ticker wait for result, like in the heart. _this2.ticker = setInterval(_this2._tick, 15000); _this2.getStats().then(function () { return resolve(_this2); }); }); } else { _this2.log(('IRI not ready on ' + _this2.opts.hostname + ':' + _this2.opts.port + ', retrying...').yellow); setTimeout(getNodeInfo, 5000); } }); }; getNodeInfo(); }); } }, { key: 'end', value: function end() { this.isHealthy = false; this._isStarted = false; this.staticNeighbors = []; this.ticker && clearTimeout(this.ticker); this.ticker = null; } /** * Returns whether the process has been started. * @returns {boolean} */ }, { key: 'isStarted', value: function isStarted() { return this._isStarted; } /** * Returns whether the IRI process is running and can be communicated with. * @returns {boolean} */ }, { key: 'isAvailable', value: function isAvailable() { return this.isStarted() && this.isHealthy; } /** * Returns whether a peer's IP or hostname is added as static neighbor in IRI. * @param {Peer} peer * @returns {boolean} */ }, { key: 'isStaticNeighbor', value: function isStaticNeighbor(peer) { return !!this.staticNeighbors.filter(function (n) { return n === peer.data.ip || n === peer.data.hostname; }).length; } /** * Removes a list of neighbors from IRI, except static neighbors. Returns list of removed peers. * @param {Peer[]} peers * @returns {Promise} */ }, { key: 'removeNeighbors', value: function removeNeighbors(peers) { var _this3 = this; if (!this.isAvailable()) { return Promise.reject(); } var myPeers = peers.filter(function (peer) { if (_this3.isStaticNeighbor(peer)) { _this3.log(('WARNING: trying to remove a static neighbor. Skipping: ' + peer.data.hostname).yellow); return false; } return true; }); if (!peers.length) { return Promise.resolve([]); } var uris = myPeers.map(this._getIRIPeerURI); return new Promise(function (resolve, reject) { _this3.api.removeNeighbors(uris, function (err) { if (err) { reject(err); return; } _this3.log('Neighbors removed (if there were any):'.red, uris.join(', ')); resolve(peers); }); }); } /** * Adds a list of peers to IRI. * @param {Peer[]} peers * @returns {Promise} */ }, { key: 'addNeighbors', value: function addNeighbors(peers) { var _this4 = this; if (!this.isAvailable()) { return Promise.reject(); } var uris = peers.map(this._getIRIPeerURI); return new Promise(function (resolve, reject) { _this4.api.addNeighbors(uris, function (error) { if (error) { reject(error); return; } _this4.log('Neighbors added:'.green, uris.join(', ')); resolve(peers); }); }); } /** * Cleans up any orphans from the IRI * @param {Peer[]} peers * @returns {Promise} */ }, { key: 'cleanupNeighbors', value: function cleanupNeighbors(peers) { var _this5 = this; if (!this.isAvailable()) { return Promise.reject(); } return new Promise(function (resolve) { _this5.api.getNeighbors(function (error, neighbors) { if (error) { return resolve(); } Promise.all(neighbors.map(function (n) { var url = new URL(n.connectionType + '://' + n.address); return getIP(url.hostname).then(function (ip) { url.ip = ip || 'none'; return url; }); })).then(function (urls) { var toRemove = urls.filter(function (url) { return !_this5.staticNeighbors.includes(url.hostname) && !_this5.staticNeighbors.includes(url.ip) && peers.filter(function (p) { return p.data.hostname === url.hostname || p.data.ip === url.hostname || p.data.hostname === url.ip || p.data.ip === url.ip; }).length === 0; }); if (!toRemove.length) { return resolve(toRemove); } _this5.api.removeNeighbors(toRemove, function (err) { if (err) { reject(err); return; } _this5.log('Removed orphans:'.red, toRemove.map(function (url) { return url.hostname; })); resolve(toRemove); }); }); }); }); } /** * Updates the list of neighbors at the IRI backend. Removes all neighbors, replacing them with * the newly provided neighbors. * @param {Peer[]} peers * @returns {Promise} */ }, { key: 'updateNeighbors', value: function updateNeighbors(peers) { var _this6 = this; if (!this.isAvailable()) { return Promise.reject(); } if (!peers || !peers.length) { return Promise.resolve([]); } return new Promise(function (resolve, reject) { var addNeighbors = function addNeighbors() { _this6.addNeighbors(peers).then(resolve).catch(reject); }; _this6.api.getNeighbors(function (error, neighbors) { if (error) { reject(error); return; } Array.isArray(neighbors) && neighbors.length ? _this6.api.removeNeighbors(neighbors.map(function (n) { return n.connectionType + '://' + n.address; }), addNeighbors) : addNeighbors(); }); }); } /** * Removes all IRI neighbors, except static neighbors. * @returns {Promise} */ }, { key: 'removeAllNeighbors', value: function removeAllNeighbors() { var _this7 = this; if (!this.isAvailable()) { return Promise.reject(); } return new Promise(function (resolve) { _this7.api.getNeighbors(function (error, neighbors) { if (error) { return resolve(); } if (Array.isArray(neighbors) && neighbors.length) { // FIXME: This is broken. staticNeighbors is just a resolved IP. n.address includes port and can be a hostname. // Hence, the filter will always be true. var toRemove = neighbors.filter(function (n) { return !_this7.staticNeighbors.includes(n.address); }); return _this7.api.removeNeighbors(toRemove.map(function (n) { return n.connectionType + '://' + n.address; }), resolve); } resolve(); }); }); } /** * Returns IRI node info * @returns {Promise} */ }, { key: 'getStats', value: function getStats() { var _this8 = this; return new Promise(function (resolve, reject) { _this8.api.getNodeInfo(function (error, data) { if (error) { return reject(); } _this8.iriStats = data; resolve(data); }); }); } /** * Checks if the IRI instance is healthy, and its list of neighbors. Calls back the result to onHealthCheck. * @private */ }, { key: '_tick', value: function _tick() { var _this9 = this; var onHealthCheck = this.opts.onHealthCheck; var onError = function onError() { _this9.isHealthy = false; onHealthCheck(false); }; this.getStats().then(function () { _this9.api.getNeighbors(function (error, neighbors) { if (error) { return onError(); } _this9.isHealthy = true; // TODO: if the address is IPV6, could that pose a problem? onHealthCheck(true, neighbors.map(function (n) { return { address: new URL(n.connectionType + '://' + n.address).hostname, numberOfRandomTransactionRequests: n.numberOfRandomTransactionRequests, numberOfAllTransactions: n.numberOfAllTransactions, numberOfNewTransactions: n.numberOfNewTransactions, numberOfInvalidTransactions: n.numberOfInvalidTransactions }; })); }); }).catch(onError); } /** * Returns URI for IRI depending on the protocol. * @param {Peer} peer * @returns {string} * @private */ }, { key: '_getIRIPeerURI', value: function _getIRIPeerURI(peer) { return peer.data.IRIProtocol === 'tcp' ? peer.getTCPURI() : peer.getUDPURI(); } }]); return IRI; }(Base); module.exports = { DEFAULT_OPTIONS: DEFAULT_OPTIONS, IRI: IRI }; ================================================ FILE: dist/node/node.js ================================================ 'use strict'; var _extends = Object.assign || function (target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) { if (Object.prototype.hasOwnProperty.call(source, key)) { target[key] = source[key]; } } } return target; }; var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }(); function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } function _possibleConstructorReturn(self, call) { if (!self) { throw new ReferenceError("this hasn't been initialised - super() hasn't been called"); } return call && (typeof call === "object" || typeof call === "function") ? call : self; } function _inherits(subClass, superClass) { if (typeof superClass !== "function" && superClass !== null) { throw new TypeError("Super expression must either be null or a function, not " + typeof superClass); } subClass.prototype = Object.create(superClass && superClass.prototype, { constructor: { value: subClass, enumerable: false, writable: true, configurable: true } }); if (superClass) Object.setPrototypeOf ? Object.setPrototypeOf(subClass, superClass) : subClass.__proto__ = superClass; } var WebSocket = require('ws'); var ip = require('ip'); var pip = require('external-ip')(); var weighted = require('weighted'); var terminal = require('./tools/terminal'); var _require = require('./base'), Base = _require.Base; var _require2 = require('./heart'), Heart = _require2.Heart; var _require3 = require('./guard'), Guard = _require3.Guard; var _require4 = require('./iri'), IRI = _require4.IRI, DEFAULT_IRI_OPTIONS = _require4.DEFAULT_OPTIONS; var _require5 = require('./peer-list'), PeerList = _require5.PeerList, DEFAULT_LIST_OPTIONS = _require5.DEFAULT_OPTIONS; var _require6 = require('./tools/utils'), getPeerIdentifier = _require6.getPeerIdentifier, getRandomInt = _require6.getRandomInt, getSecondsPassed = _require6.getSecondsPassed, getVersion = _require6.getVersion, isSameMajorVersion = _require6.isSameMajorVersion, getIP = _require6.getIP, createIdentifier = _require6.createIdentifier; process.on('unhandledRejection', function (reason, p) { console.log('Unhandled Rejection at: Promise', p, 'reason:', reason); }); var DEFAULT_OPTIONS = { name: 'Deviota Nelson', cycleInterval: 60, epochInterval: 1200, beatInterval: 10, dataPath: DEFAULT_LIST_OPTIONS.dataPath, port: 16600, IRIHostname: DEFAULT_IRI_OPTIONS.hostname, IRIPort: DEFAULT_IRI_OPTIONS.port, IRIProtocol: 'any', TCPPort: DEFAULT_IRI_OPTIONS.TCPPort, UDPPort: DEFAULT_IRI_OPTIONS.UDPPort, weightDeflation: 0.95, incomingMax: 6, outgoingMax: 5, maxShareableNodes: 6, localNodes: false, isMaster: false, temporary: false, autoStart: false, logIdent: 'NODE', neighbors: [], lazyLimit: 300, // Time, after which a peer is considered lazy, if no new TXs received lazyTimesLimit: 3, // starts to penalize peer's quality if connected so many times without new TXs onReady: function onReady(node) {}, onPeerConnected: function onPeerConnected(peer) {}, onPeerRemoved: function onPeerRemoved(peer) {} }; // TODO: add node tests. Need to mock away IRI for this. var Node = function (_Base) { _inherits(Node, _Base); function Node(options) { _classCallCheck(this, Node); var _this = _possibleConstructorReturn(this, (Node.__proto__ || Object.getPrototypeOf(Node)).call(this, _extends({}, DEFAULT_OPTIONS, options))); _this.opts.logIdent = _this.opts.port + '::NODE'; _this._onCycle = _this._onCycle.bind(_this); _this._onEpoch = _this._onEpoch.bind(_this); _this._onTick = _this._onTick.bind(_this); _this._onIRIHealth = _this._onIRIHealth.bind(_this); _this._removeNeighbor = _this._removeNeighbor.bind(_this); _this._removeNeighbors = _this._removeNeighbors.bind(_this); _this._addNeighbor = _this._addNeighbor.bind(_this); _this._addNeighbors = _this._addNeighbors.bind(_this); _this.connectPeer = _this.connectPeer.bind(_this); _this.reconnectPeers = _this.reconnectPeers.bind(_this); _this.end = _this.end.bind(_this); _this._ready = false; _this.sockets = new Map(); _this.opts.autoStart && _this.start(); // Tries to fix the issue #45 https://github.com/SemkoDev/nelson.cli/issues/45 // Reasoning: https://github.com/request/request/issues/2161#issuecomment-313375694 // Also, cleans up nelson before crashing from the sky. process.on('uncaughtException', function (err) { if (err.code !== 'ECONNRESET') { _this.end().then(function () { throw err; }); } }); return _this; } /** * Starts the node server, getting public IP, IRI interface, Peer List and Heart. */ _createClass(Node, [{ key: 'start', value: function start() { var _this2 = this; var _opts = this.opts, cycleInterval = _opts.cycleInterval, epochInterval = _opts.epochInterval, beatInterval = _opts.beatInterval, silent = _opts.silent, localNodes = _opts.localNodes; this.guard = new Guard({ beatInterval: beatInterval, silent: silent, localNodes: localNodes }); return this._setPublicIP().then(function () { return _this2._getIRI().then(function (iri) { if (!iri) { throw new Error('IRI could not be started'); } if (!iri.staticNeighbors.length && _this2.opts.outgoingMax < DEFAULT_OPTIONS.outgoingMax) { _this2.log('WARNING: you have no static neighbors and outboundMax (' + _this2.opts.outgoingMax + ') is set below the advised limit (' + DEFAULT_OPTIONS.outgoingMax + ')!'); } if (_this2.opts.incomingMax < DEFAULT_OPTIONS.incomingMax) { _this2.log('WARNING: incomingMax (' + _this2.opts.incomingMax + ') is set below the advised limit (' + DEFAULT_OPTIONS.incomingMax + ')!'); } if (_this2.opts.incomingMax <= DEFAULT_OPTIONS.outgoingMax) { _this2.log('WARNING: incomingMax (' + _this2.opts.incomingMax + ') is set below outgoingMax (' + DEFAULT_OPTIONS.outgoingMax + ')!'); } return _this2._getList().then(function () { _this2._createServer(); _this2.heart = new Heart({ silent: silent, cycleInterval: cycleInterval, epochInterval: epochInterval, beatInterval: beatInterval, logIdent: _this2.opts.port + '::HEART', onCycle: _this2._onCycle, onTick: _this2._onTick, onEpoch: _this2._onEpoch }); _this2._ready = true; _this2.opts.onReady(_this2); _this2.heart.start(); return _this2; }).catch(function (err) { throw err; }); }).catch(function (err) { throw err; }); }); } /** * Ends the node, closing HTTP server and IRI backend. * @returns {Promise.} */ }, { key: 'end', value: function end() { var _this3 = this; this.log('terminating...'); this.heart && this.heart.end(); this._ready = false; var closeServer = function closeServer() { return new Promise(function (resolve) { if (_this3.server) { _this3.server.close(); } return _this3._removeNeighbors(Array.from(_this3.sockets.keys())).then(function () { _this3.sockets = new Map(); resolve(true); }); }); }; return closeServer().then(function () { return _this3.iri ? _this3.iri.end() : true; }); } /** * Sets a new peer list and returns a list of loaded peers. * @returns {Promise.} * @private */ }, { key: '_getList', value: function _getList() { var _this4 = this; var _opts2 = this.opts, localNodes = _opts2.localNodes, temporary = _opts2.temporary, silent = _opts2.silent, neighbors = _opts2.neighbors, dataPath = _opts2.dataPath, isMaster = _opts2.isMaster, lazyLimit = _opts2.lazyLimit, lazyTimesLimit = _opts2.lazyTimesLimit; this.list = new PeerList({ multiPort: localNodes, temporary: temporary, silent: silent, dataPath: dataPath, isMaster: isMaster, lazyLimit: lazyLimit, lazyTimesLimit: lazyTimesLimit, logIdent: this.opts.port + '::LIST' }); return this.list.load(neighbors.filter(function (n) { var tokens = n.split('/'); return !_this4.isMyself(tokens[0], tokens[1]); })); } /** * Sets and returns an IRI instance * @returns {Promise.} * @private */ }, { key: '_getIRI', value: function _getIRI() { var _this5 = this; var _opts3 = this.opts, IRIHostname = _opts3.IRIHostname, IRIPort = _opts3.IRIPort, silent = _opts3.silent; return new IRI({ logIdent: this.opts.port + '::IRI', hostname: IRIHostname, port: IRIPort, onHealthCheck: this._onIRIHealth, silent: silent }).start().then(function (iri) { _this5.iri = iri; return iri; }); } /** * Tries to get the public IPs of this node. * @private * @returns {Promise} */ }, { key: '_setPublicIP', value: function _setPublicIP() { var _this6 = this; if (this.opts.localNodes) { return Promise.resolve(0); } return new Promise(function (resolve) { pip(function (err, ip) { if (!err) { _this6.ipv4 = ip; resolve(0); } }); }); } /** * Creates HTTP server for Nelson * @private */ }, { key: '_createServer', value: function _createServer() { var _this7 = this; this.server = new WebSocket.Server({ port: this.opts.port, verifyClient: function verifyClient(info, cb) { var req = info.req; var deny = function deny() { return cb(false, 401); }; var accept = function accept() { return cb(true); }; _this7._canConnect(req).then(accept).catch(deny); } }); this.server.on('connection', function (ws, req) { _this7.log('incoming connection established'.green, req.connection.remoteAddress); var address = req.connection.remoteAddress; var _getHeaderIdentifiers2 = _this7._getHeaderIdentifiers(req.headers), port = _getHeaderIdentifiers2.port, TCPPort = _getHeaderIdentifiers2.TCPPort, UDPPort = _getHeaderIdentifiers2.UDPPort, remoteKey = _getHeaderIdentifiers2.remoteKey, name = _getHeaderIdentifiers2.name, protocol = _getHeaderIdentifiers2.protocol; var IRIProtocol = _this7._negotiateProtocol(protocol); _this7.list.add({ hostname: address, port: port, TCPPort: TCPPort, UDPPort: UDPPort, remoteKey: remoteKey, name: name, IRIProtocol: IRIProtocol }).then(function (peer) { _this7._bindWebSocket(ws, peer, true); }).catch(function (e) { _this7.log('Error binding/adding'.red, address, port, e); _this7.sockets.delete(Array.from(_this7.sockets.keys()).find(function (p) { return _this7.sockets.get(p) === ws; })); ws.close(); ws.terminate(); }); }); this.server.on('headers', function (headers) { var myHeaders = _this7._getHeaders(); Object.keys(myHeaders).forEach(function (key) { return headers.push(key + ': ' + myHeaders[key]); }); }); this.server.on('error', function (err) { // basically, do nothing. Most probably a ECONNRESET error. // The peer will be cleaned up on next tick. }); this.log('server created...'); } /** * Resolves promise if the client is allowed to connect, otherwise rejection. * @param {object} req * @returns {Promise} * @private */ }, { key: '_canConnect', value: function _canConnect(req) { var _this8 = this; var address = req.connection.remoteAddress; var headers = this._getHeaderIdentifiers(req.headers); var _ref = headers || {}, port = _ref.port, nelsonID = _ref.nelsonID, version = _ref.version, remoteKey = _ref.remoteKey, protocol = _ref.protocol; var wrongRequest = !headers; return new Promise(function (resolve, reject) { if (!_this8._ready || !_this8.guard || !_this8.guard.isAllowed(address, port)) { return reject(); } if (wrongRequest || !isSameMajorVersion(version)) { _this8.log('Wrong request or other Nelson version', address, port, version, nelsonID, req.headers); return reject(); } if (!_this8.iri || !_this8.iri.isHealthy) { _this8.log('IRI down, denying connections meanwhile', address, port, nelsonID); return reject(); } if (_this8.isMyself(address, port, nelsonID)) { return reject(); } _this8.list.findByRemoteKeyOrAddress(remoteKey, address, port).then(function (peers) { if (peers.length && _this8.sockets.get(peers[0])) { _this8.log('Peer already connected', address, port); return reject(); } if (peers.length && _this8.iri.isStaticNeighbor(peers[0])) { _this8.log('Peer is already a static neighbor', address, port); return reject(); } // Deny too frequent connections from the same peer. if (peers.length && _this8.isSaturationReached() && peers[0].data.dateLastConnected && getSecondsPassed(peers[0].data.dateLastConnected) < _this8.opts.epochInterval * 2) { return reject(); } // Incompatible protocols if (peers.length[0] && !_this8._negotiateProtocol(protocol)) { _this8.log(('Couldn\'t negotiate protocol with ' + peers[0].data.hostname + ': my ' + _this8.opts.IRIProtocol + ' vs remote ' + protocol).yellow); return reject(); } var topCount = parseInt(Math.sqrt(_this8.list.all().length) * 2); var topPeers = _this8.list.getWeighted(300).sort(function (a, b) { return b[1] - a[1]; }).map(function (p) { return p[0]; }).slice(0, topCount); var isTop = false; peers.forEach(function (p) { if (topPeers.includes(p)) { isTop = true; } }); // The usual way, accept based on personality. var normalPath = function normalPath() { if (_this8._getIncomingSlotsCount() >= _this8.opts.incomingMax) { reject(); } // TODO: additional protection measure: make the client solve a computational riddle! _this8.isAllowed(remoteKey, address, port).then(function (allowed) { return allowed ? resolve() : reject(); }); }; // Accept old, established nodes. if (isTop) { if (_this8._getIncomingSlotsCount() >= _this8.opts.incomingMax) { _this8._dropRandomNeighbors(1, true).then(resolve); } else { resolve(); } } // Accept new nodes more easily. else if (!peers.length || getSecondsPassed(peers[0].data.dateCreated) <= _this8.opts.epochInterval * 10) { if (_this8._getIncomingSlotsCount() >= _this8.opts.incomingMax) { var candidates = Array.from(_this8.sockets.keys()).filter(function (p) { return getSecondsPassed(p.data.dateCreated) <= _this8.opts.epochInterval * 20; }); if (candidates.length) { _this8._dropRandomNeighbors(1, true, candidates).then(resolve); } else { normalPath(); } } else { resolve(); } } else { normalPath(); } }); }); } /** * Binds the websocket to the peer and adds callbacks. * @param {WebSocket} ws * @param {Peer} peer * @param {boolean} asServer * @private */ }, { key: '_bindWebSocket', value: function _bindWebSocket(ws, peer) { var _this9 = this; var asServer = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : false; var removeNeighbor = function removeNeighbor(e) { if (!_this9._ready || !ws || ws.removingNow) { return; } ws.removingNow = true; _this9._removeNeighbor(peer).then(function () { _this9.log('connection closed'.red, _this9.formatNode(peer.data.hostname, peer.data.port), '(' + e + ')'); }); }; var onConnected = function onConnected() { if (!_this9._ready) { return; } _this9.log('connection established'.green, _this9.formatNode(peer.data.hostname, peer.data.port)); _this9._sendNeighbors(ws); return peer.markConnected().then(function () { return _this9._ready && _this9.opts.onPeerConnected(peer); }); }; ws.isAlive = true; ws.incoming = asServer; this.sockets.set(peer, ws); ws.on('message', function (data) { return _this9._addNeighbors(data, ws.incoming ? 0 : peer.data.weight); }); ws.on('close', function () { return removeNeighbor('socket closed'); }); ws.on('error', function () { return removeNeighbor('remotely dropped'); }); ws.on('pong', function () { ws.isAlive = true; }); if (asServer) { onConnected().then(function () { return _this9._ready && _this9.iri.addNeighbors([peer]); }); } else { ws.on('upgrade', function (res) { // Check for valid headers var head = _this9._getHeaderIdentifiers(res.headers); if (!head) { _this9.log('!!', 'wrong headers received', head); return removeNeighbor(); } var port = head.port, nelsonID = head.nelsonID, TCPPort = head.TCPPort, UDPPort = head.UDPPort, remoteKey = head.remoteKey, name = head.name, protocol = head.protocol; var IRIProtocol = _this9._negotiateProtocol(protocol); _this9.list.update(peer, { port: port, nelsonID: nelsonID, TCPPort: TCPPort, UDPPort: UDPPort, remoteKey: remoteKey, name: name, IRIProtocol: IRIProtocol }).then(function () { if (IRIProtocol) { _this9._ready && _this9.iri.addNeighbors([peer]); } else { _this9.log(('Couldn\'t negotiate protocol with ' + peer.data.hostname + ': my ' + _this9.opts.IRIProtocol + ' vs remote ' + IRIProtocol).yellow); removeNeighbor(); } }); }); ws.on('open', onConnected); } } /** * Parses the headers passed between nelson instances * @param {object} headers * @returns {object} * @private */ }, { key: '_getHeaderIdentifiers', value: function _getHeaderIdentifiers(headers) { var version = headers['nelson-version']; var port = headers['nelson-port']; var nelsonID = headers['nelson-id']; var TCPPort = headers['nelson-tcp']; var UDPPort = headers['nelson-udp']; var remoteKey = headers['nelson-key']; var name = headers['nelson-name']; var protocol = headers['nelson-protocol'] || 'udp'; if (!version || !port || !nelsonID || !TCPPort || !UDPPort) { return null; } return { version: version, port: port, nelsonID: nelsonID, TCPPort: TCPPort, UDPPort: UDPPort, remoteKey: remoteKey, name: name, protocol: protocol }; } /** * Sends list of neighbors through the given socket. * @param {WebSocket} ws * @private */ }, { key: '_sendNeighbors', value: function _sendNeighbors(ws) { ws.send(JSON.stringify(this.getPeers().map(function (p) { return p[0].getHostname().replace('/0/', '/' + p[1] + '/'); }))); } /** * Negotiate protocol to be used between the peers. * If null is returned, the connection cannot be established as there is no consensus. * @param {string} protocol preferred by remote * @param {string} key key for remote * @param {string} remoteKey for this node * @returns {string|null} * @private */ }, { key: '_negotiateProtocol', value: function _negotiateProtocol(protocol) { if (protocol === 'any') { switch (this.opts.IRIProtocol) { case 'tcp': case 'prefertcp': return 'tcp'; case 'udp': case 'preferudp': case 'any': default: return 'udp'; } } else if (protocol === 'tcp') { switch (this.opts.IRIProtocol) { case 'any': case 'tcp': case 'prefertcp': case 'preferudp': return 'tcp'; case 'udp': default: return null; } } else if (protocol === 'udp') { switch (this.opts.IRIProtocol) { case 'any': case 'udp': case 'prefertcp': case 'preferudp': return 'udp'; case 'tcp': default: return null; } } else if (protocol === 'prefertcp') { switch (this.opts.IRIProtocol) { case 'any': case 'tcp': case 'prefertcp': return 'tcp'; case 'preferudp': case 'udp': default: return 'udp'; } } else if (protocol === 'preferudp') { switch (this.opts.IRIProtocol) { case 'any': case 'udp': case 'preferudp': case 'prefertcp': return 'udp'; case 'tcp': default: return 'tcp'; } } } /** * Adds a neighbor to known neighbors list. * @param {string} neighbor * @param {number} weight of the neighbor to assign * @returns {Promise} * @private */ }, { key: '_addNeighbor', value: function _addNeighbor(neighbor, weight) { // this.log('adding neighbor', neighbor); var tokens = neighbor.split('/'); if (!isFinite(tokens[1]) || !isFinite(tokens[2]) || !isFinite(tokens[3])) { return Promise.resolve(null); } return this.isMyself(tokens[0], tokens[1]) ? Promise.resolve(null) : this.list.add({ hostname: tokens[0], port: tokens[1], TCPPort: tokens[2], UDPPort: tokens[3], peerWeight: weight, weight: weight * parseFloat(tokens[4] || 0) * this.opts.weightDeflation, IRIProtocol: tokens[5] || 'udp' }); } /** * Parses raw data from peer's response and adds the provided neighbors. * @param {string} data raw from peer's response * @param {number} weight to assign to the parsed neighbors. * @returns {Promise} * @private */ }, { key: '_addNeighbors', value: function _addNeighbors(data, weight) { var _this10 = this; // this.log('add neighbors', data); return new Promise(function (resolve, reject) { try { Promise.all(JSON.parse(data).slice(0, _this10.opts.maxShareableNodes).map(function (neighbor) { return _this10._addNeighbor(neighbor, weight); })).then(resolve); } catch (e) { reject(e); } }); } /** * Returns Nelson headers for request/response purposes * @param {string} key of the peer * @returns {Object} * @private */ }, { key: '_getHeaders', value: function _getHeaders() { var key = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : ''; return { 'Content-Type': 'application/json', 'Nelson-Version': getVersion(), 'Nelson-Port': '' + this.opts.port, 'Nelson-ID': this.heart.personality.publicId, 'Nelson-TCP': this.opts.TCPPort, 'Nelson-UDP': this.opts.UDPPort, 'Nelson-Key': key, 'Nelson-Name': this.opts.name, 'Nelson-Protocol': this.opts.IRIProtocol }; } /** * Returns amount of incoming connections * @returns {Number} * @private */ }, { key: '_getIncomingSlotsCount', value: function _getIncomingSlotsCount() { var arr = Array.from(this.sockets.values()).filter(function (ws) { return ws.readyState < 2; }); return arr.filter(function (ws) { return ws.incoming; }).length; } /** * Returns amount of outgoing connections * @returns {Number} * @private */ }, { key: '_getOutgoingSlotsCount', value: function _getOutgoingSlotsCount() { var arr = Array.from(this.sockets.values()).filter(function (ws) { return ws.readyState < 2; }); return arr.filter(function (ws) { return !ws.incoming; }).length; } /** * Disconnects a peer. * @param {Peer} peer * @returns {Promise} * @private */ }, { key: '_removeNeighbor', value: function _removeNeighbor(peer) { if (!this._ready || !this.sockets.get(peer)) { return Promise.resolve([]); } // this.log('removing neighbor', this.formatNode(peer.data.hostname, peer.data.port)); return this._removeNeighbors([peer]); } /** * Disconnects several peers. * @param {Peer[]} peers * @returns {Promise} * @private */ }, { key: '_removeNeighbors', value: function _removeNeighbors(peers) { var _this11 = this; // this.log('removing neighbors'); var doRemove = function doRemove() { return Promise.all(peers.map(function (peer) { return new Promise(function (resolve) { var ws = _this11.sockets.get(peer); if (ws) { ws.close(); ws.terminate(); } _this11.sockets.delete(peer); peer.markDisconnected().then(function () { _this11.opts.onPeerRemoved(peer); resolve(peer); }); }); })); }; if (!this.iri || !this.iri.isHealthy) { return Promise.resolve(doRemove()); } return this.iri.removeNeighbors(peers).then(doRemove).catch(doRemove); } /** * Randomly removes a given amount of peers from current connections. * Low-quality peers are favored to be removed. * @param {number} amount * @param {boolean} incomingOnly - only drop incoming connections * @param {Peer[]} array - array of connected peers to use for dropping * @returns {Promise.} removed peers * @private */ }, { key: '_dropRandomNeighbors', value: function _dropRandomNeighbors() { var amount = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : 1; var _this12 = this; var incomingOnly = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : false; var array = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : null; var peers = array ? array : incomingOnly ? Array.from(this.sockets.keys()).filter(function (p) { return _this12.sockets.get(p).incoming; }) : array ? array : Array.from(this.sockets.keys()); var selectRandomPeer = function selectRandomPeer() { var weights = peers.map(function (p) { return Math.max(p.getPeerQuality(), 0.0001); }); return weighted(peers, weights); }; var toRemove = []; if (!peers.length) { return Promise.resolve([]); } for (var x = 0; x < amount; x++) { var peer = selectRandomPeer(); peers.splice(peers.indexOf(peer), 1); toRemove.push(peer); } return this._removeNeighbors(toRemove); } /** * Connects to a peer, checking if it's online and trying to get its peers. * @param {Peer} peer * @returns {Peer} */ }, { key: 'connectPeer', value: function connectPeer(peer) { this.log('connecting peer'.yellow, this.formatNode(peer.data.hostname, peer.data.port)); var key = peer.data.key || createIdentifier(); this.list.update(peer, { dateTried: new Date(), tried: (peer.data.tried || 0) + 1, key: key }); this._bindWebSocket(new WebSocket(peer.getNelsonWebsocketURI(), { headers: this._getHeaders(key), handshakeTimeout: 5000 }), peer); return peer; } /** * Connects the node to a new set of random addresses that comply with the out/in rules. * Up to a soft maximum. * @returns {Peer[]} List of new connected peers */ }, { key: 'reconnectPeers', value: function reconnectPeers() { var _this13 = this; // TODO: remove old peers by inverse weight, maybe? Not urgent. Can be added at a later point. // this.log('reconnectPeers'); // If max was reached, do nothing: var toTry = this.opts.outgoingMax - this._getOutgoingSlotsCount(); if (!this.iri || !this.iri.isHealthy || toTry < 1 || this.isMaster || this._getOutgoingSlotsCount() >= this.opts.outgoingMax) { return []; } // Get connectable peers: var list = this.list.all().filter(function (p) { return !p.data.dateTried || getSecondsPassed(p.data.dateTried) > _this13.opts.beatInterval * Math.max(2, 2 * p.data.tried || 0); }).filter(function (p) { return !_this13.iri.isStaticNeighbor(p); }); // Get allowed peers: return this.list.getWeighted(192, list).filter(function (p) { return !_this13.sockets.get(p[0]); }).slice(0, toTry).map(function (p) { return _this13.connectPeer(p[0]); }); } /** * Returns a set of peers ready to be shared with their respective weight ratios. * @returns {Array[]} */ }, { key: 'getPeers', value: function getPeers() { // The node tries to recommend best of the best, even better nodes than it tries to connect, usually. // One tries to be helpful to the others, remember? Only suggesting top-notch peers. return this.list.getWeighted(this.opts.maxShareableNodes, null, 2); } /** * Each epoch, disconnect all peers and reconnect new ones. * @private */ }, { key: '_onEpoch', value: function _onEpoch() { var _this14 = this; this.log('new epoch and new id:', this.heart.personality.id); if (!this.isSaturationReached()) { return Promise.resolve(false); } // Master node should recycle all its connections if (this.opts.isMaster) { return this._removeNeighbors(Array.from(this.sockets.keys())).then(function () { _this14.reconnectPeers(); return false; }); } return this._dropRandomNeighbors(getRandomInt(0, this._getOutgoingSlotsCount())).then(function () { _this14.reconnectPeers(); return false; }); } /** * Checks whether expired peers are still connectable. * If not, disconnect/remove them. * @private */ }, { key: '_onCycle', value: function _onCycle() { var _this15 = this; this.log('new cycle'); var promises = []; // Remove closed or dead sockets. Otherwise set as not alive and ping: this.sockets.forEach(function (ws, peer) { if (ws.readyState > 1 || !ws.isAlive) { promises.push(_this15._removeNeighbor(peer)); } else if (peer.isLazy()) { _this15.log(('Peer ' + peer.data.hostname + ' (' + peer.data.name + ') is lazy for more than ' + _this15.opts.lazyLimit + ' seconds. Removing...!').yellow); promises.push(_this15._removeNeighbor(peer)); } else if (ws.readyState === 1) { ws.isAlive = false; ws.ping('', false); } }); return Promise.all(promises).then(function () { return false; }); } /** * Try connecting to more peers. * @returns {Promise} * @private */ }, { key: '_onTick', value: function _onTick() { var _this16 = this; terminal.nodes({ nodes: this.list.all(), connected: Array.from(this.sockets.keys()).filter(function (p) { return _this16.sockets.get(p).readyState === 1; }).map(function (p) { return p.data; }) }); // Try connecting more peers. Master nodes do not actively connect (no outgoing connections). if (!this.opts.isMaster && this._getOutgoingSlotsCount() < this.opts.outgoingMax) { return new Promise(function (resolve) { _this16.reconnectPeers(); resolve(false); }); } // If for some reason the maximal nodes were overstepped, drop one. else if (this._getIncomingSlotsCount() > this.opts.incomingMax) { return this._dropRandomNeighbors(this._getIncomingSlotsCount() - this.opts.incomingMax, true).then(function () { return false; }); } else { return Promise.resolve(false); } } /** * Callback for IRI to check for health and neighbors. * If unhealthy, disconnect all. Otherwise, disconnect peers that are not in IRI list any more for any reason. * @param {boolean} healthy * @param {object[]} data * @private */ }, { key: '_onIRIHealth', value: function _onIRIHealth(healthy, data) { var _this17 = this; if (!healthy) { // Do not drop connections, yet. IRI might just be unavailable for a moment. // If it still has "old" neighbors, they will leak, causing more nodes to be added than permitted. this.log('IRI gone...'.red); return; // return this._removeNeighbors(Array.from(this.sockets.keys())); } Promise.all(data.map(function (n) { return n.address; }).map(getIP)).then(function (neighbors) { var toRemove = []; Array.from(_this17.sockets.keys()) // It might be that the neighbour was just added and not yet included in IRI... .filter(function (p) { return getSecondsPassed(p.data.dateLastConnected) > 5; }).forEach(function (peer) { if (!neighbors.includes(peer.data.hostname) && (!peer.data.ip || peer.data.ip && !neighbors.includes(peer.data.ip))) { toRemove.push(peer); } else { var index = Math.max(neighbors.indexOf(peer.data.hostname), neighbors.indexOf(peer.data.ip)); index >= 0 && peer.updateConnection(data[index]); } }); if (toRemove.length) { _this17.log('Disconnecting Nelson nodes that are missing in IRI:'.red, toRemove.map(function (p) { return p.data.hostname; })); return _this17._removeNeighbors(toRemove); } }).then(function () { return _this17.iri.cleanupNeighbors(Array.from(_this17.sockets.keys())); }); } /** * Returns whether the provided address/port/id matches this node * @param {string} address * @param {number|string} port * @param {string|null} nelsonID * @returns {boolean} */ }, { key: 'isMyself', value: function isMyself(address, port) { var nelsonID = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : null; var isPrivate = ip.isPrivate(address) || ['127.0.0.1', 'localhost'].includes(address); var sameAddress = isPrivate || address === this.ipv4; var samePort = parseInt(port) === this.opts.port; var sameID = this.heart && this.heart.personality && nelsonID === this.heart.personality.publicId; return sameID || sameAddress && (!this.opts.localNodes || samePort); } /** * Returns whether certain address can contact this instance. * @param {string} remoteKey * @param {string} address * @param {number} port * @param {boolean} checkTrust - whether to check for trusted peer * @param {number} easiness - how "easy" it is to get in * @returns {Promise} */ }, { key: 'isAllowed', value: function isAllowed(remoteKey, address, port) { var _this18 = this; var checkTrust = arguments.length > 3 && arguments[3] !== undefined ? arguments[3] : true; var easiness = arguments.length > 4 && arguments[4] !== undefined ? arguments[4] : 24; var allowed = function allowed() { return getPeerIdentifier(_this18.heart.personality.id + ':' + (_this18.opts.localNodes ? port : address)).slice(0, _this18._getMinEasiness(easiness)).indexOf(_this18.heart.personality.feature) >= 0; }; return checkTrust ? this.list.findByRemoteKeyOrAddress(remoteKey, address, port).then(function (ps) { return ps.filter(function (p) { return p.isTrusted(); }).length || allowed(); }) : Promise.resolve(allowed()); } /** * Returns whether the amount of connected nodes has reached a certain threshold. * @returns {boolean} */ }, { key: 'isSaturationReached', value: function isSaturationReached() { var ratioConnected = (this._getOutgoingSlotsCount() + this._getIncomingSlotsCount()) / (this.opts.outgoingMax + this.opts.incomingMax); return ratioConnected >= 0.75; } /** * For new nodes, make it easy to find nodes and contact them * @param {number} easiness - how easy it is to get in/out * @returns {number} updated easiness value * @private */ }, { key: '_getMinEasiness', value: function _getMinEasiness(easiness) { // New nodes are trusting less the incoming connections. // As the node matures in the community, it becomes more welcoming for inbound requests. var l = this.list.all().filter(function (p) { return p.data.connected; }).length; return Math.min(easiness, Math.max(5, parseInt(l / 2))); } }]); return Node; }(Base); module.exports = { DEFAULT_OPTIONS: DEFAULT_OPTIONS, Node: Node }; ================================================ FILE: dist/node/peer-list.js ================================================ 'use strict'; var _extends = Object.assign || function (target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) { if (Object.prototype.hasOwnProperty.call(source, key)) { target[key] = source[key]; } } } return target; }; var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }(); function _toConsumableArray(arr) { if (Array.isArray(arr)) { for (var i = 0, arr2 = Array(arr.length); i < arr.length; i++) { arr2[i] = arr[i]; } return arr2; } else { return Array.from(arr); } } function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } function _possibleConstructorReturn(self, call) { if (!self) { throw new ReferenceError("this hasn't been initialised - super() hasn't been called"); } return call && (typeof call === "object" || typeof call === "function") ? call : self; } function _inherits(subClass, superClass) { if (typeof superClass !== "function" && superClass !== null) { throw new TypeError("Super expression must either be null or a function, not " + typeof superClass); } subClass.prototype = Object.create(superClass && superClass.prototype, { constructor: { value: subClass, enumerable: false, writable: true, configurable: true } }); if (superClass) Object.setPrototypeOf ? Object.setPrototypeOf(subClass, superClass) : subClass.__proto__ = superClass; } var path = require('path'); var ip = require('ip'); var dns = require('dns'); var tmp = require('tmp'); var _require = require('url'), URL = _require.URL; var weighted = require('weighted'); var Datastore = require('nedb'); var _require2 = require('./base'), Base = _require2.Base; var _require3 = require('./peer'), Peer = _require3.Peer; var _require4 = require('./iri'), DEFAULT_IRI_OPTIONS = _require4.DEFAULT_OPTIONS; var _require5 = require('./tools/utils'), getSecondsPassed = _require5.getSecondsPassed, createIdentifier = _require5.createIdentifier; var DEFAULT_OPTIONS = { dataPath: path.join(process.cwd(), 'data/neighbors.db'), isMaster: false, multiPort: false, temporary: false, logIdent: 'LIST', ageNormalizer: 3600, lazyLimit: 300, // Time, after which a peer is considered lazy, if no new TXs received lazyTimesLimit: 3 // starts to penalize peer's quality if connected so many times without new TXs }; /** * A class that manages a list of peers and its persistence in the database * @class PeerList */ var PeerList = function (_Base) { _inherits(PeerList, _Base); function PeerList(options) { _classCallCheck(this, PeerList); var _this = _possibleConstructorReturn(this, (PeerList.__proto__ || Object.getPrototypeOf(PeerList)).call(this, _extends({}, DEFAULT_OPTIONS, options))); _this.onPeerUpdate = _this.onPeerUpdate.bind(_this); _this.loaded = false; _this.peers = []; _this.db = new Datastore({ filename: _this.opts.temporary ? tmp.tmpNameSync() : _this.opts.dataPath, autoload: true }); _this.db.persistence.setAutocompactionInterval(30000); return _this; } /** * Loads the peer database, preloading defaults, if any. * @param {string[]} defaultPeerURLs * @returns {Promise} */ _createClass(PeerList, [{ key: 'load', value: function load(defaultPeerURLs) { var _this2 = this; return new Promise(function (resolve) { _this2.db.find({}, function (err, docs) { _this2.peers = docs.map(function (data) { return new Peer(data, _this2._getPeerOptions()); }); _this2.loadDefaults(defaultPeerURLs).then(function () { _this2.log('DB and default peers loaded'); _this2.loaded = true; resolve(_this2.peers); }); }); }); } /** * Adds default peers to the database/list. * @param {string[]} defaultPeerURLs * @returns {Promise} */ }, { key: 'loadDefaults', value: function loadDefaults() { var _this3 = this; var defaultPeerURLs = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : []; return Promise.all(defaultPeerURLs.map(function (uri) { var tokens = uri.split('/'); return _this3.add({ hostname: tokens[0], port: tokens[1], TCPPort: tokens[2], UDPPort: tokens[3], weight: tokens[4] || 1.0, IRIProtocol: tokens[5] || 'udp', isTrusted: true }); })); } /** * Update callback when the peer's data has been changed from within the peer. * @param peer * @returns {Promise.} */ }, { key: 'onPeerUpdate', value: function onPeerUpdate(peer) { var data = _extends({}, peer.data); delete data._id; return this.update(peer, data, false); } /** * Partially updates a peer with the provided data. Saves into database. * @param {Peer} peer * @param {Object} data * @param {boolean} refreshPeer - whether to update the peers data. * @returns {Promise} */ }, { key: 'update', value: function update(peer, data) { var _this4 = this; var refreshPeer = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : true; var newData = _extends({}, peer.data, data); return new Promise(function (resolve) { _this4.db.update({ _id: peer.data._id }, newData, { returnUpdatedDocs: true }, function () { // this.log(`updated peer ${peer.data.hostname}:${peer.data.port}`, JSON.stringify(data)); refreshPeer ? peer.update(newData, false).then(function () { return resolve(peer); }) : resolve(peer); }); }); } /** * Returns currently loaded peers. * @returns {Peer[]} */ }, { key: 'all', value: function all() { return this.peers; } /** * Removes all peers. */ }, { key: 'clear', value: function clear() { var _this5 = this; this.log('Clearing all known peers'); this.peers = []; return new Promise(function (resolve) { return _this5.db.remove({}, { multi: true }, resolve); }); } /** * Gets the average age of all known peers * @returns {number} */ }, { key: 'getAverageAge', value: function getAverageAge() { return this.peers.map(function (p) { return getSecondsPassed(p.data.dateCreated); }).reduce(function (s, x) { return s + x; }, 0) / this.peers.length; } /** * Returns peers, whose remoteKey, hostname or IP equals the address. * Port is only considered if multiPort option is true. * If the address/port matches, the remoteKey is not considered. * @param {string} remoteKey * @param {string} address * @param {number} port * @returns {Promise} */ }, { key: 'findByRemoteKeyOrAddress', value: function findByRemoteKeyOrAddress(remoteKey, address, port) { var _this6 = this; return new Promise(function (resolve) { _this6.findByAddress(address, port).then(function (peers) { if (peers.length) { return resolve(peers); } resolve(_this6.peers.filter(function (p) { return p.data.remoteKey && p.data.remoteKey === remoteKey; })); }); }); } /** * Returns peers, whose hostname or IP equals the address. * Port is only considered if mutiPort option is true. * @param {string} address * @param {number} port * @returns {Promise} */ }, { key: 'findByAddress', value: function findByAddress(address, port) { var _this7 = this; var addr = PeerList.cleanAddress(address); return new Promise(function (resolve) { var findWithIP = function findWithIP(ip) { var peers = _this7.peers.filter(function (p) { return p.data.hostname === addr || p.data.hostname === address || ip && (p.data.hostname === ip || p.data.ip === ip); }); resolve(_this7.opts.multiPort ? peers.filter(function (p) { return p.data.port == port; }) : peers); }; if (ip.isV6Format(addr) || ip.isV4Format(addr) || _this7.opts.multiPort) { findWithIP(addr); } else { dns.resolve(addr, 'A', function (error, results) { return findWithIP(error || !results.length ? null : results[0]); }); } }); } /** * Calculates the trust score of a peer * @param {Peer} peer * @returns {number} */ }, { key: 'getPeerTrust', value: function getPeerTrust(peer) { var age = parseFloat(getSecondsPassed(peer.data.dateCreated)) / this.opts.ageNormalizer; if (this.opts.isMaster) { var _weightedAge = Math.pow(peer.data.connected || peer.isTrusted() ? age : 0, 2) * Math.pow(peer.getPeerQuality(), 2); return Math.max(_weightedAge, 0.0001); } var weightedAge = Math.pow(age, 2) * Math.pow(peer.getPeerQuality(), 2) * Math.pow(1.0 + peer.data.weight * 10, 2); return Math.max(weightedAge, 0.0001); } /** * Get a certain amount of weighted random peers. Return peers with their respective weight ratios * The weight depends on relationship age (connections) and trust (weight). * @param {number} amount * @param {Peer[]} sourcePeers list of peers to use. Optional for filtering purposes. * @param {number} power by which increase the weights * @returns {Array} */ }, { key: 'getWeighted', value: function getWeighted() { var amount = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : 0; var _this8 = this; var sourcePeers = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : null; var power = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : 1.0; amount = amount || this.peers.length; var peers = sourcePeers || Array.from(this.peers); if (!peers.length) { return []; } var allWeights = peers.map(function (p) { return Math.pow(_this8.getPeerTrust(p), power); }); var weightsMax = Math.max.apply(Math, _toConsumableArray(allWeights)); var choices = []; var getChoice = function getChoice() { var peer = weighted(peers, allWeights); var index = peers.indexOf(peer); var weight = allWeights[index]; peers.splice(index, 1); allWeights.splice(index, 1); choices.push([peer, weight / weightsMax]); }; for (var x = 0; x < amount; x++) { if (peers.length < 1) { break; } getChoice(); } return choices.filter(function (c) { return c && c[0]; }).map(function (c) { return [c[0], c[0].isTrusted() ? 1.0 : c[1]]; }); } /** * Adds a new peer to the list using an URI * @param {object} data * @returns {*} */ }, { key: 'add', value: function add(data) { var _this9 = this; var _Object$assign = Object.assign({ TCPPort: DEFAULT_IRI_OPTIONS.TCPPort, UDPPort: DEFAULT_IRI_OPTIONS.UDPPort, IRIProtocol: 'udp', isTrusted: false, peerWeight: 0.5, weight: 0, remoteKey: null }, data), hostname = _Object$assign.hostname, rawPort = _Object$assign.port, rawTCPPort = _Object$assign.TCPPort, rawUDPPort = _Object$assign.UDPPort, IRIProtocol = _Object$assign.IRIProtocol, isTrusted = _Object$assign.isTrusted, peerWeight = _Object$assign.peerWeight, weight = _Object$assign.weight, remoteKey = _Object$assign.remoteKey, name = _Object$assign.name; var port = parseInt(rawPort); var TCPPort = parseInt(rawTCPPort || DEFAULT_IRI_OPTIONS.TCPPort); var UDPPort = parseInt(rawUDPPort || DEFAULT_IRI_OPTIONS.UDPPort); return this.findByRemoteKeyOrAddress(remoteKey, hostname, port).then(function (peers) { var addr = PeerList.cleanAddress(hostname); var existing = peers.length && peers[0]; if (existing) { return _this9.update(existing, { weight: weight ? existing.data.weight ? weight * peerWeight + existing.data.weight * (1.0 - peerWeight) : weight : existing.data.weight, key: existing.data.key || createIdentifier(), remoteKey: remoteKey || existing.data.remoteKey, name: name || existing.data.name, hostname: addr, port: port, TCPPort: TCPPort, UDPPort: UDPPort, IRIProtocol: IRIProtocol }); } else { _this9.log('Adding to the list of known Nelson peers: ' + hostname + ':' + port); var peerIP = ip.isV4Format(addr) || ip.isV6Format(addr) ? addr : null; var peer = new Peer({ port: port, hostname: addr, ip: peerIP, TCPPort: TCPPort || DEFAULT_IRI_OPTIONS.TCPPort, UDPPort: UDPPort || DEFAULT_IRI_OPTIONS.UDPPort, IRIProtocol: IRIProtocol || 'udp', isTrusted: isTrusted, name: name, weight: weight, remoteKey: remoteKey, key: createIdentifier(), dateCreated: new Date() }, _this9._getPeerOptions()); _this9.peers.push(peer); return new Promise(function (resolve, reject) { _this9.db.insert(peer.data, function (err, doc) { if (err) { reject(err); } peer.update(doc); resolve(peer); }); }); } }); } }, { key: '_getPeerOptions', value: function _getPeerOptions() { var _opts = this.opts, lazyLimit = _opts.lazyLimit, lazyTimesLimit = _opts.lazyTimesLimit; return { lazyLimit: lazyLimit, lazyTimesLimit: lazyTimesLimit, onDataUpdate: this.onPeerUpdate }; } /** * Converts an address to a cleaner format. * @param {string} address * @returns {string} */ }], [{ key: 'cleanAddress', value: function cleanAddress(address) { if (!ip.isV4Format(address) && !ip.isV6Format(address)) { return address; } return ip.isPrivate(address) ? 'localhost' : address.replace('::ffff:', ''); } }]); return PeerList; }(Base); module.exports = { DEFAULT_OPTIONS: DEFAULT_OPTIONS, PeerList: PeerList }; ================================================ FILE: dist/node/peer.js ================================================ 'use strict'; var _extends = Object.assign || function (target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) { if (Object.prototype.hasOwnProperty.call(source, key)) { target[key] = source[key]; } } } return target; }; var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }(); function _toConsumableArray(arr) { if (Array.isArray(arr)) { for (var i = 0, arr2 = Array(arr.length); i < arr.length; i++) { arr2[i] = arr[i]; } return arr2; } else { return Array.from(arr); } } function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } function _possibleConstructorReturn(self, call) { if (!self) { throw new ReferenceError("this hasn't been initialised - super() hasn't been called"); } return call && (typeof call === "object" || typeof call === "function") ? call : self; } function _inherits(subClass, superClass) { if (typeof superClass !== "function" && superClass !== null) { throw new TypeError("Super expression must either be null or a function, not " + typeof superClass); } subClass.prototype = Object.create(superClass && superClass.prototype, { constructor: { value: subClass, enumerable: false, writable: true, configurable: true } }); if (superClass) Object.setPrototypeOf ? Object.setPrototypeOf(subClass, superClass) : subClass.__proto__ = superClass; } var ip = require('ip'); var dns = require('dns'); var _require = require('./base'), Base = _require.Base; var _require2 = require('./iri'), DEFAULT_IRI_OPTIONS = _require2.DEFAULT_OPTIONS; var _require3 = require('./tools/utils'), getSecondsPassed = _require3.getSecondsPassed, createIdentifier = _require3.createIdentifier; var PROTOCOLS = ['tcp', 'udp', 'prefertcp', 'preferudp', 'any']; var DEFAULT_OPTIONS = { onDataUpdate: function onDataUpdate(data) { return Promise.resolve(); }, ipRefreshTimeout: 1200, silent: true, logIdent: 'PEER', lazyLimit: 300, // Time, after which a peer is considered lazy, if no new TXs received lazyTimesLimit: 3 // starts to penalize peer's quality if connected so many times without new TXs }; var DEFAULT_PEER_DATA = { name: null, hostname: null, ip: null, port: 31337, TCPPort: DEFAULT_IRI_OPTIONS.TCPPort, UDPPort: DEFAULT_IRI_OPTIONS.UDPPort, IRIProtocol: 'udp', // Assume all old Nelsons to be running udp. seen: 1, connected: 0, tried: 0, weight: 0, dateTried: null, dateLastConnected: null, dateCreated: null, isTrusted: false, key: null, remoteKey: null, lastConnections: [] }; /** * A utility class for a peer that holds peer's data and provides a few util methods. * * @class Peer */ var Peer = function (_Base) { _inherits(Peer, _Base); function Peer() { var data = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : {}; var options = arguments[1]; _classCallCheck(this, Peer); var _this = _possibleConstructorReturn(this, (Peer.__proto__ || Object.getPrototypeOf(Peer)).call(this, _extends({}, DEFAULT_OPTIONS, options, { logIdent: data.hostname + ':' + data.port }))); _this.data = null; _this.lastConnection = null; // Make sure to migrate database if anything else is added to the defaults... _this.update(_extends({}, DEFAULT_PEER_DATA, data)); return _this; } /** * Partial peer's data update * @param {Object} data * @param {boolean} doCallback - whether to call back on data changes * @returns {Promise} - updates data */ _createClass(Peer, [{ key: 'update', value: function update(data) { var _this2 = this; var doCallback = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : true; // Reset last updated date if the hostname has changed var shouldUpdate = false; var hostnameChanged = this.data && this.data.hostname !== (data && data.hostname); this.iplastUpdated = this.data && hostnameChanged ? null : this.iplastUpdated; this.data = _extends({}, this.data, data); if (hostnameChanged && this.data.ip) { this.data.ip = null; shouldUpdate = true; } if (!this.data.ip) { this.data.ip = this._isHostnameIP() ? this.data.hostname : null; shouldUpdate = true; } return shouldUpdate && doCallback ? this.opts.onDataUpdate(this).then(function () { return _this2.data; }) : Promise.resolve(this.data); } /** * Gets the IP address of the peer. Independently if peer's address is a hostname or IP. * Update's the peer data to save the last known IP. * @returns {Promise} */ }, { key: 'getIP', value: function getIP() { var _this3 = this; return new Promise(function (resolve) { if (!_this3._hasCorrectIP() || !_this3._isHostnameIP() && _this3._isIPOutdated()) { dns.resolve(_this3.data.hostname, 'A', function (error, results) { // if there was an error, we set the hostname as ip, even if it's not the case. // It will be re-tried next refresh cycle. var ip = error || !results.length ? null : results[0]; _this3.iplastUpdated = new Date(); if (ip && ip !== _this3.data.ip) { _this3.data.ip = ip; _this3.opts.onDataUpdate(_this3).then(function () { return resolve(ip); }); } else { resolve(ip); } }); } else { resolve(_this3.data.ip); } }); } /** * Marks this node as connected. * @returns {Promise.} */ }, { key: 'markConnected', value: function markConnected() { var _this4 = this; if (this.lastConnection) { return Promise.resolve(this); } this.lastConnection = { start: new Date(), duration: 0, numberOfAllTransactions: 0, numberOfNewTransactions: 0, numberOfInvalidTransactions: 0 }; return this.update({ key: this.data.key || createIdentifier(), tried: 0, connected: this.data.connected + 1, dateLastConnected: new Date() }).then(function () { return _this4; }); } /** * Marks the node as disconnected. Saves connection stats in DB. * @returns {Promise.} */ }, { key: 'markDisconnected', value: function markDisconnected() { var _this5 = this; if (!this.lastConnection) { return Promise.resolve(this); } var lastConnections = [].concat(_toConsumableArray(this.data.lastConnections), [_extends({}, this.lastConnection, { end: new Date(), duration: this.getConnectionDuration() })]).slice(-10); this.lastConnection = null; return this.update({ lastConnections: lastConnections }).then(function () { return _this5; }); } /** * Returns time in seconds that passed since the node has been connected * @returns {number} */ }, { key: 'getConnectionDuration', value: function getConnectionDuration() { if (!this.lastConnection) { return 0; } return getSecondsPassed(this.lastConnection.start); } /** * Updates the stats of the currently connected peer * @param data */ }, { key: 'updateConnection', value: function updateConnection(data) { if (!this.lastConnection) { return; } var numberOfAllTransactions = data.numberOfAllTransactions, numberOfRandomTransactionRequests = data.numberOfRandomTransactionRequests, numberOfNewTransactions = data.numberOfNewTransactions, numberOfInvalidTransactions = data.numberOfInvalidTransactions; this.lastConnection = _extends({}, this.lastConnection, { numberOfAllTransactions: numberOfAllTransactions, numberOfRandomTransactionRequests: numberOfRandomTransactionRequests, numberOfNewTransactions: numberOfNewTransactions, numberOfInvalidTransactions: numberOfInvalidTransactions }); } /** * Returns peer's quality based on last connection stats. * @returns {number} */ }, { key: 'getPeerQuality', value: function getPeerQuality() { var history = [].concat(_toConsumableArray(this.data.lastConnections), [this.lastConnection]).filter(function (h) { return h; }); var newTrans = history.reduce(function (s, h) { return s + h.numberOfNewTransactions; }, 0); var badTrans = history.reduce(function (s, h) { return s + h.numberOfInvalidTransactions; }, 0); var rndTrans = history.reduce(function (s, h) { return s + (h.numberOfRandomTransactionRequests || 0); }, 0); var badRatio = parseFloat(badTrans * 5 + rndTrans) / (newTrans || 1); var serialPenalization = !this.isTrusted() && !newTrans && history.length >= this.opts.lazyTimesLimit ? 1.0 / history.length : 1.0; var score = Math.max(0.0, 1.0 / (badRatio || 1)) * serialPenalization; return Math.max(0.01, score); } /** * Returns whether a connected peer has not sent any new transactions for a prolonged period of time. * @returns {boolean} */ }, { key: 'isLazy', value: function isLazy() { return this.lastConnection && getSecondsPassed(this.lastConnection.start) > this.opts.lazyLimit && (this.lastConnection.numberOfNewTransactions === 0 || this.lastConnection.numberOfNewTransactions < this.lastConnection.numberOfRandomTransactionRequests); } }, { key: 'getTCPURI', value: function getTCPURI() { return 'tcp://' + this._getIPString(this.data.hostname) + ':' + this.data.TCPPort; } }, { key: 'getUDPURI', value: function getUDPURI() { return 'udp://' + this._getIPString(this.data.hostname) + ':' + this.data.UDPPort; } }, { key: 'getNelsonURI', value: function getNelsonURI() { return 'http://' + this._getIPString(this.data.hostname) + ':' + this.data.port; } }, { key: 'getNelsonWebsocketURI', value: function getNelsonWebsocketURI() { return 'ws://' + this._getIPString(this.data.hostname) + ':' + this.data.port; } }, { key: 'getHostname', value: function getHostname() { return this.data.hostname + '/' + this.data.port + '/' + this.data.TCPPort + '/' + this.data.UDPPort + '/0/' + this.data.IRIProtocol; } }, { key: 'isTrusted', value: function isTrusted() { return this.data && this.data.isTrusted; } }, { key: 'isSameIP', value: function isSameIP(ip) { return this.getIP().then(function (myIP) { return myIP && myIP === ip; }); } }, { key: '_isHostnameIP', value: function _isHostnameIP() { return ip.isV4Format(this.data.hostname) || ip.isV6Format(this.data.hostname); } }, { key: '_hasCorrectIP', value: function _hasCorrectIP() { return this.data.ip && (ip.isV4Format(this.data.ip) || ip.isV6Format(this.data.ip)); } }, { key: '_getIPString', value: function _getIPString(ipOrHostname) { return ipOrHostname.includes(':') ? '[' + ipOrHostname + ']' : ipOrHostname; } }, { key: '_isIPOutdated', value: function _isIPOutdated() { return !this.iplastUpdated || getSecondsPassed(this.iplastUpdated) > this.opts.ipRefreshTimeout; } }]); return Peer; }(Base); module.exports = { DEFAULT_OPTIONS: DEFAULT_OPTIONS, DEFAULT_PEER_DATA: DEFAULT_PEER_DATA, PROTOCOLS: PROTOCOLS, Peer: Peer }; ================================================ FILE: dist/node/tools/terminal.js ================================================ 'use strict'; var blessed = require('blessed'); var contrib = require('blessed-contrib'); require('colors'); var moment = require('moment'); var momentDurationFormatSetup = require("moment-duration-format"); momentDurationFormatSetup(moment); var screen = null; var mainBox = null; var statusBox = null; var peersBox = null; var progress = null; module.exports = { init: init, exit: ensureScreen(exit), log: log, beat: ensureScreen(beat), settings: ensureScreen(settings), ports: ensureScreen(ports), nodes: ensureScreen(nodes) }; function init(name, version, onExit) { screen = blessed.screen({ smartCSR: true }); screen.key(['escape', 'q', 'C-c'], function () { exit(); return onExit(); }); mainBox = blessed.box({ top: '51%', left: 'center', width: '100%', height: '49%', content: 'Nelson Console:', scrollable: true, alwaysScroll: true, tags: true, border: { type: 'line' }, style: { fg: 'white', border: { fg: '#f0f0f0' } } }); statusBox = blessed.box({ top: '0%', left: '0%', width: '30%', height: '51%', content: (name + ' v.' + version + ' - Status').green.bold, tags: true, border: { type: 'line' }, style: { fg: 'white', border: { fg: '#f0f0f0' } } }); peersBox = blessed.box({ top: '0%', left: '50%', width: '50%', height: '51%', content: 'Peers'.green.bold, tags: true, border: { type: 'line' }, style: { fg: 'white', border: { fg: '#f0f0f0' } } }); progress = contrib.donut({ top: '0%', left: '30%', width: '20%', height: '51%', radius: 8, arcWidth: 3, remainColor: 'black', yPadding: 2, border: { type: 'line' }, style: { fg: 'white', border: { fg: '#f0f0f0' } } }); screen.append(mainBox); screen.append(statusBox); screen.append(peersBox); screen.append(progress); mainBox.focus(); screen.render(); } function log() { var msg = Array.from(arguments).join(' '); if (!screen) { console.log(msg); return; } mainBox.pushLine(msg); mainBox.setScrollPerc(100); screen.render(); } function beat(_ref) { var epoch = _ref.epoch, cycle = _ref.cycle, startDate = _ref.startDate, pctEpoch = _ref.pctEpoch, pctCycle = _ref.pctCycle; var duration = moment.duration(moment().diff(startDate)).format('d [days] h [hours] m [minutes]'); statusBox.setLine(3, ('Online: ' + duration).bold.yellow); statusBox.setLine(4, ('Epoch: ' + epoch).bold); statusBox.setLine(5, ('Cycle: ' + cycle).bold); progress.setData([{ percent: pctEpoch, label: 'epoch', color: 'green' }, { percent: pctCycle, label: 'cycle', color: 'green' }]); screen.render(); } function settings(_ref2) { var epochInterval = _ref2.epochInterval, cycleInterval = _ref2.cycleInterval, startDate = _ref2.startDate; var startDateString = moment(startDate).format('dddd, MMMM Do YYYY, HH:mm:ss.SSS'); statusBox.setLine(2, ('Started on: ' + startDateString).yellow); statusBox.setLine(6, 'Epoch Interval: ' + epochInterval + 's'); statusBox.setLine(7, 'Cycle Interval: ' + cycleInterval + 's'); screen.render(); } function ports(_ref3) { var port = _ref3.port, apiPort = _ref3.apiPort, IRIPort = _ref3.IRIPort, TCPPort = _ref3.TCPPort, UDPPort = _ref3.UDPPort; statusBox.setLine(8, ('Port: ' + port).dim.cyan); statusBox.setLine(9, ('API Port: ' + apiPort).dim.cyan); statusBox.setLine(10, ('IRI Port: ' + IRIPort).dim.cyan); statusBox.setLine(11, ('TCP Port: ' + TCPPort).dim.cyan); statusBox.setLine(12, ('UDP Port: ' + UDPPort).dim.cyan); screen.render(); } function nodes(_ref4) { var nodes = _ref4.nodes, connected = _ref4.connected; peersBox.setLine(2, ('Count: ' + nodes.length + ' (Connected: ' + (connected.length || 0) + ')').bold); peersBox.setLine(4, 'Connections:'.bold); var lines = peersBox.getLines().length; for (var i = lines - 1; i >= 5; i--) { peersBox.clearLine(i); } if (!Array.isArray(connected) || connected.length === 0) { peersBox.setLine(5, 'do not worry, this may take a while...'.dim); } else { connected.forEach(function (connection, i) { var id = ((connection.hostname || connection.ip) + ':' + connection.port).bold.cyan; id = connection.name ? (id + ' (' + connection.name + ')').bold.cyan : id; // const weight = `[trust: ${(connection.trust * 100).toFixed(6)}]`.green; peersBox.setLine(5 + i, id + ' -> ' + (connection.IRIProtocol || 'udp')); }); } screen.render(); } function ensureScreen(f) { return function () { if (!screen) { return; } return f.apply(undefined, arguments); }; } function exit() { screen.destroy(); screen = null; } ================================================ FILE: dist/node/tools/utils.js ================================================ 'use strict'; var ip = require('ip'); var dns = require('dns'); var version = require('../../../package.json').version; var crypto = require('crypto'); var md5 = require('md5'); /** * Resolves IP or hostname to IP. If failed, returns the input. * @param {string} ipOrHostName * @returns {Promise} */ function getIP(ipOrHostName) { return new Promise(function (resolve) { if (ip.isV4Format(ipOrHostName) || ip.isV6Format(ipOrHostName)) { return resolve(ipOrHostName); } dns.resolve(ipOrHostName, 'A', function (error, results) { resolve(error ? ipOrHostName : results[0]); }); }); } /** * Returns number of seconds that passed starting from a given time. * @param time * @returns {number} */ function getSecondsPassed(time) { if (!time) { return 0; } return (new Date().getTime() - time.getTime()) / 1000; } /** * Creates a random 96-char-long hexadecimal identifier. * @returns {string} */ function createIdentifier() { return crypto.randomBytes(48).toString('hex'); } /** * Creates an MD5 hash from the given address * @param {string} address * @returns {string} */ function getPeerIdentifier(address) { return md5(address); } /** * Returns a random number * @param {number} min * @param {number} max * @returns {number} */ function getRandomInt(min, max) { min = Math.ceil(min); max = Math.floor(max); return Math.floor(Math.random() * (max - min)) + min; //The maximum is exclusive and the minimum is inclusive } /** * Shuffles the array * @param {Array} array * @returns {Array} */ function shuffleArray(array) { return array.sort(function () { return Math.random() - 0.5; }); } /** * Returns Nelson version number */ function getVersion() { return version; } /** * Returns whether the provided version number is the same major version as the current Nelson. * @param {string} otherVersion */ function isSameMajorVersion(otherVersion) { return version.split('.')[0] === otherVersion.split('.')[0]; } /** * Returns whether the provided string is a valid Nelson neighbor representation * @param str * @returns {boolean} */ function validNeighbor(str) { var tokens = str.split('/'); return tokens.length >= 2 && tokens.length <= 5 && Number.isInteger(parseInt(tokens[1])) && (!tokens[2] || Number.isInteger(parseInt(tokens[2]))) && (!tokens[3] || Number.isInteger(parseInt(tokens[3]))) && (!tokens[4] || !!parseFloat(tokens[4])); } module.exports = { getIP: getIP, createIdentifier: createIdentifier, getPeerIdentifier: getPeerIdentifier, getRandomInt: getRandomInt, getSecondsPassed: getSecondsPassed, getVersion: getVersion, isSameMajorVersion: isSameMajorVersion, shuffleArray: shuffleArray, validNeighbor: validNeighbor }; ================================================ FILE: dist/simulation/bin/nelson.js ================================================ #!/usr/bin/env node 'use strict'; var program = require('commander'); var _require = require('../index'), initMockedNode = _require.initMockedNode; var version = require('../../../package.json').version; var parseNeighbors = function parseNeighbors(val) { return val.split(' '); }; var parseProtocol = function parseProtocol(val) { return val.toLowerCase(); }; var parseNumber = function parseNumber(v) { return parseInt(v); }; process.on('unhandledRejection', function (reason, p) { console.log('Unhandled Rejection at: Promise', p, 'reason:', reason); // application specific logging, throwing an error, or other logic here }); program.version(version).option('-n, --neighbors [value]', 'Trusted neighbors', parseNeighbors, []).option('-c, --cycle [value]', 'Cycle interval in seconds', parseNumber, 10).option('-e, --epoch [value]', 'Epoch interval in seconds', parseNumber, 60).option('-p, --port [value]', 'Nelson port', parseNumber, 14265).option('--IRIProtocol [value]', 'IRI protocol to use: udp or tcp, prefertcp, preferudp or any', parseProtocol, 'any').option('--master [value]', 'Is master node', false).option('-s, --silent [value]', 'Silent', false).parse(process.argv); initMockedNode({ port: program.port, dataPort: program.dataPort, silent: program.silent, cycleInterval: program.cycle, epochInterval: program.epoch, neighbors: program.neighbors, isMaster: program.master, IRIProtocol: program.IRIProtocol }); ================================================ FILE: dist/simulation/bin/network.js ================================================ #!/usr/bin/env node 'use strict'; var _extends = Object.assign || function (target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) { if (Object.prototype.hasOwnProperty.call(source, key)) { target[key] = source[key]; } } } return target; }; var program = require('commander'); var _require = require('../network'), spawnMockedNetwork = _require.spawnMockedNetwork, DEFAULT_OPTS = _require.DEFAULT_OPTS; var version = require('../../../package.json').version; var parseNumber = function parseNumber(v) { return parseInt(v); }; process.on('unhandledRejection', function (reason, p) { console.log('Unhandled Rejection at: Promise', p, 'reason:', reason); // application specific logging, throwing an error, or other logic here }); program.version(version).option('-c, --cycleInterval [value]', 'Cycle interval in seconds', parseNumber, DEFAULT_OPTS.cycleInterval).option('-e, --epochInterval [value]', 'Epoch interval in seconds', parseNumber, DEFAULT_OPTS.epochInterval).option('-p, --startingPort [value]', 'Starting port', parseNumber, DEFAULT_OPTS.startingPort).option('-n, --nodesCount [value]', 'Normal nodes amount', parseNumber, DEFAULT_OPTS.nodesCount).option('-m, --masterNodesCount [value]', 'Master nodes amount', parseNumber, DEFAULT_OPTS.masterNodesCount).option('-s, --silent', 'Silent', DEFAULT_OPTS.silent).parse(process.argv); var proc = spawnMockedNetwork(_extends({}, program, { callback: stats.onCallback })); process.on('SIGINT', proc.end); process.on('SIGTERM', proc.end); ================================================ FILE: dist/simulation/index.js ================================================ 'use strict'; var _require = require('./node'), initMockedNode = _require.initMockedNode; var _require2 = require('./network'), spawnMockedNetwork = _require2.spawnMockedNetwork; module.exports = { initMockedNode: initMockedNode, spawnMockedNetwork: spawnMockedNetwork }; ================================================ FILE: dist/simulation/network.js ================================================ 'use strict'; var _extends = Object.assign || function (target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) { if (Object.prototype.hasOwnProperty.call(source, key)) { target[key] = source[key]; } } } return target; }; function _toConsumableArray(arr) { if (Array.isArray(arr)) { for (var i = 0, arr2 = Array(arr.length); i < arr.length; i++) { arr2[i] = arr[i]; } return arr2; } else { return Array.from(arr); } } var _require = require('../node'), utils = _require.utils, peer = _require.peer; var _require2 = require('./node'), spawnNode = _require2.spawnNode; var DEFAULT_OPTS = { silent: false, cycleInterval: 12, epochInterval: 36, nodesCount: 47, masterNodesCount: 3, startingPort: 14265, nodeStartDelayRange: [0, 6000], callbackInterval: 5000, onStats: function onStats(nodeStats) {}, onError: function onError(nodeStats) {} }; // TODO: update jsdoc /** * Initializes and starts a simulation with a set of mocked nodes. * First, the master nodes are started all at once. Then the normal nodes are added sequentially * in a random interval (using options.nodeStartDelayRange). * * @param {object} options * @param {number} options.nodesCount - the amount of "normal" nodes to start * @param {number} options.masterNodesCount - the amount of "master" nodes to start * @param {number} options.startingPort - Port number to start the nodes from incrementally * @param {function} options.onStats - periodic callback with connection summary * @param {function} options.onError - on child process exit or error * @param {boolean} options.callbackInterval - in seconds * @param {number[]} options.nodeStartDelayRange - how many ms to wait between normal nodes starting * @returns {{stop: (function()), onPeersAdded: (function())}} */ function spawnMockedNetwork(options) { var _DEFAULT_OPTS$options = _extends({}, DEFAULT_OPTS, options), nodesCount = _DEFAULT_OPTS$options.nodesCount, masterNodesCount = _DEFAULT_OPTS$options.masterNodesCount, startingPort = _DEFAULT_OPTS$options.startingPort, silent = _DEFAULT_OPTS$options.silent, cycleInterval = _DEFAULT_OPTS$options.cycleInterval, epochInterval = _DEFAULT_OPTS$options.epochInterval, onStats = _DEFAULT_OPTS$options.onStats, onError = _DEFAULT_OPTS$options.onError, callbackInterval = _DEFAULT_OPTS$options.callbackInterval, nodeStartDelayRange = _DEFAULT_OPTS$options.nodeStartDelayRange; var baseNodeOptions = { silent: silent, cycleInterval: cycleInterval, epochInterval: epochInterval }; var allNodes = []; var masterNodeURIs = []; var stats = {}; var ended = false; var cbInterval = null; var hasEnded = function hasEnded() { return ended; }; var prc = function prc(p, port) { p.on('message', function (s) { return stats[port] = s; }); p.on('error', onError); p.on('exit', function () { return !hasEnded() && onError(); }); }; // Start the master nodes for (var x = 0; x < masterNodesCount; x++) { var port = startingPort + x; var TCPPort = port + 10000; var UDPPort = port + 20000; if (ended) { break; } var node = spawnNode(_extends({}, baseNodeOptions, { port: port, isMaster: true, neighbors: masterNodeURIs })); prc(node, port); allNodes.push(node); masterNodeURIs.push('localhost/' + port + '/' + TCPPort + '/' + UDPPort); } // Sequentially start the normal nodes var promise = ".".repeat(nodesCount).split('').reduce(function (promise, value, y) { return hasEnded() ? promise : promise.then(function (nodes) { return new Promise(function (resolve) { if (hasEnded()) { return resolve(nodes); } setTimeout(function () { if (hasEnded()) { return resolve(nodes); } var port = startingPort + masterNodesCount + y; var TCPPort = port + 10000; var UDPPort = port + 20000; var node = spawnNode(_extends({}, baseNodeOptions, { port: port, TCPPort: TCPPort, UDPPort: UDPPort, neighbors: masterNodeURIs, IRIProtocol: peer.PROTOCOLS[utils.getRandomInt(0, peer.PROTOCOLS.length)] })); prc(node, port); resolve([].concat(_toConsumableArray(nodes), [node])); }, utils.getRandomInt(nodeStartDelayRange[0], nodeStartDelayRange[1])); }); }); }, Promise.resolve(allNodes)); var end = function end() { ended = true; cbInterval && clearInterval(cbInterval); return promise.then(function (nodes) { !silent && console.log('STOPPING NETWORK'); nodes.forEach(function (n) { return n.kill(); }); }); }; if (callbackInterval) { cbInterval = setInterval(function () { return onStats(stats); }, callbackInterval); } return { end: end, onPeersAdded: function onPeersAdded() { return promise; }, getStats: function getStats() { return stats; }, getNodeProcesses: function getNodeProcesses() { return allNodes; } }; } module.exports = { DEFAULT_OPTS: DEFAULT_OPTS, spawnMockedNetwork: spawnMockedNetwork }; ================================================ FILE: dist/simulation/node.js ================================================ 'use strict'; var cp = require('child_process'); var _require = require('../node/__mocks__/node'), Node = _require.Node; /** * Initializes a mocked node with given options * @param {object} options - see Node options for details. * @returns {Promise} */ function initMockedNode(options) { var node = new Node(options); return node.start().then(function (n) { n.log('initialized!'); return node; }); } /** * Spawns a node process * @param {object} options */ function spawnNode() { var options = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : {}; var silent = arguments[1]; var opts = []; options.port && opts.push('-p') && opts.push('' + options.port); options.isMaster && opts.push('--master'); options.IRIProtocol && opts.push('--IRIProtocol') && opts.push(options.IRIProtocol); options.neighbors && options.neighbors.length && opts.push('-n') && opts.push('' + options.neighbors.join(' ')); options.silent && opts.push('-s'); options.cycleInterval && opts.push('-c') && opts.push('' + options.cycleInterval); options.epochInterval && opts.push('-e') && opts.push('' + options.epochInterval); return cp.fork(__dirname + '/bin/nelson.js', opts, { silent: silent }); } module.exports = { spawnNode: spawnNode, initMockedNode: initMockedNode }; ================================================ FILE: package.json ================================================ { "name": "@semkodev/nelson.cli", "version": "0.4.1", "repository": { "type": "git", "url": "https://github.com/SemkoDev/nelson.cli.git" }, "description": "P2P manager for IOTA's IRI node", "main": "dist/index.js", "bin": { "nelson": "./dist/nelson.js" }, "files": [ "src", "dist", "README.md" ], "scripts": { "test": "jest -b", "build": "rimraf dist/ && babel ./src --out-dir dist/ --ignore ./node_modules --ignore __tests__ --copy-files", "make:binaries": "rimraf builds/ && pkg -t node6-linux,node6-win,node6-macos -o builds/nelson-`cat package.json | jq -r '.version'` dist/nelson.js", "make": "npm run test && npm run build && npm run make:binaries", "postversion": "git push --follow-tags" }, "keywords": [ "blockchain", "IOTA", "tangle", "p2p" ], "homepage": "https://semkodev.com", "author": "Roman Semko (http://twitter.com/RomanSemko)", "license": "ISC", "jest": { "testMatch": [ "**/__tests__/**/*-test.js?(x)" ], "roots": [ "src" ] }, "dependencies": { "blessed": "^0.1.81", "blessed-contrib": "^4.8.5", "body-parser": "^1.18.2", "colors": "^1.1.2", "commander": "^2.11.0", "express": "^4.16.2", "express-basic-auth": "^1.1.3", "external-ip": "^1.3.1", "helmet": "^3.10.0", "httpdispatcher": "^2.1.1", "ini": "^1.3.5", "iota.lib.js": "0.4.7", "ip": "^1.1.5", "json2csv": "^3.11.5", "md5": "^2.2.1", "moment": "^2.19.4", "moment-duration-format": "^2.0.1", "nedb": "^1.8.0", "request": "^2.83.0", "tmp": "^0.0.33", "weighted": "^0.3.0", "ws": "4.0.0" }, "devDependencies": { "babel-cli": "^6.26.0", "babel-preset-es2015": "^6.24.1", "babel-preset-stage-2": "^6.24.1", "jest": "^21.2.1", "pkg": "^4.3.0-beta.1", "rimraf": "^2.6.2" } } ================================================ FILE: src/api/__tests__/api-test.js ================================================ const request = require('request'); const { Node } = require('../../node/node'); const { createAPI } = require('../index'); jest.mock('../../node/iri'); const API_DATA = [ 'config', 'connectedPeers', 'heart', 'iriStats', 'isIRIHealthy', 'name', 'peerStats', 'ready', 'totalPeers','version' ]; describe('API', () => { it('should get node info correctly', (done) => { const node = new Node({ silent: true, temporary: true, port: 16601 }); const server = createAPI({ node, apiPort: 12345, apiHostname: 'localhost' }); node.start().then(() => { request.get('http://localhost:12345/', (err, resp, body) => { const answer = JSON.parse(body); const keys = Object.keys(answer); keys.sort(); expect(keys).toEqual(API_DATA); server.close(); node.end().then(done); }); }) }); it('should deny public access to protected when no password set', (done) => { const node = new Node({ silent: true, temporary: true, port: 16602 }); const server = createAPI({ node, apiPort: 12345, apiHostname: 'localhost', username: 'pass', password: 'pass' }); node.start().then(() => { request.get('http://localhost:12345/', (err, resp, body) => { expect(resp.statusCode).toEqual(401); expect(body).toBeFalsy; server.close(); node.end().then(done); }); }) }); it('should deny public access to protected when wrong pass', (done) => { const node = new Node({ silent: true, temporary: true, port: 16603 }); const server = createAPI({ node, apiPort: 12345, apiHostname: 'localhost', username: 'pass', password: 'pass' }); node.start().then(() => { request.get({ url: 'http://localhost:12345/', auth: { user: 'pass', pass: 'nopass' } }, (err, resp, body) => { expect(resp.statusCode).toEqual(401); expect(body).toBeFalsy; server.close(); node.end().then(done); }); }) }); it('should allow access to protected when auth ok', (done) => { const node = new Node({ silent: true, temporary: true, port: 16604 }); const server = createAPI({ node, apiPort: 12345, apiHostname: 'localhost', username: 'pass', password: 'pass' }); node.start().then(() => { request.get({ url: 'http://localhost:12345/', auth: { user: 'pass', pass: 'pass' } }, (err, resp, body) => { const answer = JSON.parse(body); const keys = Object.keys(answer); keys.sort(); expect(keys).toEqual(API_DATA); server.close(); node.end().then(done); }); }) }); it('should get peer stats info correctly', (done) => { const node = new Node({ silent: true, temporary: true, port: 16605 }); const server = createAPI({ node, apiPort: 12346, apiHostname: 'localhost' }); node.start().then(() => { request.get('http://localhost:12346/peer-stats', (err, resp, body) => { const summary = JSON.parse(body); expect(summary.newNodes).toBeTruthy; expect(summary.activeNodes).toBeTruthy; server.close(); node.end().then(done); }); }) }); it('should get peers info correctly', (done) => { const node = new Node({ silent: true, temporary: true, port: 16606 }); const server = createAPI({ node, apiPort: 12347, apiHostname: 'localhost' }); node.start().then(() => { request.get('http://localhost:12347/peers', (err, resp, body) => { const peers = JSON.parse(body); expect(Array.isArray(peers)).toBeTruthy; server.close(); node.end().then(done); }); }) }); }); ================================================ FILE: src/api/__tests__/node-test.js ================================================ const { Node } = require('../../node/node'); const { getSummary, getNodeStats } = require('../node'); jest.mock('../../node/iri'); const ALLOWED_DATA = [ 'config', 'connectedPeers', 'heart', 'iriStats', 'isIRIHealthy', 'name', 'peerStats', 'ready', 'totalPeers', 'version' ]; describe('API Node utils', () => { it('should correctly return summary', (done) => { const node = new Node({ silent: true, temporary: true, port: 16607 }); node.start().then(() => { const summary = getSummary(node); expect(summary.newNodes).toBeTruthy; expect(summary.activeNodes).toBeTruthy; node.end().then(done) }) }); it('should correctly return node stats', (done) => { const node = new Node({ silent: true, temporary: true, port: 16608 }); node.start().then(() => { const stats = getNodeStats(node); const keys = Object.keys(stats); keys.sort(); expect(keys).toEqual(ALLOWED_DATA); node.end().then(done) }) }) }); ================================================ FILE: src/api/__tests__/peer-test.js ================================================ const {Peer} = require('../../node/peer'); const {getPeerStats} = require('../peer'); const ALLOWED_DATA = [ 'IRIProtocol', 'TCPPort', 'UDPPort', 'connected', 'dateCreated', 'dateLastConnected', 'dateTried', 'hostname', 'ip', 'isTrusted', 'lastConnections', 'name', 'port', 'protocol', 'seen', 'tried', 'weight' ]; describe('API Peer utils', () => { it('should display only public data', () => { const stats = getPeerStats(new Peer()); const keys = Object.keys(stats); keys.sort(); expect(keys).toEqual(ALLOWED_DATA); }) }); ================================================ FILE: src/api/__tests__/webhooks-test.js ================================================ const express = require('express'); const bodyParser = require('body-parser'); const { Node } = require('../../node/node'); const { startWebhooks } = require('../webhooks'); jest.mock('../../node/iri'); const API_DATA = [ 'config', 'connectedPeers', 'heart', 'iriStats', 'isIRIHealthy', 'name', 'peerStats', 'ready', 'totalPeers','version' ]; describe('API Webhooks', () => { it('should start webhooks correctly', (done) => { const node = new Node({ silent: true, temporary: true, port: 16609 }); node.start().then(() => { const startDate = new Date(); const hook = startWebhooks(node, [ 'http://localhost:12348' ], 2); const app = express(); app.use(bodyParser.urlencoded({ extended: false })); app.use(bodyParser.json()); const server = app.listen(12348); app.post('/', (req) => { const timePassed = (new Date()) - startDate; const keys = Object.keys(req.body); keys.sort(); expect(keys).toEqual(API_DATA); expect(timePassed).toBeGreaterThanOrEqual(2000); expect(timePassed).toBeLessThanOrEqual(2100); server.close(); hook.stop(); node.end().then(done); }); }) }); }); ================================================ FILE: src/api/index.js ================================================ const express = require('express'); const helmet = require('helmet'); const bodyParser = require('body-parser'); const basicAuth = require('express-basic-auth'); const { getNodeStats, getSummary } = require('./node'); const { getPeerStats } = require('./peer'); const { startWebhooks } = require('./webhooks'); const DEFAULT_OPTIONS = { node: null, webhooks: [], webhookInterval: 30, username: null, password: null, apiPort: 18600, apiHostname: '127.0.0.1' }; /** * Creates an Express APP instance, also starts regular webhooks callbacks. * @param options * @returns {*|Function} */ function createAPI (options) { const opts = { ...DEFAULT_OPTIONS, ...options }; // Start webhook callbacks if (opts.webhooks && opts.webhooks.length) { startWebhooks(opts.node, opts.webhooks, opts.webhookInterval) } // Start API server const app = express(); app.set('node', opts.node); // Basic app protection app.use(helmet()); // Enable basic HTTP Auth if (opts.username && opts.password) { app.use(basicAuth({ users: { [opts.username]: opts.password } })) } // parse application/x-www-form-urlencoded app.use(bodyParser.urlencoded({ extended: false })); // parse application/json app.use(bodyParser.json()); //////////////////////// ENDPOINTS //////////////////////// app.get('/', (req, res) => { res.json(getNodeStats(opts.node)) }); app.get('/peer-stats', (req, res) => { res.json(getSummary(opts.node)) }); app.get('/peers', (req, res) => { res.json(opts.node.list.all().map(getPeerStats)) }); return app.listen(opts.apiPort, opts.apiHostname); } module.exports = { createAPI, DEFAULT_OPTIONS }; ================================================ FILE: src/api/node.js ================================================ const { getPeerStats } = require('./peer'); const version = require('../../package.json').version; /** * Returns summary of the node stats * @param {Node} node * @returns {{newNodes: {hourAgo, fourAgo, twelveAgo, dayAgo, weekAgo}, activeNodes: {hourAgo, fourAgo, twelveAgo, dayAgo, weekAgo}}} */ function getSummary (node) { const now = new Date(); const hour = 3600000; const hourAgo = new Date(now - hour); const fourAgo = new Date(now - (hour * 4)); const twelveAgo = new Date(now - (hour * 12)); const dayAgo = new Date(now - (hour * 24)); const weekAgo = new Date(now - (hour * 24 * 7)); return { newNodes: { hourAgo: node.list.all().filter(p => p.data.dateCreated >= hourAgo).length, fourAgo: node.list.all().filter(p => p.data.dateCreated >= fourAgo).length, twelveAgo: node.list.all().filter(p => p.data.dateCreated >= twelveAgo).length, dayAgo: node.list.all().filter(p => p.data.dateCreated >= dayAgo).length, weekAgo: node.list.all().filter(p => p.data.dateCreated >= weekAgo).length, }, activeNodes: { hourAgo: node.list.all().filter(p => p.data.dateLastConnected >= hourAgo).length, fourAgo: node.list.all().filter(p => p.data.dateLastConnected >= fourAgo).length, twelveAgo: node.list.all().filter(p => p.data.dateLastConnected >= twelveAgo).length, dayAgo: node.list.all().filter(p => p.data.dateLastConnected >= dayAgo).length, weekAgo: node.list.all().filter(p => p.data.dateLastConnected >= weekAgo).length, } } } /** * Returns clean node stats to be used in the API * @param {Node} node * @returns {{name, version, ready: (boolean|*|null), isIRIHealthy: (*|boolean), iriStats: *, peerStats: {newNodes: {hourAgo, fourAgo, twelveAgo, dayAgo, weekAgo}, activeNodes: {hourAgo, fourAgo, twelveAgo, dayAgo, weekAgo}}, totalPeers, connectedPeers: Array, config: {cycleInterval: (Command.opts.cycleInterval|*), epochInterval: (Command.opts.epochInterval|*), beatInterval: (Command.opts.beatInterval|*), dataPath: (Command.opts.dataPath|*), port: (Command.opts.port|*), apiPort: (Command.opts.apiPort|*), IRIPort: (Command.opts.IRIPort|*), TCPPort: (Command.opts.TCPPort|*), UDPPort: (Command.opts.UDPPort|*), IRIProtocol: (Command.opts.IRIProtocol|*), isMaster: (Command.opts.isMaster|*), temporary: (Command.opts.temporary|*)}, heart: {lastCycle: (heart.lastCycle|Heart.lastCycle|_require2.Heart.lastCycle), lastEpoch: (heart.lastEpoch|Heart.lastEpoch|_require2.Heart.lastEpoch), personality: (heart.personality|Heart.personality|_require2.Heart.personality), currentCycle: (heart.currentCycle|Heart.currentCycle|_require2.Heart.currentCycle), currentEpoch: (heart.currentEpoch|Heart.currentEpoch|_require2.Heart.currentEpoch), startDate: (heart.startDate|Heart.startDate|_require2.Heart.startDate)}}} */ function getNodeStats (node) { const { cycleInterval, epochInterval, beatInterval, dataPath, port, apiPort, IRIPort, TCPPort, UDPPort, isMaster, IRIProtocol, temporary } = node.opts; const { lastCycle, lastEpoch, personality, currentCycle, currentEpoch, startDate } = node.heart; const totalPeers = node.list.all().length; const isIRIHealthy = node.iri && node.iri.isHealthy; const iriStats = node.iri && node.iri.iriStats; const connectedPeers = Array.from(node.sockets.keys()) .filter((p) => node.sockets.get(p).readyState === 1) .map(getPeerStats); return { name: node.opts.name, version, ready: node._ready, isIRIHealthy, iriStats, peerStats: getSummary(node), totalPeers, connectedPeers, config: { cycleInterval, epochInterval, beatInterval, dataPath, port, apiPort, IRIPort, TCPPort, UDPPort, IRIProtocol, isMaster, temporary }, heart: { lastCycle, lastEpoch, personality, currentCycle, currentEpoch, startDate } } } module.exports = { getSummary, getNodeStats }; ================================================ FILE: src/api/peer.js ================================================ /** * Returns a clean Peer object that can be used in the API * @param {Peer} peer * @returns {{name, hostname, ip, port, TCPPort, UDPPort, protocol, IRIProtocol, seen, connected, tried, weight, dateTried, dateLastConnected, dateCreated, isTrusted, lastConnections}} */ function getPeerStats (peer) { const { name, hostname, ip, port, TCPPort, UDPPort, protocol, seen, connected, tried, weight, dateTried, dateLastConnected, dateCreated, IRIProtocol, isTrusted, lastConnections } = peer.data; return { name, hostname, ip, port, TCPPort, UDPPort, protocol, IRIProtocol, seen, connected, tried, weight, dateTried, dateLastConnected, dateCreated, isTrusted, lastConnections } } module.exports = { getPeerStats }; ================================================ FILE: src/api/utils.js ================================================ ================================================ FILE: src/api/webhooks.js ================================================ const request = require('request'); const { getNodeStats } = require('./node'); function startWebhooks (node, webhooks, webhookInterval) { const interval = setInterval(() => { webhooks.forEach((uri) => request({ uri, method: 'POST', json: getNodeStats(node) }, (err) => { if (err) { node.log(`Webhook returned error: ${uri}`.yellow); } })); }, webhookInterval * 1000); return { stop: () => { clearInterval(interval) } } } module.exports = { startWebhooks }; ================================================ FILE: src/index.js ================================================ require('colors'); const request = require('request'); const terminal = require('./node/tools/terminal'); const node = require('./node').node; const api = require('./api/index'); const utils = require('./node').utils; // Some general TODOs: // TODO: add linting // TODO: add editor config module.exports = { initNode: (opts) => { const init = (options) => { const _node = new node.Node(options); const terminate = () => _node.end().then( () => { process.exit(0); } ); process.on('SIGINT', terminate); process.on('SIGTERM', terminate); opts.gui && terminal.init(opts.name, utils.getVersion(), terminate); _node.start().then((n) => { api.createAPI({ node: n, webhooks: opts.webhooks, webhookInterval: opts.webhookInterval, apiPort: opts.apiPort, apiHostname: opts.apiHostname, username: opts.apiAuth && opts.apiAuth.username, password: opts.apiAuth && opts.apiAuth.password, }); terminal.ports(n.opts); n.log(`Nelson v.${utils.getVersion()} initialized`.green.bold); }); }; if (opts.getNeighbors) { if (typeof opts.getNeighbors === 'boolean') { opts.getNeighbors = 'https://raw.githubusercontent.com/SemkoDev/nelson.cli/master/ENTRYNODES' } let neighbors = []; request(opts.getNeighbors, (err, resp, body) => { if (err) { throw err } neighbors = body.split('\n').map((str) => { if (!str || !str.length) { return null; } if (utils.validNeighbor(str)) { console.log('Downloaded entry neighbor:', str); return str; } else { console.log('Wrong entry neighbor format:', str); return null; } }).filter(n => n); opts.neighbors = [ ...(opts.neighbors ? opts.neighbors : []), ...neighbors ]; init(opts); }); } else { init(opts); } }, ...node }; ================================================ FILE: src/nelson.js ================================================ #!/usr/bin/env node require('colors'); const ini = require('ini'); const fs = require('fs'); const { URL } = require('url'); const program = require('commander'); const { initNode } = require('./index'); const { DEFAULT_OPTIONS } = require('./node/node'); const { PROTOCOLS } = require('./node/peer'); const { DEFAULT_OPTIONS: DEFAULT_LIST_OPTIONS } = require('./node/peer-list'); const { DEFAULT_OPTIONS: DEFAULT_API_OPTIONS } = require('./api/index'); const version = require('../package.json').version; const parseNeighbors = (val) => val.split(' '); const parseURLs = (val) => val.split(' ').map((v) => new URL(v)).map((u) => u.href); const parseProtocol = (val) => { const lower = val.toLowerCase(); return PROTOCOLS.includes(lower) ? lower : DEFAULT_OPTIONS.IRIProtocol }; const parseNumber = (v) => parseInt(v); const parseAuth = (v) => { const tokens = v.split(':'); if(!tokens.length === 2) { throw new Error('Wrong apiAuth format! Use: "username.password"'); } if (!tokens[0].length) { throw new Error('apiAuth username not provided!'); } if(!tokens[1].length) { throw new Error('apiAuth password not provided!'); } return { username: tokens[0], password: tokens[1]} }; program .version(version) .option('--name [value]', 'Name of your node instance', DEFAULT_OPTIONS.name) .option('-n, --neighbors [value]', 'Trusted neighbors', parseNeighbors, []) .option('--getNeighbors [url]', 'Download default set of neighbors', false) .option('-c, --cycleInterval [value]', 'Cycle interval in seconds', parseNumber, DEFAULT_OPTIONS.cycleInterval) .option('-e, --epochInterval [value]', 'Epoch interval in seconds', parseNumber, DEFAULT_OPTIONS.epochInterval) .option('--incomingMax [value]', 'Maximal incoming connection slots', parseNumber, DEFAULT_OPTIONS.incomingMax) .option('--outgoingMax [value]', 'Maximal outgoing connection slots', parseNumber, DEFAULT_OPTIONS.outgoingMax) .option('--lazyLimit [value]', 'Seconds after which neighbor is dropped for not having provided any new TXs', parseNumber, DEFAULT_OPTIONS.lazyLimit) .option('--lazyTimesLimit [value]', 'How many consecutive times a lazy neighbor can connect before getting penalized', parseNumber, DEFAULT_OPTIONS.lazyTimesLimit) .option('--apiAuth [value]', 'Nelson API username:password', parseAuth, null) .option('-a, --apiPort [value]', 'Nelson API port', parseNumber, DEFAULT_API_OPTIONS.apiPort) .option('-o, --apiHostname [value]', 'Nelson API hostname', DEFAULT_API_OPTIONS.apiHostname) .option('-w, --webhooks [value]', 'Nelson API webhook URLs', parseURLs, DEFAULT_API_OPTIONS.webhooks) .option('--webhookInterval [value]', 'Webhooks callback interval in seconds', parseNumber, DEFAULT_API_OPTIONS.webhookInterval) .option('-p, --port [value]', 'Nelson port', parseNumber, DEFAULT_OPTIONS.port) .option('-r, --IRIHostname [value]', 'IRI API hostname', DEFAULT_OPTIONS.IRIHostname) .option('-i, --IRIPort [value]', 'IRI API port', parseNumber, DEFAULT_OPTIONS.IRIPort) .option('-t, --TCPPort [value]', 'IRI TCP port', parseNumber, DEFAULT_OPTIONS.TCPPort) .option('-u, --UDPPort [value]', 'IRI UDP port', parseNumber, DEFAULT_OPTIONS.UDPPort) .option('--IRIProtocol [value]', 'IRI protocol to use: udp, tcp, prefertcp, preferudp or any', parseProtocol, DEFAULT_OPTIONS.IRIProtocol) .option('-d, --dataPath [value]', 'Peer database path', DEFAULT_LIST_OPTIONS.dataPath) .option('-m, --isMaster [value]', 'Is a master node', false) .option('-s, --silent [value]', 'Silent', false) .option('-g, --gui [value]', 'GUI', false) .option('--temporary [value]', 'Create a temporary node', false) .option('--config [value]', 'Config file path', null) .parse(process.argv); const configPath = process.env.NELSON_CONFIG || program.config; initNode(configPath ? ini.parse(fs.readFileSync(configPath, 'utf-8')).nelson : program); ================================================ FILE: src/node/__mocks__/iri.js ================================================ const req = require.requireActual ? require.requireActual : require; const { IRI: BaseIRI, DEFAULT_OPTIONS } = req('../iri'); /** * Class responsible to RUN and communicate with local IRI instance * @class */ class IRI extends BaseIRI { /** * Starts the IRI process, returning self on success. * @returns {Promise} */ start () { return new Promise((resolve) => { this._isStarted = true; this.isHealthy = true; this.ticker = setInterval(this._tick, 15000); this.getStats().then(() => resolve(this)); }) } /** * Removes a list of neighbors from IRI, except static neighbors. Returns list of removed peers. * @param {Peer[]} peers * @returns {Promise} */ removeNeighbors (peers) { if (!this.isAvailable()) { return Promise.reject(); } return new Promise ((resolve) => { resolve(peers) }); } /** * Adds a list of peers to IRI. * @param {Peer[]} peers * @returns {Promise} */ addNeighbors (peers) { if (!this.isAvailable()) { return Promise.reject(); } return new Promise((resolve) => { resolve(peers); }); } /** * Cleans up any orphans from the IRI * @param {Peer[]} peers * @returns {Promise} */ cleanupNeighbors (peers) { if (!this.isAvailable()) { return Promise.reject(); } return new Promise((resolve) => { resolve([]); }); } /** * Updates the list of neighbors at the IRI backend. Removes all neighbors, replacing them with * the newly provided neighbors. * @param {Peer[]} peers * @returns {Promise} */ updateNeighbors (peers) { if (!this.isAvailable()) { return Promise.reject(); } if (!peers || !peers.length) { return Promise.resolve([]); } return new Promise((resolve, reject) => { const addNeighbors = () => { this.addNeighbors(peers).then(resolve).catch(reject); }; addNeighbors(); }); } /** * Removes all IRI neighbors, except static neighbors. * @returns {Promise} */ removeAllNeighbors () { if (!this.isAvailable()) { return Promise.reject(); } return new Promise((resolve) => { resolve(); }); } /** * Returns IRI node info * @returns {Promise} */ getStats () { return new Promise((resolve) => { this.iriStats = { mock: true }; resolve(this.iriStats); }); } _tick () { const { onHealthCheck } = this.opts; this.getStats().then(() => { onHealthCheck(true, []); }); } } IRI.isMocked = true; module.exports = { IRI, DEFAULT_OPTIONS }; ================================================ FILE: src/node/__mocks__/node.js ================================================ const { Node: BaseNode, DEFAULT_OPTIONS: DEFAULT_NODE_OPTIONS } = require('../node'); const { getRandomInt } = require('../tools/utils'); const { IRI } = require('./iri'); const DEFAULT_OPTIONS = { ...DEFAULT_NODE_OPTIONS, localNodes: true, beatInterval: 2, cycleInterval: 3, epochInterval: 30, lazyLimit: 6, testnet: true, temporary: true, }; /** * This is a mock for the "real" node. What it does are several things: * * 1. Mock away IRI backend so we do not start it. We just want to test the P2P functionality. * 2. Create a separate neighbor database for each node. * 3. Report stats to the parent process * * @class Node */ class Node extends BaseNode { constructor (options) { super({ ...DEFAULT_OPTIONS, ...options }); this.sendStats = this.sendStats.bind(this); setInterval(this.sendStats, 1000); } _getIRI () { const { APIPort, TCPPort, UDPPort, testnet, silent, temporary } = this.opts; return (new IRI({ APIPort, TCPPort, UDPPort, testnet, silent, temporary, logIdent: `${this.opts.port}::IRI` })).start().then((iri) => { this.iri = iri; return iri; }) } _setPublicIP () { this.ipv4 = 'localhost'; return Promise.resolve(0); } _onIRIHealth () { Array.from(this.sockets.keys()).forEach((peer) => { peer.updateConnection({ numberOfAllTransactions: getRandomInt(0, 1000), numberOfNewTransactions: getRandomInt(0, 150), numberOfRandomTransactionRequests: getRandomInt(0, 100), numberOfInvalidTransactions: getRandomInt(0, 10) }); }) } /////////////////////////////////// MOCK SPECIFICS /////////////////////////////////// sendStats () { const sockets = Array.from(this.sockets.values()); process.send({ isMaster: this.opts.isMaster, peers: this.list ? this.list.all().map((p) => p.data.port) : [], connections: { list: Array.from(this.sockets.keys()).filter(k => this.sockets.get(k).readyState === 1).map( (peer) => `${peer.data.port}` ), connecting: sockets.filter(s => s.readyState === 0).length, connected: sockets.filter(s => s.readyState === 1).length, closed: sockets.filter(s => s.readyState > 1).length } }); } } module.exports = { DEFAULT_OPTIONS, Node }; ================================================ FILE: src/node/__tests__/guard-test.js ================================================ const { Guard } = require('../guard'); describe('Guard', () => { it('should guard correctly', (done) => { const guard = new Guard(); expect(guard.isAllowed('localhost')).toBeTruthy; expect(guard.isAllowed('localhost')).toBeFalsy; setTimeout(() => { expect(guard.isAllowed('localhost')).toBeTruthy; expect(guard.isAllowed('localhost')).toBeFalsy; setTimeout(() => { expect(guard.isAllowed('localhost')).toBeFalsy; done(); }, 1001); }, 2001); }); }); ================================================ FILE: src/node/__tests__/heart-test.js ================================================ const { Heart } = require('../heart'); jasmine.DEFAULT_TIMEOUT_INTERVAL = 10000; describe('Heart', () => { it('doesnt tick, if not started', (done) => { const heart = new Heart({ cycleInterval: 1, epochInterval: 3, silent: true }); setTimeout(() => { expect(heart.personality.id).toBeFalsy; done(); }, 4000); }, 5000); it('ticks when started', (done) => { const heart = new Heart({ cycleInterval: 1, epochInterval: 3, silent: true, autoStart: true }); setTimeout(() => { expect(heart.personality.id).toBeTruthy; done(); }, 2000); }, 3000); it('ticks when started #2', (done) => { const heart = new Heart({ cycleInterval: 1, epochInterval: 3, silent: true }); heart.start(); setTimeout(() => { expect(heart.personality.id).toBeTruthy; done(); }, 2000); }, 3000); /* it('updates personality correctly', (done) => { const heart = new Heart({ cycleInterval: 1, epochInterval: 2, silent: true, autoStart: true }); const p1 = heart.personality; setTimeout(() => { const p2 = heart.personality; expect(p1).not.toEqual(p2); setTimeout(() => { const p3 = heart.personality; expect(p3).not.toEqual(p2); expect(p3).not.toEqual(p1); done(); }, 2100); }, 2100); }, 10000); */ it('Does not update personality, if epoch off', (done) => { const heart = new Heart({ cycleInterval: 1, epochInterval: 2, silent: true, autoStart: true, onEpoch: () => Promise.resolve(true) }); const p1 = heart.personality; setTimeout(() => { const p2 = heart.personality; expect(p1).toEqual(p2); setTimeout(() => { const p3 = heart.personality; expect(p3).toEqual(p2); done(); }, 2100); }, 2100); }, 7000); it('Does not update personality #2, if cycle off', (done) => { const heart = new Heart({ cycleInterval: 1, epochInterval: 2, silent: true, autoStart: true, onCycle: () => Promise.resolve(true) }); const p1 = heart.personality; setTimeout(() => { const p2 = heart.personality; expect(p1).toEqual(p2); setTimeout(() => { const p3 = heart.personality; expect(p3).toEqual(p2); done(); }, 2100); }, 2100); }, 7000); }); ================================================ FILE: src/node/__tests__/node-test.js ================================================ const { Node } = require('../node'); const { IRI } = require('../iri'); jest.mock('../iri'); const DEFAULT_OPTIONS = { localNodes: true, beatInterval: 2, cycleInterval: 3, epochInterval: 30, lazyLimit: 6, testnet: true, temporary: true, }; describe('Node', () => { it('should mock IRI correctly', () => { expect(IRI.isMocked).toBeTruthy; }); it('should initialize Node correctly', (done) => { const node = new Node({ ...DEFAULT_OPTIONS, silent: true, port: 16610 }); node.start().then((n) => { expect(n.iri && n.iri.isAvailable()).toBeTruthy; expect(n.heart && n.heart.personality && n.heart.personality.id).toBeTruthy; node.end().then(done); }).catch(done); }); }); ================================================ FILE: src/node/__tests__/peer-list-test.js ================================================ const dns = require("dns"); const { PeerList } = require("../peer-list"); jasmine.DEFAULT_TIMEOUT_INTERVAL = 20000; describe("PeerListTest", () => { it("should create a list correctly", done => { const list = new PeerList({ temporary: true, silent: true }); list.load([ "somehost.com/1234/345/567", "122.232.223.0/14265/11111/22222" ]).then(() => { expect(list.peers[0].data.key).toBeTruthy; expect(list.peers[1].data.key).toBeTruthy; expect(list.peers).toHaveLength(2); expect(list.peers.map(p => p.data.hostname).sort()).toEqual( ["somehost.com", "122.232.223.0"].sort() ); expect(list.peers.map(p => p.data.port).sort()).toEqual( [1234, 14265].sort() ); done(); }); }); it("should add to a list correctly", done => { const list = new PeerList({ temporary: true, silent: true }); list.load([ "somehost.com/1234/345/567", "122.232.223.0/14265/11111/22222" ]).then(() => { expect(list.peers).toHaveLength(2); expect(list.peers.map(p => p.data.hostname).sort()).toEqual( ["somehost.com", "122.232.223.0"].sort() ); list.add({ hostname: "some-other-peer.org", port: 334, TCPPort: 335, UDPPort: 336 }).then(() => { expect(list.peers[0].data.key).toBeTruthy; expect(list.peers[1].data.key).toBeTruthy; expect(list.peers[2].data.key).toBeTruthy; expect(list.peers.map(p => p.data.hostname).sort()).toEqual( [ "somehost.com", "122.232.223.0", "some-other-peer.org" ].sort() ); expect(list.peers.map(p => p.data.port).sort()).toEqual( [334, 1234, 14265].sort() ); done(); }); }); }); it("should return all peers", done => { const list = new PeerList({ temporary: true, silent: true }); list.load([ "somehost.com/1234/345/567", "122.232.223.0/14265/11111/22222" ]).then(() => { expect(list.all()).toHaveLength(2); done(); }); }); it("should return average age correctly", done => { const list = new PeerList({ temporary: true, silent: true }); list.load([ "somehost.com/1234/345/567", "122.232.223.0/14265/11111/22222" ]).then(() => { setTimeout(() => { list.add({ hostname: "somehost2.com", port: 1234, TCPPort: 666, UDPPort: 777 }).then(() => { expect(list.peers).toHaveLength(3); expect(list.getAverageAge()).toBeGreaterThan(2.6); done(); }); }, 4000); }); }); it("should update the list correctly", done => { const list = new PeerList({ temporary: true, silent: true }); list.load([ "somehost.com/1234/345/567", "122.232.223.0/14265/11111/22222" ]).then(() => { list.add({ hostname: "somehost.com", port: 1234, TCPPort: 666, UDPPort: 777 }).then(() => { expect(list.peers).toHaveLength(2); expect(list.all().sort()[1].data.TCPPort).toEqual(666); done(); }); }); }); it("should update the list correctly by key", done => { const list = new PeerList({ temporary: true, silent: true }); list.load([ "somehost.com/1234/345/567", "122.232.223.0/14265/11111/22222" ]).then(() => { list.add({ hostname: "somehost.com", port: 1234, TCPPort: 666, UDPPort: 777, remoteKey: "213213" }).then(() => { list.add({ hostname: "someanotherhost.com", port: 1234, TCPPort: 668, UDPPort: 777, remoteKey: "213213" }).then(() => { expect(list.peers).toHaveLength(2); expect(list.all().sort()[1].data.TCPPort).toEqual(668); done(); }); }); }); }); it("should add to the list correctly, same host, when multiPort allowed", done => { const list = new PeerList({ temporary: true, multiPort: true, silent: true }); list.load([ "somehost.com/1234/345/567", "122.232.223.0/14265/11111/22222" ]).then(() => { list.add({ hostname: "somehost.com", port: 12345, TCPPort: 333, UDPPort: 444 }).then(() => { expect(list.peers).toHaveLength(3); expect(list.all()[0].data.port).toEqual(1234); expect(list.all()[2].data.port).toEqual(12345); done(); }); }); }); it("should update a peer correctly", done => { const list = new PeerList({ temporary: true, silent: true }); list.load([ "somehost.com/1234/345/567", "122.232.223.0/14265/11111/22222" ]).then(() => { const peer = list.all()[1]; const data = { ...peer.data, port: 6789 }; list.update(peer, { port: 6789 }).then(() => { expect(list.all()[1].data).toEqual(data); done(); }); }); }); it("should find in list correctly", done => { const list = new PeerList({ temporary: true, silent: true }); list.load([ "somehost.com/1234/345/567", "122.232.223.0/14265/11111/22222" ]).then(() => { list.findByAddress("somehost.com", 2345).then(peers => { expect(peers).toHaveLength(1); done(); }); }); }); it("should find in list with unknown remote key, but existing address", done => { const list = new PeerList({ temporary: true, silent: true }); list.load([ "somehost.com/1234/345/567", "122.232.223.0/14265/11111/22222" ]).then(() => { list.findByRemoteKeyOrAddress("abcdef", "somehost.com", 2345).then( peers => { expect(peers).toHaveLength(1); done(); } ); }); }); it("should find in list with known remote key, but unknown address", done => { const list = new PeerList({ temporary: true, silent: true }); list.add({ hostname: "somehost.com", port: 12345, TCPPort: 333, UDPPort: 444, remoteKey: "abcdef" }).then(() => { list.findByRemoteKeyOrAddress( "abcdef", "unknownhost.com", 2345 ).then(peers => { expect(peers).toHaveLength(1); expect(peers[0].data.hostname).toEqual( list.all()[0].data.hostname ); done(); }); }); }); it("should not find in list with unknown remote key and address", done => { const list = new PeerList({ temporary: true, silent: true }); list.add({ hostname: "somehost.com", port: 12345, TCPPort: 333, UDPPort: 444, remoteKey: "abcdef" }).then(() => { list.findByRemoteKeyOrAddress("xyz", "unknownhost.com", 2345).then( peers => { expect(peers).toHaveLength(0); done(); } ); }); }); it("should not find in list correctly, multiPort + different ports", done => { const list = new PeerList({ temporary: true, multiPort: true, silent: true }); list.load([ "somehost.com/1234/345/567", "122.232.223.0/14265/11111/22222" ]).then(() => { list.findByAddress("somehost.com", 2345).then(peers => { expect(peers).toHaveLength(0); done(); }); }); }); it("should find in list correctly by using IP", done => { const list = new PeerList({ temporary: true, multiPort: false, silent: true }); list.load([ "deviota.com/1234/345/567", "122.232.223.0/14265/11111/22222", "iota.org/123/345/546" ]).then(() => { Promise.all(list.peers.map(p => p.getIP())).then(() => { dns.resolve("deviota.com", "A", (error, results) => { const ip = error || !results.length ? null : results[0]; if (ip) { list.findByAddress(ip, 2345).then(peers => { expect(peers).toHaveLength(1); done(); }); } else { done(); } }); }); }); }); it("should return correct peer trusts", done => { const list = new PeerList({ temporary: true, ageNormalizer: 60, silent: true }); list.load([ "somehost.com/1234/345/567", "122.232.223.0/14265/11111/22222" ]).then(() => { setTimeout(() => { list.add({ hostname: "some-other-peer.org", port: 334, TCPPort: 335, UDPPort: 336 }).then(() => { setTimeout(() => { list.add({ hostname: "some-other-peer2.org", port: 334, TCPPort: 335, UDPPort: 336 }).then(() => { expect( list.getPeerTrust(list.peers[0]) ).toBeGreaterThan(0.5); expect( list.getPeerTrust(list.peers[1]) ).toBeGreaterThan(0.5); expect( list.getPeerTrust(list.peers[2]) ).toBeGreaterThan(0.0001); expect( list.getPeerTrust(list.peers[2]) ).toBeLessThan(0.01); expect( list.getPeerTrust(list.peers[3]) ).toBeGreaterThanOrEqual(0.0001); expect( list.getPeerTrust(list.peers[3]) ).toBeLessThan(0.001); done(); }); }, 1000); }); }, 3000); }); }); it( "should return correct peer weights", done => { const list = new PeerList({ temporary: true, ageNormalizer: 60, silent: true }); list.load([ "somehost.com/1234/345/567", "122.232.223.0/14265/11111/22222" ]).then(() => { setTimeout(() => { list.add({ hostname: "some-other-peer.org", port: 334, TCPPort: 335, UDPPort: 336 }).then(() => { setTimeout(() => { list.add({ hostname: "some-other-peer2.org", port: 334, TCPPort: 335, UDPPort: 336 }).then(() => { const weights = list .getWeighted() .map(w => w[1]) .sort(); weights.reverse(); expect(weights[0]).toEqual(1); expect(weights[1]).toEqual(1); expect(weights[2]).toBeCloseTo(0.002, 3); expect(weights[3]).toBeCloseTo(0.000075, 4); done(); }); }, 3000); }); }, 3000); }); }, 12000 ); }); ================================================ FILE: src/node/__tests__/peer-test.js ================================================ const dns = require("dns"); const { Peer, DEFAULT_PEER_DATA } = require("../peer"); jasmine.DEFAULT_TIMEOUT_INTERVAL = 10000; describe("Peer", () => { it("should add default peer data", () => { const peer = new Peer(); expect(peer.data).toEqual(DEFAULT_PEER_DATA); }); it("should add non-default data to peer", () => { const peer = new Peer({ hostname: "tangle.com", port: 666 }); expect(peer.data.hostname).toEqual("tangle.com"); expect(peer.data.port).toEqual(666); }); it("should return correct tcp, http and hostname strings", () => { const peer = new Peer({ hostname: "tangle.com", port: 666, TCPPort: 777, UDPPort: 888 }); expect(peer.getTCPURI()).toEqual("tcp://tangle.com:777"); expect(peer.getUDPURI()).toEqual("udp://tangle.com:888"); expect(peer.getNelsonURI()).toEqual("http://tangle.com:666"); expect(peer.getNelsonWebsocketURI()).toEqual("ws://tangle.com:666"); expect(peer.getHostname()).toEqual("tangle.com/666/777/888/0/udp"); }); it("should update the peer data", () => { const peer = new Peer({ hostname: "tangle.com", port: 666 }); expect(peer.data.hostname).toEqual("tangle.com"); peer.update({ hostname: "iota.org" }); expect(peer.data.hostname).toEqual("iota.org"); expect(peer.data.port).toEqual(666); }); it("should reset IP if hostname changed", () => { const peer = new Peer({ hostname: "tangle.com", port: 666, ip: "123.123.123.123" }); expect(peer.data.ip).toEqual("123.123.123.123"); peer.update({ hostname: "iota.org" }); expect(peer.data.ip).toEqual(null); }); it("should return a resolved ip if hostname is an ip", done => { const peer = new Peer({ hostname: "192.168.0.1", port: 666 }); peer.getIP().then(ip => { expect(ip).toEqual("192.168.0.1"); done(); }); }); it("should return a resolved ip if hostname is not an ip", done => { const peer = new Peer({ hostname: "deviota.com", port: 666 }); dns.resolve(peer.data.hostname, "A", (error, results) => { peer.getIP().then(ip => { expect(ip).toEqual(results[0]); done(); }); }); }); it("should compare an ip-hostname correctly", done => { const peer = new Peer({ hostname: "192.168.0.1", port: 666 }); peer.isSameIP("192.168.0.1").then(result => { expect(result).toBeTruthy; done(); }); }); it("should compare an ip-hostname correctly #2", done => { const peer = new Peer({ hostname: "192.168.0.1", port: 666 }); peer.isSameIP("192.168.0.2").then(result => { expect(result).toBeFalsy; done(); }); }); it("should compare a hostname and ip correctly", done => { const peer = new Peer({ hostname: "deviota.com", port: 666 }); dns.resolve(peer.data.hostname, "A", (error, results) => { peer.isSameIP(results[0]).then(result => { expect(result).toBeTruthy; done(); }); }); }); it( "should record connection data", done => { const peer = new Peer({ hostname: "tangle.com", port: 666, TCPPort: 777, UDPPort: 888 }); recordPeerConnection(peer, 3).then(() => { expect(peer.data.lastConnections).toHaveLength(1); done(); }); }, 4000 ); it( "should record multiple connection data", done => { const peer = new Peer({ hostname: "tangle.com", port: 666, TCPPort: 777, UDPPort: 888 }); const durations = [1, 4, 2]; const datas = [null, null, null]; recordPeerConnections(peer, durations, datas).then(() => { expect(peer.data.lastConnections).toHaveLength(3); done(); }); }, 8000 ); it("should mark a peer as lazy", done => { const peer = new Peer( { hostname: "tangle.com", port: 666, TCPPort: 777, UDPPort: 888 }, { lazyLimit: 3 } ); peer.markConnected().then(() => { peer.updateConnection({ numberOfAllTransactions: 0, numberOfRandomTransactionRequests: 0, numberOfNewTransactions: 0, numberOfInvalidTransactions: 0 }); setTimeout(() => { expect(peer.isLazy()).toBeFalsy; setTimeout(() => { expect(peer.isLazy()).toBeTruthy; done(); }, 2100); }, 1000); }); }); it("should correctly calculate quality #1", done => { const peer = new Peer({ hostname: "tangle.com", port: 666, TCPPort: 777, UDPPort: 888 }); const durations = [1, 1, 1]; const datas = [ { numberOfAllTransactions: 0, numberOfRandomTransactionRequests: 0, numberOfNewTransactions: 0, numberOfInvalidTransactions: 0 }, { numberOfAllTransactions: 0, numberOfRandomTransactionRequests: 0, numberOfNewTransactions: 0, numberOfInvalidTransactions: 0 }, { numberOfAllTransactions: 0, numberOfRandomTransactionRequests: 0, numberOfNewTransactions: 0, numberOfInvalidTransactions: 0 } ]; recordPeerConnections(peer, durations, datas).then(() => { expect(peer.getPeerQuality()).toBeCloseTo(0.3333, 4); done(); }); }); it("should correctly calculate quality #2", done => { const peer = new Peer({ hostname: "tangle.com", port: 666, TCPPort: 777, UDPPort: 888 }); const durations = [1, 1]; const datas = [ { numberOfAllTransactions: 0, numberOfRandomTransactionRequests: 0, numberOfNewTransactions: 0, numberOfInvalidTransactions: 0 }, { numberOfAllTransactions: 0, numberOfRandomTransactionRequests: 0, numberOfNewTransactions: 0, numberOfInvalidTransactions: 0 } ]; recordPeerConnections(peer, durations, datas).then(() => { expect(peer.getPeerQuality()).toBeCloseTo(1.0, 4); done(); }); }); it("should correctly calculate quality #3", done => { const peer = new Peer({ hostname: "tangle.com", port: 666, TCPPort: 777, UDPPort: 888 }); const durations = [1, 1, 1, 1, 1]; const datas = [ { numberOfAllTransactions: 0, numberOfRandomTransactionRequests: 0, numberOfNewTransactions: 0, numberOfInvalidTransactions: 0 }, { numberOfAllTransactions: 0, numberOfRandomTransactionRequests: 0, numberOfNewTransactions: 0, numberOfInvalidTransactions: 0 }, { numberOfAllTransactions: 0, numberOfRandomTransactionRequests: 0, numberOfNewTransactions: 0, numberOfInvalidTransactions: 0 }, { numberOfAllTransactions: 0, numberOfRandomTransactionRequests: 0, numberOfNewTransactions: 0, numberOfInvalidTransactions: 0 }, { numberOfAllTransactions: 0, numberOfRandomTransactionRequests: 0, numberOfNewTransactions: 0, numberOfInvalidTransactions: 0 } ]; recordPeerConnections(peer, durations, datas).then(() => { expect(peer.getPeerQuality()).toBeCloseTo(0.2, 4); done(); }); }); it("should correctly calculate quality #4", done => { const peer = new Peer({ hostname: "tangle.com", port: 666, TCPPort: 777, UDPPort: 888 }); const durations = [1, 1, 1, 1, 1]; const datas = [ { numberOfAllTransactions: 0, numberOfRandomTransactionRequests: 0, numberOfNewTransactions: 0, numberOfInvalidTransactions: 0 }, { numberOfAllTransactions: 0, numberOfRandomTransactionRequests: 0, numberOfNewTransactions: 0, numberOfInvalidTransactions: 0 }, { numberOfAllTransactions: 0, numberOfRandomTransactionRequests: 0, numberOfNewTransactions: 0, numberOfInvalidTransactions: 0 }, { numberOfAllTransactions: 0, numberOfRandomTransactionRequests: 0, numberOfNewTransactions: 0, numberOfInvalidTransactions: 0 }, { numberOfAllTransactions: 0, numberOfRandomTransactionRequests: 0, numberOfNewTransactions: 10, numberOfInvalidTransactions: 2 } ]; recordPeerConnections(peer, durations, datas).then(() => { expect(peer.getPeerQuality()).toBeCloseTo(1, 4); done(); }); }); }); function recordPeerConnections(peer, durations, datas) { expect(durations.length).toEqual(datas.length); return durations.reduce( (promise, duration, index) => promise.then(() => recordPeerConnection(peer, duration, datas[index]) ), Promise.resolve() ); } function recordPeerConnection(peer, duration, data) { return new Promise(resolve => { peer.markConnected().then(() => { peer.updateConnection( data || { numberOfAllTransactions: 0, numberOfRandomTransactionRequests: 0, numberOfNewTransactions: 0, numberOfInvalidTransactions: 0 } ); setTimeout(() => { expect(peer.getConnectionDuration()).toBeCloseTo(duration, 1); peer.markDisconnected().then(() => { resolve(peer); }); }, duration * 1000); }); }); } ================================================ FILE: src/node/base.js ================================================ require('colors'); const terminal = require('./tools/terminal'); const DEFAULT_OPTIONS = { silent: false, logIdent: 'BASE', logIdentWidth: 12, }; /** * Base class with generic functionality. * @class Base */ class Base { constructor (options) { this.opts = { ...DEFAULT_OPTIONS, ...options }; } log () { if (!this.opts || !this.opts.silent || arguments[0] === '!!') { const date = new Date(); const timeString = `${date.toLocaleTimeString()}.${this.formatMilliseconds(date.getMilliseconds())}`.dim; const space = this.opts.logIdent.length > this.opts.logIdentWidth ? `\n${' '.repeat(this.opts.logIdentWidth)}` : ' '.repeat(this.opts.logIdentWidth - this.opts.logIdent.length); const logIdent = `${this.opts.logIdent}${space}`.dim.bold; terminal.log(`${timeString}\t${logIdent}`, ...arguments); } } formatNode (hostname, port) { return `${hostname}:${port}`.cyan } formatMilliseconds(milliseconds){ var formatted = milliseconds / 1000; formatted = formatted.toFixed(3); formatted = formatted.toString(); return formatted.slice(2); } start () {} end () {} } module.exports = { DEFAULT_OPTIONS, Base }; ================================================ FILE: src/node/guard.js ================================================ const { Base } = require('./base'); const { getSecondsPassed } = require('./tools/utils'); const DEFAULT_OPTIONS = { beatInterval: 1, throttleInterval: 2, // Minimal amount of beats to pass until a remote address is allowed again. localNodes: false, logIdent: 'GUARD', }; /** * Simple throttling system for incoming connections. * @class Heart */ class Guard extends Base { constructor (options) { super({ ...DEFAULT_OPTIONS, ...options }); this.requests = {}; } isAllowed (address, port) { const target = `${this.opts.localNodes ? port : address}`; if (!this.requests[target]) { this.requests[target] = new Date(); return true; } else { const allowed = getSecondsPassed(this.requests[target]) >= this.opts.beatInterval * this.opts.throttleInterval; this.requests[target] = new Date(); return allowed; } } } module.exports = { DEFAULT_OPTIONS, Guard }; ================================================ FILE: src/node/heart.js ================================================ const { Base } = require('./base'); const { getSecondsPassed, getRandomInt, createIdentifier } = require('./tools/utils'); const terminal = require('./tools/terminal'); const DEFAULT_OPTIONS = { cycleInterval: 300, epochInterval: 900, beatInterval: 1, autoStart: false, logIdent: 'HEART', onEpoch: (currentEpoch) => Promise.resolve(false), onCycle: (currentCycle) => Promise.resolve(false), onTick: (currentCycle) => Promise.resolve(0), }; /** * Manages epoch and cycle updates * @class Heart */ class Heart extends Base { constructor (options) { super({ ...DEFAULT_OPTIONS, ...options }); this.id = null; this.ticker = null; this.lastCycle = null; this.lastEpoch = null; this.personality = {}; this.currentCycle = 0; this.currentEpoch = 0; this.startDate = null; this._tick = this._tick.bind(this); this.opts.autoStart && this.start() } start () { this.startDate = new Date(); this.startNewEpoch(); this.lastCycle = new Date(); this.log('Cycle/epoch intervals:', this.opts.cycleInterval, this.opts.epochInterval); terminal.settings({ epochInterval: this.opts.epochInterval, cycleInterval: this.opts.cycleInterval, startDate: this.startDate }); this._tick(); } end () { this.ticker && clearTimeout(this.ticker) } /** * Starts new epoch, resetting node identifiers and memorizing last epoch switch datetime. */ startNewEpoch () { this.setNewPersonality(); this.lastEpoch = new Date(); this.currentEpoch += 1; } /** * Sets this heart's personality: ID, feature, etc. */ setNewPersonality () { const id = createIdentifier(); this.personality = { id, publicId: id.slice(0, 8), feature: id[getRandomInt(0, id.length)] }; this.log('new personality', this.personality.feature, this.personality.id); } /** * Ticker that handles cycle and epoch changes. * @private */ _tick () { this.opts.onTick(this.currentCycle).then(() => { const passedSecondsEpoch = getSecondsPassed(this.lastEpoch); const passedSecondsCycle = getSecondsPassed(this.lastCycle); const pctEpoch = passedSecondsEpoch / this.opts.epochInterval; const pctCycle = passedSecondsCycle / this.opts.cycleInterval; terminal.beat({ epoch: this.currentEpoch, cycle: this.currentCycle, startDate: this.startDate, pctEpoch, pctCycle }); if (passedSecondsCycle > this.opts.cycleInterval) { this.opts.onCycle(this.currentCycle).then((skipABeat) => { if (!skipABeat) { this.lastCycle = new Date(); this.currentCycle += 1; if (passedSecondsEpoch > this.opts.epochInterval) { this.opts.onEpoch(this.currentEpoch).then((skipAge) => { !skipAge && this.startNewEpoch(); this._setTicker(); }); return; } } this._setTicker(); }); return; } this._setTicker(); }); } /** * Sets the ticker for the next beat * @private */ _setTicker () { this.ticker && clearTimeout(this.ticker); this.ticker = setTimeout(this._tick, this.opts.beatInterval * 1000); } } module.exports = { DEFAULT_OPTIONS, Heart }; ================================================ FILE: src/node/index.js ================================================ const base = require('./base'); const heart = require('./heart'); const iri = require('./iri'); const node = require('./node'); const peer = require('./peer'); const peerList = require('./peer-list'); const utils = require('./tools/utils'); module.exports = { base, heart, iri, node, peer, peerList, utils }; ================================================ FILE: src/node/iri.js ================================================ const IOTA = require('iota.lib.js'); const { URL } = require('url'); const tmp = require('tmp'); const { Base } = require('./base'); const { getIP } = require('./tools/utils'); tmp.setGracefulCleanup(); const DEFAULT_OPTIONS = { hostname: 'localhost', port: 14265, TCPPort: 15600, UDPPort: 14600, logIdent: 'IRI', onHealthCheck: (isHealthy, neighbors) => {} }; /** * Class responsible to RUN and communicate with local IRI instance * @class */ class IRI extends Base { constructor (options) { super({ ...DEFAULT_OPTIONS, ...options }); this.api = (new IOTA({ host: `http://${this.opts.hostname}`, port: this.opts.port })).api; this.removeNeighbors = this.removeNeighbors.bind(this); this.addNeighbors = this.addNeighbors.bind(this); this.updateNeighbors = this.updateNeighbors.bind(this); this._tick = this._tick.bind(this); this._getIRIPeerURI = this._getIRIPeerURI.bind(this); this.ticker = null; this.isHealthy = false; this.iriStats = {}; this.staticNeighbors = []; } /** * Starts the IRI process, returning self on success. * @returns {Promise} */ start () { return new Promise((resolve) => { const getNodeInfo = () => this.api.getNeighbors((error, neighbors) => { if (!error) { const addresses = neighbors.map((n) => (new URL(`${n.connectionType}://${n.address}`)).hostname); Promise.all(addresses.map(getIP)).then((ips) => { this._isStarted = true; this.isHealthy = true; this.staticNeighbors = ips.concat(addresses); this.log(`Static neighbors: ${addresses}`); // TODO: make ticker wait for result, like in the heart. this.ticker = setInterval(this._tick, 15000); this.getStats().then(() => resolve(this)); }); } else { this.log(`IRI not ready on ${this.opts.hostname}:${this.opts.port}, retrying...`.yellow); setTimeout(getNodeInfo, 5000); } }); getNodeInfo(); }) } end () { this.isHealthy = false; this._isStarted = false; this.staticNeighbors = []; this.ticker && clearTimeout(this.ticker); this.ticker = null; } /** * Returns whether the process has been started. * @returns {boolean} */ isStarted () { return this._isStarted } /** * Returns whether the IRI process is running and can be communicated with. * @returns {boolean} */ isAvailable () { return this.isStarted() && this.isHealthy } /** * Returns whether a peer's IP or hostname is added as static neighbor in IRI. * @param {Peer} peer * @returns {boolean} */ isStaticNeighbor (peer) { return !!this.staticNeighbors.filter((n) => n === peer.data.ip || n === peer.data.hostname).length; } /** * Removes a list of neighbors from IRI, except static neighbors. Returns list of removed peers. * @param {Peer[]} peers * @returns {Promise} */ removeNeighbors (peers) { if (!this.isAvailable()) { return Promise.reject(); } const myPeers = peers.filter((peer) => { if (this.isStaticNeighbor(peer)) { this.log(`WARNING: trying to remove a static neighbor. Skipping: ${peer.data.hostname}`.yellow); return false; } return true; }); if (!peers.length) { return Promise.resolve([]); } const uris = myPeers.map(this._getIRIPeerURI); return new Promise ((resolve, reject) => { this.api.removeNeighbors(uris, (err) => { if (err) { reject(err); return; } this.log('Neighbors removed (if there were any):'.red, uris.join(', ')); resolve(peers) }); }); } /** * Adds a list of peers to IRI. * @param {Peer[]} peers * @returns {Promise} */ addNeighbors (peers) { if (!this.isAvailable()) { return Promise.reject(); } const uris = peers.map(this._getIRIPeerURI); return new Promise((resolve, reject) => { this.api.addNeighbors(uris, (error) => { if(error) { reject(error); return; } this.log('Neighbors added:'.green, uris.join(', ')); resolve(peers); }); }); } /** * Cleans up any orphans from the IRI * @param {Peer[]} peers * @returns {Promise} */ cleanupNeighbors (peers) { if (!this.isAvailable()) { return Promise.reject(); } return new Promise((resolve) => { this.api.getNeighbors((error, neighbors) => { if(error) { return resolve(); } Promise.all(neighbors.map((n) => { const url = new URL(`${n.connectionType}://${n.address}`); return getIP(url.hostname).then((ip) => { url.ip = ip || 'none'; return url; }) })).then((urls) => { const toRemove = urls.filter((url) => !this.staticNeighbors.includes(url.hostname) && !this.staticNeighbors.includes(url.ip) && peers.filter((p) => ( p.data.hostname === url.hostname || p.data.ip === url.hostname || p.data.hostname === url.ip || p.data.ip === url.ip )).length === 0 ); if (!toRemove.length) { return resolve(toRemove); } this.api.removeNeighbors(toRemove, (err) => { if (err) { reject(err); return; } this.log('Removed orphans:'.red, toRemove.map((url) => url.hostname)); resolve(toRemove) }); }); }); }); } /** * Updates the list of neighbors at the IRI backend. Removes all neighbors, replacing them with * the newly provided neighbors. * @param {Peer[]} peers * @returns {Promise} */ updateNeighbors (peers) { if (!this.isAvailable()) { return Promise.reject(); } if (!peers || !peers.length) { return Promise.resolve([]); } return new Promise((resolve, reject) => { const addNeighbors = () => { this.addNeighbors(peers).then(resolve).catch(reject); }; this.api.getNeighbors((error, neighbors) => { if(error) { reject(error); return; } Array.isArray(neighbors) && neighbors.length ? this.api.removeNeighbors(neighbors.map((n) => `${n.connectionType}://${n.address}`), addNeighbors) : addNeighbors(); }); }); } /** * Removes all IRI neighbors, except static neighbors. * @returns {Promise} */ removeAllNeighbors () { if (!this.isAvailable()) { return Promise.reject(); } return new Promise((resolve) => { this.api.getNeighbors((error, neighbors) => { if(error) { return resolve(); } if (Array.isArray(neighbors) && neighbors.length) { // FIXME: This is broken. staticNeighbors is just a resolved IP. n.address includes port and can be a hostname. // Hence, the filter will always be true. const toRemove = neighbors.filter((n) => !this.staticNeighbors.includes(n.address)); return this.api.removeNeighbors(toRemove.map((n) => `${n.connectionType}://${n.address}`), resolve); } resolve(); }); }); } /** * Returns IRI node info * @returns {Promise} */ getStats () { return new Promise((resolve, reject) => { this.api.getNodeInfo((error, data) => { if(error) { return reject(); } this.iriStats = data; resolve(data); }); }); } /** * Checks if the IRI instance is healthy, and its list of neighbors. Calls back the result to onHealthCheck. * @private */ _tick () { const { onHealthCheck } = this.opts; const onError = () => { this.isHealthy = false; onHealthCheck(false); }; this.getStats().then(() => { this.api.getNeighbors((error, neighbors) => { if(error) { return onError(); } this.isHealthy = true; // TODO: if the address is IPV6, could that pose a problem? onHealthCheck(true, neighbors.map((n) => ({ address: (new URL(`${n.connectionType}://${n.address}`)).hostname, numberOfRandomTransactionRequests: n.numberOfRandomTransactionRequests, numberOfAllTransactions: n.numberOfAllTransactions, numberOfNewTransactions: n.numberOfNewTransactions, numberOfInvalidTransactions: n.numberOfInvalidTransactions }))); }); }).catch(onError); } /** * Returns URI for IRI depending on the protocol. * @param {Peer} peer * @returns {string} * @private */ _getIRIPeerURI (peer) { return peer.data.IRIProtocol === 'tcp' ? peer.getTCPURI() : peer.getUDPURI(); } } module.exports = { DEFAULT_OPTIONS, IRI }; ================================================ FILE: src/node/node.js ================================================ const WebSocket = require("ws"); const ip = require("ip"); const pip = require("external-ip")(); const weighted = require("weighted"); const terminal = require("./tools/terminal"); const { Base } = require("./base"); const { Heart } = require("./heart"); const { Guard } = require("./guard"); const { IRI, DEFAULT_OPTIONS: DEFAULT_IRI_OPTIONS } = require("./iri"); const { PeerList, DEFAULT_OPTIONS: DEFAULT_LIST_OPTIONS } = require("./peer-list"); const { getPeerIdentifier, getRandomInt, getSecondsPassed, getVersion, isSameMajorVersion, getIP, createIdentifier } = require("./tools/utils"); process.on("unhandledRejection", (reason, p) => { console.log("Unhandled Rejection at: Promise", p, "reason:", reason); }); const DEFAULT_OPTIONS = { name: "Deviota Nelson", cycleInterval: 60, epochInterval: 1200, beatInterval: 10, dataPath: DEFAULT_LIST_OPTIONS.dataPath, port: 16600, IRIHostname: DEFAULT_IRI_OPTIONS.hostname, IRIPort: DEFAULT_IRI_OPTIONS.port, IRIProtocol: "any", TCPPort: DEFAULT_IRI_OPTIONS.TCPPort, UDPPort: DEFAULT_IRI_OPTIONS.UDPPort, weightDeflation: 0.95, incomingMax: 6, outgoingMax: 5, maxShareableNodes: 6, localNodes: false, isMaster: false, temporary: false, autoStart: false, logIdent: "NODE", neighbors: [], lazyLimit: 300, // Time, after which a peer is considered lazy, if no new TXs received lazyTimesLimit: 3, // starts to penalize peer's quality if connected so many times without new TXs onReady: node => {}, onPeerConnected: peer => {}, onPeerRemoved: peer => {} }; // TODO: add node tests. Need to mock away IRI for this. class Node extends Base { constructor(options) { super({ ...DEFAULT_OPTIONS, ...options }); this.opts.logIdent = `${this.opts.port}::NODE`; this._onCycle = this._onCycle.bind(this); this._onEpoch = this._onEpoch.bind(this); this._onTick = this._onTick.bind(this); this._onIRIHealth = this._onIRIHealth.bind(this); this._removeNeighbor = this._removeNeighbor.bind(this); this._removeNeighbors = this._removeNeighbors.bind(this); this._addNeighbor = this._addNeighbor.bind(this); this._addNeighbors = this._addNeighbors.bind(this); this.connectPeer = this.connectPeer.bind(this); this.reconnectPeers = this.reconnectPeers.bind(this); this.end = this.end.bind(this); this._ready = false; this.sockets = new Map(); this.opts.autoStart && this.start(); // Tries to fix the issue #45 https://github.com/SemkoDev/nelson.cli/issues/45 // Reasoning: https://github.com/request/request/issues/2161#issuecomment-313375694 // Also, cleans up nelson before crashing from the sky. process.on("uncaughtException", err => { if (err.code !== "ECONNRESET") { this.end().then(() => { throw err; }); } }); } /** * Starts the node server, getting public IP, IRI interface, Peer List and Heart. */ start() { const { cycleInterval, epochInterval, beatInterval, silent, localNodes } = this.opts; this.guard = new Guard({ beatInterval, silent, localNodes }); return this._setPublicIP().then(() => { return this._getIRI() .then(iri => { if (!iri) { throw new Error("IRI could not be started"); } if ( !iri.staticNeighbors.length && this.opts.outgoingMax < DEFAULT_OPTIONS.outgoingMax ) { this.log( `WARNING: you have no static neighbors and outboundMax (${ this.opts.outgoingMax }) is set below the advised limit (${ DEFAULT_OPTIONS.outgoingMax })!` ); } if (this.opts.incomingMax < DEFAULT_OPTIONS.incomingMax) { this.log( `WARNING: incomingMax (${ this.opts.incomingMax }) is set below the advised limit (${ DEFAULT_OPTIONS.incomingMax })!` ); } if (this.opts.incomingMax <= DEFAULT_OPTIONS.outgoingMax) { this.log( `WARNING: incomingMax (${ this.opts.incomingMax }) is set below outgoingMax (${ DEFAULT_OPTIONS.outgoingMax })!` ); } return this._getList() .then(() => { this._createServer(); this.heart = new Heart({ silent, cycleInterval, epochInterval, beatInterval, logIdent: `${this.opts.port}::HEART`, onCycle: this._onCycle, onTick: this._onTick, onEpoch: this._onEpoch }); this._ready = true; this.opts.onReady(this); this.heart.start(); return this; }) .catch(err => { throw err; }); }) .catch(err => { throw err; }); }); } /** * Ends the node, closing HTTP server and IRI backend. * @returns {Promise.} */ end() { this.log("terminating..."); this.heart && this.heart.end(); this._ready = false; const closeServer = () => { return new Promise(resolve => { if (this.server) { this.server.close(); } return this._removeNeighbors( Array.from(this.sockets.keys()) ).then(() => { this.sockets = new Map(); resolve(true); }); }); }; return closeServer().then(() => { return this.iri ? this.iri.end() : true; }); } /** * Sets a new peer list and returns a list of loaded peers. * @returns {Promise.} * @private */ _getList() { const { localNodes, temporary, silent, neighbors, dataPath, isMaster, lazyLimit, lazyTimesLimit } = this.opts; this.list = new PeerList({ multiPort: localNodes, temporary, silent, dataPath, isMaster, lazyLimit, lazyTimesLimit, logIdent: `${this.opts.port}::LIST` }); return this.list.load( neighbors.filter(n => { const tokens = n.split("/"); return !this.isMyself(tokens[0], tokens[1]); }) ); } /** * Sets and returns an IRI instance * @returns {Promise.} * @private */ _getIRI() { const { IRIHostname, IRIPort, silent } = this.opts; return new IRI({ logIdent: `${this.opts.port}::IRI`, hostname: IRIHostname, port: IRIPort, onHealthCheck: this._onIRIHealth, silent }) .start() .then(iri => { this.iri = iri; return iri; }); } /** * Tries to get the public IPs of this node. * @private * @returns {Promise} */ _setPublicIP() { if (this.opts.localNodes) { return Promise.resolve(0); } return new Promise(resolve => { pip((err, ip) => { if (!err) { this.ipv4 = ip; resolve(0); } }); }); } /** * Creates HTTP server for Nelson * @private */ _createServer() { this.server = new WebSocket.Server({ port: this.opts.port, verifyClient: (info, cb) => { const { req } = info; const deny = () => cb(false, 401); const accept = () => cb(true); this._canConnect(req) .then(accept) .catch(deny); } }); this.server.on("connection", (ws, req) => { this.log( "incoming connection established".green, req.connection.remoteAddress ); const { remoteAddress: address } = req.connection; const { port, TCPPort, UDPPort, remoteKey, name, protocol } = this._getHeaderIdentifiers(req.headers); const IRIProtocol = this._negotiateProtocol(protocol); this.list .add({ hostname: address, port, TCPPort, UDPPort, remoteKey, name, IRIProtocol }) .then(peer => { this._bindWebSocket(ws, peer, true); }) .catch(e => { this.log("Error binding/adding".red, address, port, e); this.sockets.delete( Array.from(this.sockets.keys()).find( p => this.sockets.get(p) === ws ) ); ws.close(); ws.terminate(); }); }); this.server.on("headers", headers => { const myHeaders = this._getHeaders(); Object.keys(myHeaders).forEach(key => headers.push(`${key}: ${myHeaders[key]}`) ); }); this.server.on("error", function(err) { // basically, do nothing. Most probably a ECONNRESET error. // The peer will be cleaned up on next tick. }); this.log("server created..."); } /** * Resolves promise if the client is allowed to connect, otherwise rejection. * @param {object} req * @returns {Promise} * @private */ _canConnect(req) { const { remoteAddress: address } = req.connection; const headers = this._getHeaderIdentifiers(req.headers); const { port, nelsonID, version, remoteKey, protocol } = headers || {}; const wrongRequest = !headers; return new Promise((resolve, reject) => { if ( !this._ready || !this.guard || !this.guard.isAllowed(address, port) ) { return reject(); } if (wrongRequest || !isSameMajorVersion(version)) { this.log( "Wrong request or other Nelson version", address, port, version, nelsonID, req.headers ); return reject(); } if (!this.iri || !this.iri.isHealthy) { this.log( "IRI down, denying connections meanwhile", address, port, nelsonID ); return reject(); } if (this.isMyself(address, port, nelsonID)) { return reject(); } this.list .findByRemoteKeyOrAddress(remoteKey, address, port) .then(peers => { if (peers.length && this.sockets.get(peers[0])) { this.log("Peer already connected", address, port); return reject(); } if (peers.length && this.iri.isStaticNeighbor(peers[0])) { this.log( "Peer is already a static neighbor", address, port ); return reject(); } // Deny too frequent connections from the same peer. if ( peers.length && this.isSaturationReached() && peers[0].data.dateLastConnected && getSecondsPassed(peers[0].data.dateLastConnected) < this.opts.epochInterval * 2 ) { return reject(); } // Incompatible protocols if (peers.length[0] && !this._negotiateProtocol(protocol)) { this.log( `Couldn't negotiate protocol with ${ peers[0].data.hostname }: my ${ this.opts.IRIProtocol } vs remote ${protocol}`.yellow ); return reject(); } const topCount = parseInt( Math.sqrt(this.list.all().length) * 2 ); const topPeers = this.list .getWeighted(300) .sort((a, b) => b[1] - a[1]) .map(p => p[0]) .slice(0, topCount); let isTop = false; peers.forEach(p => { if (topPeers.includes(p)) { isTop = true; } }); // The usual way, accept based on personality. const normalPath = () => { if ( this._getIncomingSlotsCount() >= this.opts.incomingMax ) { reject(); } // TODO: additional protection measure: make the client solve a computational riddle! this.isAllowed(remoteKey, address, port).then( allowed => (allowed ? resolve() : reject()) ); }; // Accept old, established nodes. if (isTop) { if ( this._getIncomingSlotsCount() >= this.opts.incomingMax ) { this._dropRandomNeighbors(1, true).then(resolve); } else { resolve(); } } // Accept new nodes more easily. else if ( !peers.length || getSecondsPassed(peers[0].data.dateCreated) <= this.opts.epochInterval * 10 ) { if ( this._getIncomingSlotsCount() >= this.opts.incomingMax ) { const candidates = Array.from( this.sockets.keys() ).filter( p => getSecondsPassed(p.data.dateCreated) <= this.opts.epochInterval * 20 ); if (candidates.length) { this._dropRandomNeighbors( 1, true, candidates ).then(resolve); } else { normalPath(); } } else { resolve(); } } else { normalPath(); } }); }); } /** * Binds the websocket to the peer and adds callbacks. * @param {WebSocket} ws * @param {Peer} peer * @param {boolean} asServer * @private */ _bindWebSocket(ws, peer, asServer = false) { const removeNeighbor = e => { if (!this._ready || !ws || ws.removingNow) { return; } ws.removingNow = true; this._removeNeighbor(peer).then(() => { this.log( "connection closed".red, this.formatNode(peer.data.hostname, peer.data.port), `(${e})` ); }); }; const onConnected = () => { if (!this._ready) { return; } this.log( "connection established".green, this.formatNode(peer.data.hostname, peer.data.port) ); this._sendNeighbors(ws); return peer .markConnected() .then(() => this._ready && this.opts.onPeerConnected(peer)); }; ws.isAlive = true; ws.incoming = asServer; this.sockets.set(peer, ws); ws.on("message", data => this._addNeighbors(data, ws.incoming ? 0 : peer.data.weight) ); ws.on("close", () => removeNeighbor("socket closed")); ws.on("error", () => removeNeighbor("remotely dropped")); ws.on("pong", () => { ws.isAlive = true; }); if (asServer) { onConnected().then( () => this._ready && this.iri.addNeighbors([peer]) ); } else { ws.on("upgrade", res => { // Check for valid headers const head = this._getHeaderIdentifiers(res.headers); if (!head) { this.log("!!", "wrong headers received", head); return removeNeighbor(); } const { port, nelsonID, TCPPort, UDPPort, remoteKey, name, protocol } = head; const IRIProtocol = this._negotiateProtocol(protocol); this.list .update(peer, { port, nelsonID, TCPPort, UDPPort, remoteKey, name, IRIProtocol }) .then(() => { if (IRIProtocol) { this._ready && this.iri.addNeighbors([peer]); } else { this.log( `Couldn't negotiate protocol with ${ peer.data.hostname }: my ${ this.opts.IRIProtocol } vs remote ${IRIProtocol}`.yellow ); removeNeighbor(); } }); }); ws.on("open", onConnected); } } /** * Parses the headers passed between nelson instances * @param {object} headers * @returns {object} * @private */ _getHeaderIdentifiers(headers) { const version = headers["nelson-version"]; const port = headers["nelson-port"]; const nelsonID = headers["nelson-id"]; const TCPPort = headers["nelson-tcp"]; const UDPPort = headers["nelson-udp"]; const remoteKey = headers["nelson-key"]; const name = headers["nelson-name"]; const protocol = headers["nelson-protocol"] || "udp"; if (!version || !port || !nelsonID || !TCPPort || !UDPPort) { return null; } return { version, port, nelsonID, TCPPort, UDPPort, remoteKey, name, protocol }; } /** * Sends list of neighbors through the given socket. * @param {WebSocket} ws * @private */ _sendNeighbors(ws) { ws.send( JSON.stringify( this.getPeers().map(p => p[0].getHostname().replace("/0/", `/${p[1]}/`) ) ) ); } /** * Negotiate protocol to be used between the peers. * If null is returned, the connection cannot be established as there is no consensus. * @param {string} protocol preferred by remote * @param {string} key key for remote * @param {string} remoteKey for this node * @returns {string|null} * @private */ _negotiateProtocol(protocol) { if (protocol === "any") { switch (this.opts.IRIProtocol) { case "tcp": case "prefertcp": return "tcp"; case "udp": case "preferudp": case "any": default: return "udp"; } } else if (protocol === "tcp") { switch (this.opts.IRIProtocol) { case "any": case "tcp": case "prefertcp": case "preferudp": return "tcp"; case "udp": default: return null; } } else if (protocol === "udp") { switch (this.opts.IRIProtocol) { case "any": case "udp": case "prefertcp": case "preferudp": return "udp"; case "tcp": default: return null; } } else if (protocol === "prefertcp") { switch (this.opts.IRIProtocol) { case "any": case "tcp": case "prefertcp": return "tcp"; case "preferudp": case "udp": default: return "udp"; } } else if (protocol === "preferudp") { switch (this.opts.IRIProtocol) { case "any": case "udp": case "preferudp": case "prefertcp": return "udp"; case "tcp": default: return "tcp"; } } } /** * Adds a neighbor to known neighbors list. * @param {string} neighbor * @param {number} weight of the neighbor to assign * @returns {Promise} * @private */ _addNeighbor(neighbor, weight) { // this.log('adding neighbor', neighbor); const tokens = neighbor.split("/"); if ( !isFinite(tokens[1]) || !isFinite(tokens[2]) || !isFinite(tokens[3]) ) { return Promise.resolve(null); } return this.isMyself(tokens[0], tokens[1]) ? Promise.resolve(null) : this.list.add({ hostname: tokens[0], port: tokens[1], TCPPort: tokens[2], UDPPort: tokens[3], peerWeight: weight, weight: weight * parseFloat(tokens[4] || 0) * this.opts.weightDeflation, IRIProtocol: tokens[5] || "udp" }); } /** * Parses raw data from peer's response and adds the provided neighbors. * @param {string} data raw from peer's response * @param {number} weight to assign to the parsed neighbors. * @returns {Promise} * @private */ _addNeighbors(data, weight) { // this.log('add neighbors', data); return new Promise((resolve, reject) => { try { Promise.all( JSON.parse(data) .slice(0, this.opts.maxShareableNodes) .map(neighbor => this._addNeighbor(neighbor, weight)) ).then(resolve); } catch (e) { reject(e); } }); } /** * Returns Nelson headers for request/response purposes * @param {string} key of the peer * @returns {Object} * @private */ _getHeaders(key = "") { return { "Content-Type": "application/json", "Nelson-Version": getVersion(), "Nelson-Port": `${this.opts.port}`, "Nelson-ID": this.heart.personality.publicId, "Nelson-TCP": this.opts.TCPPort, "Nelson-UDP": this.opts.UDPPort, "Nelson-Key": key, "Nelson-Name": this.opts.name, "Nelson-Protocol": this.opts.IRIProtocol }; } /** * Returns amount of incoming connections * @returns {Number} * @private */ _getIncomingSlotsCount() { const arr = Array.from(this.sockets.values()).filter( ws => ws.readyState < 2 ); return arr.filter(ws => ws.incoming).length; } /** * Returns amount of outgoing connections * @returns {Number} * @private */ _getOutgoingSlotsCount() { const arr = Array.from(this.sockets.values()).filter( ws => ws.readyState < 2 ); return arr.filter(ws => !ws.incoming).length; } /** * Disconnects a peer. * @param {Peer} peer * @returns {Promise} * @private */ _removeNeighbor(peer) { if (!this._ready || !this.sockets.get(peer)) { return Promise.resolve([]); } // this.log('removing neighbor', this.formatNode(peer.data.hostname, peer.data.port)); return this._removeNeighbors([peer]); } /** * Disconnects several peers. * @param {Peer[]} peers * @returns {Promise} * @private */ _removeNeighbors(peers) { // this.log('removing neighbors'); const doRemove = () => { return Promise.all( peers.map( peer => new Promise(resolve => { const ws = this.sockets.get(peer); if (ws) { ws.close(); ws.terminate(); } this.sockets.delete(peer); peer.markDisconnected().then(() => { this.opts.onPeerRemoved(peer); resolve(peer); }); }) ) ); }; if (!this.iri || !this.iri.isHealthy) { return Promise.resolve(doRemove()); } return this.iri .removeNeighbors(peers) .then(doRemove) .catch(doRemove); } /** * Randomly removes a given amount of peers from current connections. * Low-quality peers are favored to be removed. * @param {number} amount * @param {boolean} incomingOnly - only drop incoming connections * @param {Peer[]} array - array of connected peers to use for dropping * @returns {Promise.} removed peers * @private */ _dropRandomNeighbors(amount = 1, incomingOnly = false, array = null) { const peers = array ? array : incomingOnly ? Array.from(this.sockets.keys()).filter( p => this.sockets.get(p).incoming ) : array ? array : Array.from(this.sockets.keys()); const selectRandomPeer = () => { const weights = peers.map(p => Math.max(p.getPeerQuality(), 0.0001) ); return weighted(peers, weights); }; const toRemove = []; if (!peers.length) { return Promise.resolve([]); } for (let x = 0; x < amount; x++) { const peer = selectRandomPeer(); peers.splice(peers.indexOf(peer), 1); toRemove.push(peer); } return this._removeNeighbors(toRemove); } /** * Connects to a peer, checking if it's online and trying to get its peers. * @param {Peer} peer * @returns {Peer} */ connectPeer(peer) { this.log( "connecting peer".yellow, this.formatNode(peer.data.hostname, peer.data.port) ); const key = peer.data.key || createIdentifier(); this.list.update(peer, { dateTried: new Date(), tried: (peer.data.tried || 0) + 1, key }); this._bindWebSocket( new WebSocket(peer.getNelsonWebsocketURI(), { headers: this._getHeaders(key), handshakeTimeout: 5000 }), peer ); return peer; } /** * Connects the node to a new set of random addresses that comply with the out/in rules. * Up to a soft maximum. * @returns {Peer[]} List of new connected peers */ reconnectPeers() { // TODO: remove old peers by inverse weight, maybe? Not urgent. Can be added at a later point. // this.log('reconnectPeers'); // If max was reached, do nothing: const toTry = this.opts.outgoingMax - this._getOutgoingSlotsCount(); if ( !this.iri || !this.iri.isHealthy || toTry < 1 || this.isMaster || this._getOutgoingSlotsCount() >= this.opts.outgoingMax ) { return []; } // Get connectable peers: const list = this.list .all() .filter( p => !p.data.dateTried || getSecondsPassed(p.data.dateTried) > this.opts.beatInterval * Math.max(2, 2 * p.data.tried || 0) ) .filter(p => !this.iri.isStaticNeighbor(p)); // Get allowed peers: return this.list .getWeighted(192, list) .filter(p => !this.sockets.get(p[0])) .slice(0, toTry) .map(p => this.connectPeer(p[0])); } /** * Returns a set of peers ready to be shared with their respective weight ratios. * @returns {Array[]} */ getPeers() { // The node tries to recommend best of the best, even better nodes than it tries to connect, usually. // One tries to be helpful to the others, remember? Only suggesting top-notch peers. return this.list.getWeighted(this.opts.maxShareableNodes, null, 2); } /** * Each epoch, disconnect all peers and reconnect new ones. * @private */ _onEpoch() { this.log("new epoch and new id:", this.heart.personality.id); if (!this.isSaturationReached()) { return Promise.resolve(false); } // Master node should recycle all its connections if (this.opts.isMaster) { return this._removeNeighbors(Array.from(this.sockets.keys())).then( () => { this.reconnectPeers(); return false; } ); } return this._dropRandomNeighbors( getRandomInt(0, this._getOutgoingSlotsCount()) ).then(() => { this.reconnectPeers(); return false; }); } /** * Checks whether expired peers are still connectable. * If not, disconnect/remove them. * @private */ _onCycle() { this.log("new cycle"); const promises = []; // Remove closed or dead sockets. Otherwise set as not alive and ping: this.sockets.forEach((ws, peer) => { if (ws.readyState > 1 || !ws.isAlive) { promises.push(this._removeNeighbor(peer)); } else if (peer.isLazy()) { this.log( `Peer ${peer.data.hostname} (${ peer.data.name }) is lazy for more than ${ this.opts.lazyLimit } seconds. Removing...!`.yellow ); promises.push(this._removeNeighbor(peer)); } else if (ws.readyState === 1) { ws.isAlive = false; ws.ping("", false); } }); return Promise.all(promises).then(() => false); } /** * Try connecting to more peers. * @returns {Promise} * @private */ _onTick() { terminal.nodes({ nodes: this.list.all(), connected: Array.from(this.sockets.keys()) .filter(p => this.sockets.get(p).readyState === 1) .map(p => p.data) }); // Try connecting more peers. Master nodes do not actively connect (no outgoing connections). if ( !this.opts.isMaster && this._getOutgoingSlotsCount() < this.opts.outgoingMax ) { return new Promise(resolve => { this.reconnectPeers(); resolve(false); }); } // If for some reason the maximal nodes were overstepped, drop one. else if (this._getIncomingSlotsCount() > this.opts.incomingMax) { return this._dropRandomNeighbors( this._getIncomingSlotsCount() - this.opts.incomingMax, true ).then(() => false); } else { return Promise.resolve(false); } } /** * Callback for IRI to check for health and neighbors. * If unhealthy, disconnect all. Otherwise, disconnect peers that are not in IRI list any more for any reason. * @param {boolean} healthy * @param {object[]} data * @private */ _onIRIHealth(healthy, data) { if (!healthy) { // Do not drop connections, yet. IRI might just be unavailable for a moment. // If it still has "old" neighbors, they will leak, causing more nodes to be added than permitted. this.log("IRI gone...".red); return; // return this._removeNeighbors(Array.from(this.sockets.keys())); } Promise.all(data.map(n => n.address).map(getIP)) .then(neighbors => { const toRemove = []; Array.from(this.sockets.keys()) // It might be that the neighbour was just added and not yet included in IRI... .filter(p => getSecondsPassed(p.data.dateLastConnected) > 5) .forEach(peer => { if ( !neighbors.includes(peer.data.hostname) && (!peer.data.ip || (peer.data.ip && !neighbors.includes(peer.data.ip))) ) { toRemove.push(peer); } else { const index = Math.max( neighbors.indexOf(peer.data.hostname), neighbors.indexOf(peer.data.ip) ); index >= 0 && peer.updateConnection(data[index]); } }); if (toRemove.length) { this.log( "Disconnecting Nelson nodes that are missing in IRI:" .red, toRemove.map(p => p.data.hostname) ); return this._removeNeighbors(toRemove); } }) .then(() => this.iri.cleanupNeighbors(Array.from(this.sockets.keys())) ); } /** * Returns whether the provided address/port/id matches this node * @param {string} address * @param {number|string} port * @param {string|null} nelsonID * @returns {boolean} */ isMyself(address, port, nelsonID = null) { const isPrivate = ip.isPrivate(address) || ["127.0.0.1", "localhost"].includes(address); const sameAddress = isPrivate || address === this.ipv4; const samePort = parseInt(port) === this.opts.port; const sameID = this.heart && this.heart.personality && nelsonID === this.heart.personality.publicId; return sameID || (sameAddress && (!this.opts.localNodes || samePort)); } /** * Returns whether certain address can contact this instance. * @param {string} remoteKey * @param {string} address * @param {number} port * @param {boolean} checkTrust - whether to check for trusted peer * @param {number} easiness - how "easy" it is to get in * @returns {Promise} */ isAllowed(remoteKey, address, port, checkTrust = true, easiness = 24) { const allowed = () => getPeerIdentifier( `${this.heart.personality.id}:${ this.opts.localNodes ? port : address }` ) .slice(0, this._getMinEasiness(easiness)) .indexOf(this.heart.personality.feature) >= 0; return checkTrust ? this.list .findByRemoteKeyOrAddress(remoteKey, address, port) .then(ps => ps.filter(p => p.isTrusted()).length || allowed()) : Promise.resolve(allowed()); } /** * Returns whether the amount of connected nodes has reached a certain threshold. * @returns {boolean} */ isSaturationReached() { const ratioConnected = (this._getOutgoingSlotsCount() + this._getIncomingSlotsCount()) / (this.opts.outgoingMax + this.opts.incomingMax); return ratioConnected >= 0.75; } /** * For new nodes, make it easy to find nodes and contact them * @param {number} easiness - how easy it is to get in/out * @returns {number} updated easiness value * @private */ _getMinEasiness(easiness) { // New nodes are trusting less the incoming connections. // As the node matures in the community, it becomes more welcoming for inbound requests. const l = this.list.all().filter(p => p.data.connected).length; return Math.min(easiness, Math.max(5, parseInt(l / 2))); } } module.exports = { DEFAULT_OPTIONS, Node }; ================================================ FILE: src/node/peer-list.js ================================================ const path = require("path"); const ip = require("ip"); const dns = require("dns"); const tmp = require("tmp"); const { URL } = require("url"); const weighted = require("weighted"); const Datastore = require("nedb"); const { Base } = require("./base"); const { Peer } = require("./peer"); const { DEFAULT_OPTIONS: DEFAULT_IRI_OPTIONS } = require("./iri"); const { getSecondsPassed, createIdentifier } = require("./tools/utils"); const DEFAULT_OPTIONS = { dataPath: path.join(process.cwd(), "data/neighbors.db"), isMaster: false, multiPort: false, temporary: false, logIdent: "LIST", ageNormalizer: 3600, lazyLimit: 300, // Time, after which a peer is considered lazy, if no new TXs received lazyTimesLimit: 3 // starts to penalize peer's quality if connected so many times without new TXs }; /** * A class that manages a list of peers and its persistence in the database * @class PeerList */ class PeerList extends Base { constructor(options) { super({ ...DEFAULT_OPTIONS, ...options }); this.onPeerUpdate = this.onPeerUpdate.bind(this); this.loaded = false; this.peers = []; this.db = new Datastore({ filename: this.opts.temporary ? tmp.tmpNameSync() : this.opts.dataPath, autoload: true }); this.db.persistence.setAutocompactionInterval(30000); } /** * Loads the peer database, preloading defaults, if any. * @param {string[]} defaultPeerURLs * @returns {Promise} */ load(defaultPeerURLs) { return new Promise(resolve => { this.db.find({}, (err, docs) => { this.peers = docs.map( data => new Peer(data, this._getPeerOptions()) ); this.loadDefaults(defaultPeerURLs).then(() => { this.log("DB and default peers loaded"); this.loaded = true; resolve(this.peers); }); }); }); } /** * Adds default peers to the database/list. * @param {string[]} defaultPeerURLs * @returns {Promise} */ loadDefaults(defaultPeerURLs = []) { return Promise.all( defaultPeerURLs.map(uri => { const tokens = uri.split("/"); return this.add({ hostname: tokens[0], port: tokens[1], TCPPort: tokens[2], UDPPort: tokens[3], weight: tokens[4] || 1.0, IRIProtocol: tokens[5] || "udp", isTrusted: true }); }) ); } /** * Update callback when the peer's data has been changed from within the peer. * @param peer * @returns {Promise.} */ onPeerUpdate(peer) { const data = { ...peer.data }; delete data._id; return this.update(peer, data, false); } /** * Partially updates a peer with the provided data. Saves into database. * @param {Peer} peer * @param {Object} data * @param {boolean} refreshPeer - whether to update the peers data. * @returns {Promise} */ update(peer, data, refreshPeer = true) { const newData = { ...peer.data, ...data }; return new Promise(resolve => { this.db.update( { _id: peer.data._id }, newData, { returnUpdatedDocs: true }, () => { // this.log(`updated peer ${peer.data.hostname}:${peer.data.port}`, JSON.stringify(data)); refreshPeer ? peer.update(newData, false).then(() => resolve(peer)) : resolve(peer); } ); }); } /** * Returns currently loaded peers. * @returns {Peer[]} */ all() { return this.peers; } /** * Removes all peers. */ clear() { this.log("Clearing all known peers"); this.peers = []; return new Promise(resolve => this.db.remove({}, { multi: true }, resolve) ); } /** * Gets the average age of all known peers * @returns {number} */ getAverageAge() { return ( this.peers .map(p => getSecondsPassed(p.data.dateCreated)) .reduce((s, x) => s + x, 0) / this.peers.length ); } /** * Returns peers, whose remoteKey, hostname or IP equals the address. * Port is only considered if multiPort option is true. * If the address/port matches, the remoteKey is not considered. * @param {string} remoteKey * @param {string} address * @param {number} port * @returns {Promise} */ findByRemoteKeyOrAddress(remoteKey, address, port) { return new Promise(resolve => { this.findByAddress(address, port).then(peers => { if (peers.length) { return resolve(peers); } resolve( this.peers.filter( p => p.data.remoteKey && p.data.remoteKey === remoteKey ) ); }); }); } /** * Returns peers, whose hostname or IP equals the address. * Port is only considered if mutiPort option is true. * @param {string} address * @param {number} port * @returns {Promise} */ findByAddress(address, port) { const addr = PeerList.cleanAddress(address); return new Promise(resolve => { const findWithIP = ip => { const peers = this.peers.filter( p => p.data.hostname === addr || p.data.hostname === address || (ip && (p.data.hostname === ip || p.data.ip === ip)) ); resolve( this.opts.multiPort ? peers.filter(p => p.data.port == port) : peers ); }; if ( ip.isV6Format(addr) || ip.isV4Format(addr) || this.opts.multiPort ) { findWithIP(addr); } else { dns.resolve(addr, "A", (error, results) => findWithIP(error || !results.length ? null : results[0]) ); } }); } /** * Calculates the trust score of a peer * @param {Peer} peer * @returns {number} */ getPeerTrust(peer) { const age = parseFloat(getSecondsPassed(peer.data.dateCreated)) / this.opts.ageNormalizer; if (this.opts.isMaster) { const weightedAge = (peer.data.connected || peer.isTrusted() ? age : 0) ** 2 * peer.getPeerQuality() ** 2; return Math.max(weightedAge, 0.0001); } const weightedAge = age ** 2 * peer.getPeerQuality() ** 2 * (1.0 + peer.data.weight * 10) ** 2; return Math.max(weightedAge, 0.0001); } /** * Get a certain amount of weighted random peers. Return peers with their respective weight ratios * The weight depends on relationship age (connections) and trust (weight). * @param {number} amount * @param {Peer[]} sourcePeers list of peers to use. Optional for filtering purposes. * @param {number} power by which increase the weights * @returns {Array} */ getWeighted(amount = 0, sourcePeers = null, power = 1.0) { amount = amount || this.peers.length; const peers = sourcePeers || Array.from(this.peers); if (!peers.length) { return []; } const allWeights = peers.map(p => this.getPeerTrust(p) ** power); const weightsMax = Math.max(...allWeights); const choices = []; const getChoice = () => { const peer = weighted(peers, allWeights); const index = peers.indexOf(peer); const weight = allWeights[index]; peers.splice(index, 1); allWeights.splice(index, 1); choices.push([peer, weight / weightsMax]); }; for (let x = 0; x < amount; x++) { if (peers.length < 1) { break; } getChoice(); } return choices .filter(c => c && c[0]) .map(c => [c[0], c[0].isTrusted() ? 1.0 : c[1]]); } /** * Adds a new peer to the list using an URI * @param {object} data * @returns {*} */ add(data) { const { hostname, port: rawPort, TCPPort: rawTCPPort, UDPPort: rawUDPPort, IRIProtocol, isTrusted, peerWeight, weight, remoteKey, name } = Object.assign( { TCPPort: DEFAULT_IRI_OPTIONS.TCPPort, UDPPort: DEFAULT_IRI_OPTIONS.UDPPort, IRIProtocol: "udp", isTrusted: false, peerWeight: 0.5, weight: 0, remoteKey: null }, data ); const port = parseInt(rawPort); const TCPPort = parseInt(rawTCPPort || DEFAULT_IRI_OPTIONS.TCPPort); const UDPPort = parseInt(rawUDPPort || DEFAULT_IRI_OPTIONS.UDPPort); return this.findByRemoteKeyOrAddress(remoteKey, hostname, port).then( peers => { const addr = PeerList.cleanAddress(hostname); const existing = peers.length && peers[0]; if (existing) { return this.update(existing, { weight: weight ? existing.data.weight ? weight * peerWeight + existing.data.weight * (1.0 - peerWeight) : weight : existing.data.weight, key: existing.data.key || createIdentifier(), remoteKey: remoteKey || existing.data.remoteKey, name: name || existing.data.name, hostname: addr, port, TCPPort, UDPPort, IRIProtocol }); } else { this.log( `Adding to the list of known Nelson peers: ${hostname}:${port}` ); const peerIP = ip.isV4Format(addr) || ip.isV6Format(addr) ? addr : null; const peer = new Peer( { port, hostname: addr, ip: peerIP, TCPPort: TCPPort || DEFAULT_IRI_OPTIONS.TCPPort, UDPPort: UDPPort || DEFAULT_IRI_OPTIONS.UDPPort, IRIProtocol: IRIProtocol || "udp", isTrusted, name, weight, remoteKey, key: createIdentifier(), dateCreated: new Date() }, this._getPeerOptions() ); this.peers.push(peer); return new Promise((resolve, reject) => { this.db.insert(peer.data, (err, doc) => { if (err) { reject(err); } peer.update(doc); resolve(peer); }); }); } } ); } _getPeerOptions() { const { lazyLimit, lazyTimesLimit } = this.opts; return { lazyLimit, lazyTimesLimit, onDataUpdate: this.onPeerUpdate }; } /** * Converts an address to a cleaner format. * @param {string} address * @returns {string} */ static cleanAddress(address) { if (!ip.isV4Format(address) && !ip.isV6Format(address)) { return address; } return ip.isPrivate(address) ? "localhost" : address.replace("::ffff:", ""); } } module.exports = { DEFAULT_OPTIONS, PeerList }; ================================================ FILE: src/node/peer.js ================================================ const ip = require('ip'); const dns = require('dns'); const { Base } = require('./base'); const { DEFAULT_OPTIONS: DEFAULT_IRI_OPTIONS } = require('./iri'); const { getSecondsPassed, createIdentifier } = require('./tools/utils'); const PROTOCOLS = ['tcp', 'udp', 'prefertcp', 'preferudp', 'any']; const DEFAULT_OPTIONS = { onDataUpdate: (data) => Promise.resolve(), ipRefreshTimeout: 1200, silent: true, logIdent: 'PEER', lazyLimit: 300, // Time, after which a peer is considered lazy, if no new TXs received lazyTimesLimit: 3 // starts to penalize peer's quality if connected so many times without new TXs }; const DEFAULT_PEER_DATA = { name: null, hostname: null, ip: null, port: 31337, TCPPort: DEFAULT_IRI_OPTIONS.TCPPort, UDPPort: DEFAULT_IRI_OPTIONS.UDPPort, IRIProtocol: 'udp', // Assume all old Nelsons to be running udp. seen: 1, connected: 0, tried: 0, weight: 0, dateTried: null, dateLastConnected: null, dateCreated: null, isTrusted: false, key: null, remoteKey: null, lastConnections: [] }; /** * A utility class for a peer that holds peer's data and provides a few util methods. * * @class Peer */ class Peer extends Base { constructor (data = {}, options) { super({ ...DEFAULT_OPTIONS, ...options, logIdent: `${data.hostname}:${data.port}`}); this.data = null; this.lastConnection = null; // Make sure to migrate database if anything else is added to the defaults... this.update({ ...DEFAULT_PEER_DATA, ...data }); } /** * Partial peer's data update * @param {Object} data * @param {boolean} doCallback - whether to call back on data changes * @returns {Promise} - updates data */ update (data, doCallback=true) { // Reset last updated date if the hostname has changed let shouldUpdate = false; const hostnameChanged = this.data && (this.data.hostname !== (data && data.hostname)); this.iplastUpdated = this.data && hostnameChanged ? null : this.iplastUpdated; this.data = { ...this.data, ...data }; if (hostnameChanged && this.data.ip) { this.data.ip = null; shouldUpdate = true; } if (!this.data.ip) { this.data.ip = this._isHostnameIP() ? this.data.hostname : null; shouldUpdate = true; } return shouldUpdate && doCallback ? this.opts.onDataUpdate(this).then(() => this.data) : Promise.resolve(this.data); } /** * Gets the IP address of the peer. Independently if peer's address is a hostname or IP. * Update's the peer data to save the last known IP. * @returns {Promise} */ getIP () { return new Promise ((resolve) => { if (!this._hasCorrectIP() || (!this._isHostnameIP() && this._isIPOutdated())) { dns.resolve(this.data.hostname, 'A', (error, results) => { // if there was an error, we set the hostname as ip, even if it's not the case. // It will be re-tried next refresh cycle. const ip = error || !results.length ? null : results[0]; this.iplastUpdated = new Date(); if (ip && ip !== this.data.ip) { this.data.ip = ip; this.opts.onDataUpdate(this).then(() => resolve(ip)); } else { resolve(ip) } }) } else { resolve(this.data.ip) } }) } /** * Marks this node as connected. * @returns {Promise.} */ markConnected () { if (this.lastConnection) { return Promise.resolve(this); } this.lastConnection = { start: new Date(), duration: 0, numberOfAllTransactions: 0, numberOfNewTransactions: 0, numberOfInvalidTransactions: 0 }; return this.update({ key: this.data.key || createIdentifier(), tried: 0, connected: this.data.connected + 1, dateLastConnected: new Date() }).then(() => this); } /** * Marks the node as disconnected. Saves connection stats in DB. * @returns {Promise.} */ markDisconnected () { if (!this.lastConnection) { return Promise.resolve(this); } const lastConnections = [ ...this.data.lastConnections, { ...this.lastConnection, end: new Date(), duration: this.getConnectionDuration() }].slice(-10); this.lastConnection = null; return this.update({ lastConnections }).then(() => this); } /** * Returns time in seconds that passed since the node has been connected * @returns {number} */ getConnectionDuration () { if (!this.lastConnection) { return 0; } return getSecondsPassed(this.lastConnection.start) } /** * Updates the stats of the currently connected peer * @param data */ updateConnection (data) { if (!this.lastConnection) { return; } const { numberOfAllTransactions, numberOfRandomTransactionRequests, numberOfNewTransactions, numberOfInvalidTransactions } = data; this.lastConnection = { ...this.lastConnection, numberOfAllTransactions, numberOfRandomTransactionRequests, numberOfNewTransactions, numberOfInvalidTransactions } } /** * Returns peer's quality based on last connection stats. * @returns {number} */ getPeerQuality () { const history = [ ...this.data.lastConnections, this.lastConnection].filter(h => h); const newTrans = history.reduce((s, h) => s + h.numberOfNewTransactions, 0); const badTrans = history.reduce((s, h) => s + h.numberOfInvalidTransactions, 0); const rndTrans = history.reduce((s, h) => s + (h.numberOfRandomTransactionRequests || 0), 0); const badRatio = parseFloat(badTrans * 5 + rndTrans) / (newTrans || 1); const serialPenalization = !this.isTrusted() && !newTrans && history.length >= this.opts.lazyTimesLimit ? 1.0 / history.length : 1.0; const score = Math.max(0.0, 1.0 / (badRatio || 1)) * serialPenalization; return Math.max(0.01, score); } /** * Returns whether a connected peer has not sent any new transactions for a prolonged period of time. * @returns {boolean} */ isLazy () { return this.lastConnection && getSecondsPassed(this.lastConnection.start) > this.opts.lazyLimit && ( this.lastConnection.numberOfNewTransactions === 0 || this.lastConnection.numberOfNewTransactions < this.lastConnection.numberOfRandomTransactionRequests ) } getTCPURI () { return `tcp://${this._getIPString(this.data.hostname)}:${this.data.TCPPort}` } getUDPURI () { return `udp://${this._getIPString(this.data.hostname)}:${this.data.UDPPort}` } getNelsonURI () { return `http://${this._getIPString(this.data.hostname)}:${this.data.port}` } getNelsonWebsocketURI () { return `ws://${this._getIPString(this.data.hostname)}:${this.data.port}` } getHostname () { return `${this.data.hostname}/${this.data.port}/${this.data.TCPPort}/${this.data.UDPPort}/0/${this.data.IRIProtocol}` } isTrusted () { return this.data && this.data.isTrusted } isSameIP (ip) { return this.getIP().then((myIP) => myIP && myIP === ip); } _isHostnameIP () { return ip.isV4Format(this.data.hostname) || ip.isV6Format(this.data.hostname) } _hasCorrectIP () { return this.data.ip && (ip.isV4Format(this.data.ip) || ip.isV6Format(this.data.ip)) } _getIPString (ipOrHostname) { return ipOrHostname.includes(':') ? `[${ipOrHostname}]` : ipOrHostname; } _isIPOutdated () { return !this.iplastUpdated || getSecondsPassed(this.iplastUpdated) > this.opts.ipRefreshTimeout } } module.exports = { DEFAULT_OPTIONS, DEFAULT_PEER_DATA, PROTOCOLS, Peer }; ================================================ FILE: src/node/tools/terminal.js ================================================ const blessed = require('blessed'); const contrib = require('blessed-contrib'); require('colors'); const moment = require('moment'); const momentDurationFormatSetup = require("moment-duration-format"); momentDurationFormatSetup(moment); var screen = null; var mainBox = null; var statusBox = null; var peersBox = null; var progress = null; module.exports = { init, exit: ensureScreen(exit), log: log, beat: ensureScreen(beat), settings: ensureScreen(settings), ports: ensureScreen(ports), nodes: ensureScreen(nodes) }; function init (name, version, onExit) { screen = blessed.screen({ smartCSR: true }); screen.key(['escape', 'q', 'C-c'], function() { exit(); return onExit(); }); mainBox = blessed.box({ top: '51%', left: 'center', width: '100%', height: '49%', content: 'Nelson Console:', scrollable: true, alwaysScroll: true, tags: true, border: { type: 'line' }, style: { fg: 'white', border: { fg: '#f0f0f0' } } }); statusBox = blessed.box({ top: '0%', left: '0%', width: '30%', height: '51%', content: `${name} v.${version} - Status`.green.bold, tags: true, border: { type: 'line' }, style: { fg: 'white', border: { fg: '#f0f0f0' } } }); peersBox = blessed.box({ top: '0%', left: '50%', width: '50%', height: '51%', content: 'Peers'.green.bold, tags: true, border: { type: 'line' }, style: { fg: 'white', border: { fg: '#f0f0f0' } } }); progress = contrib.donut({ top: '0%', left: '30%', width: '20%', height: '51%', radius: 8, arcWidth: 3, remainColor: 'black', yPadding: 2, border: { type: 'line' }, style: { fg: 'white', border: { fg: '#f0f0f0' } } }); screen.append(mainBox); screen.append(statusBox); screen.append(peersBox); screen.append(progress); mainBox.focus(); screen.render(); } function log () { const msg = Array.from(arguments).join(' '); if (!screen) { console.log(msg); return; } mainBox.pushLine(msg); mainBox.setScrollPerc(100); screen.render(); } function beat ({ epoch, cycle, startDate, pctEpoch, pctCycle }) { const duration = moment.duration(moment().diff(startDate)).format('d [days] h [hours] m [minutes]'); statusBox.setLine(3, `Online: ${duration}`.bold.yellow); statusBox.setLine(4, `Epoch: ${epoch}`.bold); statusBox.setLine(5, `Cycle: ${cycle}`.bold); progress.setData([ {percent: pctEpoch, label: 'epoch', color: 'green'}, {percent: pctCycle, label: 'cycle', color: 'green'} ]); screen.render(); } function settings ({ epochInterval, cycleInterval, startDate }) { const startDateString = moment(startDate).format('dddd, MMMM Do YYYY, HH:mm:ss.SSS'); statusBox.setLine(2, `Started on: ${startDateString}`.yellow); statusBox.setLine(6, `Epoch Interval: ${epochInterval}s`); statusBox.setLine(7, `Cycle Interval: ${cycleInterval}s`); screen.render(); } function ports ({ port, apiPort, IRIPort, TCPPort, UDPPort }) { statusBox.setLine(8, `Port: ${port}`.dim.cyan); statusBox.setLine(9, `API Port: ${apiPort}`.dim.cyan); statusBox.setLine(10, `IRI Port: ${IRIPort}`.dim.cyan); statusBox.setLine(11, `TCP Port: ${TCPPort}`.dim.cyan); statusBox.setLine(12, `UDP Port: ${UDPPort}`.dim.cyan); screen.render(); } function nodes ({ nodes, connected }) { peersBox.setLine(2, `Count: ${nodes.length} (Connected: ${connected.length || 0})`.bold); peersBox.setLine(4, `Connections:`.bold); const lines = peersBox.getLines().length; for (let i = lines -1; i >= 5; i--) { peersBox.clearLine(i) } if (!Array.isArray(connected) || connected.length === 0) { peersBox.setLine(5, 'do not worry, this may take a while...'.dim); } else { connected.forEach((connection, i) => { let id = `${connection.hostname||connection.ip}:${connection.port}`.bold.cyan; id = connection.name ? `${id} (${connection.name})`.bold.cyan : id; // const weight = `[trust: ${(connection.trust * 100).toFixed(6)}]`.green; peersBox.setLine(5 + i, `${id} -> ${connection.IRIProtocol || 'udp'}`); }); } screen.render(); } function ensureScreen (f) { return function () { if (!screen) { return; } return f(...arguments); } } function exit () { screen.destroy(); screen = null; } ================================================ FILE: src/node/tools/utils.js ================================================ const ip = require('ip'); const dns = require('dns'); const version = require('../../../package.json').version; const crypto = require('crypto'); const md5 = require('md5'); /** * Resolves IP or hostname to IP. If failed, returns the input. * @param {string} ipOrHostName * @returns {Promise} */ function getIP (ipOrHostName) { return new Promise((resolve) => { if (ip.isV4Format(ipOrHostName) || ip.isV6Format(ipOrHostName)) { return resolve(ipOrHostName); } dns.resolve(ipOrHostName, 'A', (error, results) => { resolve(error ? ipOrHostName : results[0]); }) }); } /** * Returns number of seconds that passed starting from a given time. * @param time * @returns {number} */ function getSecondsPassed (time) { if (!time) { return 0; } return ((new Date()).getTime() - time.getTime()) / 1000 } /** * Creates a random 96-char-long hexadecimal identifier. * @returns {string} */ function createIdentifier () { return crypto.randomBytes(48).toString('hex') } /** * Creates an MD5 hash from the given address * @param {string} address * @returns {string} */ function getPeerIdentifier (address) { return md5(address) } /** * Returns a random number * @param {number} min * @param {number} max * @returns {number} */ function getRandomInt(min, max) { min = Math.ceil(min); max = Math.floor(max); return Math.floor(Math.random() * (max - min)) + min; //The maximum is exclusive and the minimum is inclusive } /** * Shuffles the array * @param {Array} array * @returns {Array} */ function shuffleArray (array) { return array.sort(() => Math.random() - 0.5) } /** * Returns Nelson version number */ function getVersion () { return version; } /** * Returns whether the provided version number is the same major version as the current Nelson. * @param {string} otherVersion */ function isSameMajorVersion (otherVersion) { return version.split('.')[0] === otherVersion.split('.')[0] } /** * Returns whether the provided string is a valid Nelson neighbor representation * @param str * @returns {boolean} */ function validNeighbor(str) { const tokens = str.split('/'); return ( (tokens.length >= 2 && tokens.length <= 5) && (Number.isInteger(parseInt(tokens[1]))) && (!tokens[2] || Number.isInteger(parseInt(tokens[2]))) && (!tokens[3] || Number.isInteger(parseInt(tokens[3]))) && (!tokens[4] || !!parseFloat(tokens[4])) ); } module.exports = { getIP, createIdentifier, getPeerIdentifier, getRandomInt, getSecondsPassed, getVersion, isSameMajorVersion, shuffleArray, validNeighbor }; ================================================ FILE: src/simulation/__tests__/node-network-integration-test.js ================================================ const { spawnMockedNetwork } = require('../network'); const { DEFAULT_OPTIONS } = require('../../node/node'); describe('Node Network', () => { it('should run correctly', (done) => { const onError = () => { throw new Error('A node exited for some reason...'); }; const network = spawnMockedNetwork({ onError, silent: true }); process.on('SIGINT', network.end); process.on('SIGTERM', network.end); setTimeout(() => { network.end().then(() => { const stats = Object.values(network.getStats()); const connections = stats.filter(s => !s.isMaster).map(s => s.connections.connected); const allConnections = stats.map(s => s.connections.connected); const peers = stats.filter(s => !s.isMaster).map(s => s.peers.length); // All normal nodes should be connected to at least one neighbor expect(connections.filter(n => n < 1)).toHaveLength(0); // All nodes should respect the maximal neighbors settings. expect(allConnections.filter(n => n > DEFAULT_OPTIONS.incomingMax + DEFAULT_OPTIONS.outgoingMax + 1)) .toHaveLength(0); // All nodes should have a considerable amount of peers in their DB. expect(peers.filter(n => n < 40)).toHaveLength(0); done(); }); }, 390000) }, 400000); }); ================================================ FILE: src/simulation/bin/nelson.js ================================================ #!/usr/bin/env node const program = require('commander'); const { initMockedNode } = require('../index'); const version = require('../../../package.json').version; const parseNeighbors = (val) => val.split(' '); const parseProtocol = (val) => val.toLowerCase(); const parseNumber = (v) => parseInt(v); process.on('unhandledRejection', (reason, p) => { console.log('Unhandled Rejection at: Promise', p, 'reason:', reason); // application specific logging, throwing an error, or other logic here }); program .version(version) .option('-n, --neighbors [value]', 'Trusted neighbors', parseNeighbors, []) .option('-c, --cycle [value]', 'Cycle interval in seconds', parseNumber, 10) .option('-e, --epoch [value]', 'Epoch interval in seconds', parseNumber, 60) .option('-p, --port [value]', 'Nelson port', parseNumber, 14265) .option('--IRIProtocol [value]', 'IRI protocol to use: udp or tcp, prefertcp, preferudp or any', parseProtocol ,'any') .option('--master [value]', 'Is master node', false) .option('-s, --silent [value]', 'Silent', false) .parse(process.argv); initMockedNode({ port: program.port, dataPort: program.dataPort, silent: program.silent, cycleInterval: program.cycle, epochInterval: program.epoch, neighbors: program.neighbors, isMaster: program.master, IRIProtocol: program.IRIProtocol }); ================================================ FILE: src/simulation/bin/network.js ================================================ #!/usr/bin/env node const program = require('commander'); const { spawnMockedNetwork, DEFAULT_OPTS } = require('../network'); const version = require('../../../package.json').version; const parseNumber = (v) => parseInt(v); process.on('unhandledRejection', (reason, p) => { console.log('Unhandled Rejection at: Promise', p, 'reason:', reason); // application specific logging, throwing an error, or other logic here }); program .version(version) .option('-c, --cycleInterval [value]', 'Cycle interval in seconds', parseNumber, DEFAULT_OPTS.cycleInterval) .option('-e, --epochInterval [value]', 'Epoch interval in seconds', parseNumber, DEFAULT_OPTS.epochInterval) .option('-p, --startingPort [value]', 'Starting port', parseNumber, DEFAULT_OPTS.startingPort) .option('-n, --nodesCount [value]', 'Normal nodes amount', parseNumber, DEFAULT_OPTS.nodesCount) .option('-m, --masterNodesCount [value]', 'Master nodes amount', parseNumber, DEFAULT_OPTS.masterNodesCount) .option('-s, --silent', 'Silent', DEFAULT_OPTS.silent) .parse(process.argv); const proc = spawnMockedNetwork({ ...program, callback: stats.onCallback }); process.on('SIGINT', proc.end); process.on('SIGTERM', proc.end); ================================================ FILE: src/simulation/index.js ================================================ const { initMockedNode } = require('./node'); const { spawnMockedNetwork } = require('./network'); module.exports = { initMockedNode, spawnMockedNetwork }; ================================================ FILE: src/simulation/network.js ================================================ const { utils, peer } = require('../node'); const { spawnNode } = require('./node'); const DEFAULT_OPTS = { silent: false, cycleInterval: 12, epochInterval: 36, nodesCount: 47, masterNodesCount: 3, startingPort: 14265, nodeStartDelayRange: [0, 6000], callbackInterval: 5000, onStats: (nodeStats) => {}, onError: (nodeStats) => {} }; // TODO: update jsdoc /** * Initializes and starts a simulation with a set of mocked nodes. * First, the master nodes are started all at once. Then the normal nodes are added sequentially * in a random interval (using options.nodeStartDelayRange). * * @param {object} options * @param {number} options.nodesCount - the amount of "normal" nodes to start * @param {number} options.masterNodesCount - the amount of "master" nodes to start * @param {number} options.startingPort - Port number to start the nodes from incrementally * @param {function} options.onStats - periodic callback with connection summary * @param {function} options.onError - on child process exit or error * @param {boolean} options.callbackInterval - in seconds * @param {number[]} options.nodeStartDelayRange - how many ms to wait between normal nodes starting * @returns {{stop: (function()), onPeersAdded: (function())}} */ function spawnMockedNetwork (options) { const { nodesCount, masterNodesCount, startingPort, silent, cycleInterval, epochInterval, onStats, onError, callbackInterval, nodeStartDelayRange } = { ...DEFAULT_OPTS, ...options}; const baseNodeOptions = { silent, cycleInterval, epochInterval }; const allNodes = []; const masterNodeURIs = []; const stats = {}; let ended = false; let cbInterval = null; const hasEnded = () => ended; const prc = (p, port) => { p.on('message', (s) => stats[port] = s); p.on('error', onError); p.on('exit', () => !hasEnded() && onError()); }; // Start the master nodes for(let x = 0; x < masterNodesCount; x++) { const port = startingPort + x; const TCPPort = port + 10000; const UDPPort = port + 20000; if (ended) { break } const node = spawnNode({ ...baseNodeOptions, port, isMaster: true, neighbors: masterNodeURIs }); prc(node, port); allNodes.push(node); masterNodeURIs.push(`localhost/${port}/${TCPPort}/${UDPPort}`); } // Sequentially start the normal nodes const promise = ".".repeat(nodesCount).split('').reduce((promise, value, y) => { return hasEnded() ? promise : promise.then((nodes) => { return new Promise((resolve) => { if (hasEnded()) { return resolve(nodes) } setTimeout(() => { if (hasEnded()) { return resolve(nodes) } const port = startingPort + masterNodesCount + y; const TCPPort = port + 10000; const UDPPort = port + 20000; const node = spawnNode({ ...baseNodeOptions, port, TCPPort, UDPPort, neighbors: masterNodeURIs, IRIProtocol: peer.PROTOCOLS[utils.getRandomInt(0, peer.PROTOCOLS.length)] }); prc(node, port); resolve([ ...nodes, node ]) }, utils.getRandomInt(nodeStartDelayRange[0], nodeStartDelayRange[1])); }) }); }, Promise.resolve(allNodes)); const end = () => { ended = true; cbInterval && clearInterval(cbInterval); return promise.then((nodes) => { !silent && console.log('STOPPING NETWORK'); nodes.forEach(n => n.kill()); }) }; if (callbackInterval) { cbInterval = setInterval(() => onStats(stats), callbackInterval); } return { end, onPeersAdded: () => { return promise }, getStats: () => stats, getNodeProcesses: () => allNodes } } module.exports = { DEFAULT_OPTS, spawnMockedNetwork }; ================================================ FILE: src/simulation/node.js ================================================ const cp = require('child_process'); const { Node } = require('../node/__mocks__/node'); /** * Initializes a mocked node with given options * @param {object} options - see Node options for details. * @returns {Promise} */ function initMockedNode (options) { const node = new Node(options); return node.start().then((n) => { n.log('initialized!'); return node; }); } /** * Spawns a node process * @param {object} options */ function spawnNode(options={}, silent) { let opts = []; options.port && opts.push('-p') && opts.push(`${options.port}`); options.isMaster && opts.push(`--master`); options.IRIProtocol && opts.push(`--IRIProtocol`) && opts.push(options.IRIProtocol); options.neighbors && options.neighbors.length && opts.push('-n') && opts.push(`${options.neighbors.join(' ')}`); options.silent && opts.push('-s'); options.cycleInterval && opts.push('-c') && opts.push(`${options.cycleInterval}`); options.epochInterval && opts.push('-e') && opts.push(`${options.epochInterval}`); return cp.fork(`${__dirname}/bin/nelson.js`, opts, { silent }); } module.exports = { spawnNode, initMockedNode };