Showing preview only (2,236K chars total). Download the full file or copy to clipboard to get everything.
Repository: npms-io/npms-analyzer
Branch: master
Commit: bfee077ab3dc
Files: 314
Total size: 2.1 MB
Directory structure:
gitextract_mgcbttt5/
├── .editorconfig
├── .eslintignore
├── .eslintrc.json
├── .gitignore
├── .travis.yml
├── LICENSE
├── README.md
├── cli.js
├── cmd/
│ ├── consume.js
│ ├── observe.js
│ ├── scoring.js
│ ├── tasks/
│ │ ├── check-gh-tokens.js
│ │ ├── clean-extraneous.js
│ │ ├── enqueue-missing.js
│ │ ├── enqueue-outdated.js
│ │ ├── enqueue-view.js
│ │ ├── migrate.js
│ │ ├── process-package.js
│ │ ├── re-evaluate.js
│ │ └── re-metadata.js
│ ├── tasks.js
│ └── util/
│ ├── bootstrap.js
│ └── stats/
│ ├── index.js
│ ├── process.js
│ ├── progress.js
│ ├── queue.js
│ └── tokens.js
├── config/
│ ├── couchdb/
│ │ ├── npms-analyzer-npm.json
│ │ └── npms-analyzer-npms.json
│ ├── default.json5
│ └── elasticsearch/
│ └── npms.json5
├── docs/
│ ├── architecture.md
│ ├── deploys.md
│ ├── diagrams/
│ │ ├── analysis.xml
│ │ └── continuous-scoring.xml
│ └── setup.md
├── ecosystem.json5
├── lib/
│ ├── analyze/
│ │ ├── collect/
│ │ │ ├── bin/
│ │ │ │ └── david-json
│ │ │ ├── github.js
│ │ │ ├── index.js
│ │ │ ├── metadata.js
│ │ │ ├── npm.js
│ │ │ ├── source.js
│ │ │ └── util/
│ │ │ ├── fileContents.js
│ │ │ ├── fileSize.js
│ │ │ ├── pointsToRanges.js
│ │ │ └── promisePropsSettled.js
│ │ ├── download/
│ │ │ ├── git.js
│ │ │ ├── github.js
│ │ │ ├── index.js
│ │ │ ├── npm.js
│ │ │ └── util/
│ │ │ ├── assertFilesCount.js
│ │ │ ├── findPackageDir.js
│ │ │ ├── mergePackageJson.js
│ │ │ └── untar.js
│ │ ├── evaluate/
│ │ │ ├── index.js
│ │ │ ├── maintenance.js
│ │ │ ├── popularity.js
│ │ │ └── quality.js
│ │ ├── index.js
│ │ └── util/
│ │ ├── exec.js
│ │ ├── gotRetry.js
│ │ ├── hostedGitInfo.js
│ │ ├── normalizePackageJson.js
│ │ └── packageJsonFromData.js
│ ├── configure.js
│ ├── observers/
│ │ ├── realtime.js
│ │ └── stale.js
│ ├── queue.js
│ └── scoring/
│ ├── aggregate.js
│ ├── finalize.js
│ ├── prepare.js
│ ├── score.js
│ └── util/
│ └── paperNumerical.js
├── package.json
└── test/
├── .eslintrc.json
├── bin/
│ └── download-fixtures
├── fixtures/
│ └── analyze/
│ ├── collect/
│ │ ├── modules/
│ │ │ ├── 0/
│ │ │ │ ├── data.json
│ │ │ │ └── expected-source.json
│ │ │ ├── @bcoe%2fexpress-oauth-server/
│ │ │ │ ├── data.json
│ │ │ │ └── expected-metadata.json
│ │ │ ├── babel-jest/
│ │ │ │ ├── data.json
│ │ │ │ └── expected-source.json
│ │ │ ├── backoff/
│ │ │ │ ├── data.json
│ │ │ │ └── expected-source.json
│ │ │ ├── cross-spawn/
│ │ │ │ ├── data.json
│ │ │ │ ├── expected-github.json
│ │ │ │ ├── expected-metadata.json
│ │ │ │ ├── expected-npm.json
│ │ │ │ └── expected-source.json
│ │ │ ├── hapi/
│ │ │ │ ├── data.json
│ │ │ │ └── expected-source.json
│ │ │ ├── planify/
│ │ │ │ ├── data.json
│ │ │ │ └── expected-source.json
│ │ │ ├── react/
│ │ │ │ ├── data.json
│ │ │ │ └── expected-source.json
│ │ │ └── react-router/
│ │ │ ├── data.json
│ │ │ └── expected-source.json
│ │ └── recorded/
│ │ ├── github/
│ │ │ ├── 0bfbe2f1c03ff5ed9c3baa91d588e218
│ │ │ ├── 0bfbe2f1c03ff5ed9c3baa91d588e218.headers
│ │ │ ├── 2236c266c85b15946d7ca69cc2e1e091
│ │ │ ├── 2236c266c85b15946d7ca69cc2e1e091.headers
│ │ │ ├── 24d4b4797edc40614848f01802bbe2b3
│ │ │ ├── 24d4b4797edc40614848f01802bbe2b3.headers
│ │ │ ├── 2f87db1cf50593ec3f80835f624ec88a
│ │ │ ├── 2f87db1cf50593ec3f80835f624ec88a.headers
│ │ │ ├── 3a1a79735cab3e2c46da0f739eccd595
│ │ │ ├── 3a1a79735cab3e2c46da0f739eccd595.headers
│ │ │ ├── 3b5e34c45a594730608f6170cacf31fe
│ │ │ ├── 3b5e34c45a594730608f6170cacf31fe.headers
│ │ │ ├── 3d15804db16c597c23a14946a78b8e1b
│ │ │ ├── 3d15804db16c597c23a14946a78b8e1b.headers
│ │ │ ├── 4109ed740591855f9e48eb868f40db86
│ │ │ ├── 4109ed740591855f9e48eb868f40db86.headers
│ │ │ ├── 5bdd0f1e3c86f0114eb714a2ce79e905
│ │ │ ├── 5bdd0f1e3c86f0114eb714a2ce79e905.headers
│ │ │ ├── 6913bcb008bea8f6a0384da9bbae2293
│ │ │ ├── 6913bcb008bea8f6a0384da9bbae2293.headers
│ │ │ ├── 85c5e5ee4a806e7d405984c325f29007
│ │ │ ├── 85c5e5ee4a806e7d405984c325f29007.headers
│ │ │ ├── 92a4188d8af9e8c1ff665859b3cd86b8
│ │ │ ├── 92a4188d8af9e8c1ff665859b3cd86b8.headers
│ │ │ ├── 9ccf24e28c94543e4ae601d5aa9c8cba
│ │ │ ├── 9ccf24e28c94543e4ae601d5aa9c8cba.headers
│ │ │ ├── a3360f53aa71342ad67cded65f6ea1da
│ │ │ ├── a3360f53aa71342ad67cded65f6ea1da.headers
│ │ │ ├── a7d2cca72c7267fd27fe769cb4d2f611
│ │ │ ├── a7d2cca72c7267fd27fe769cb4d2f611.headers
│ │ │ ├── afcd15ede3deaa855315f5a1fbc3e61d
│ │ │ ├── afcd15ede3deaa855315f5a1fbc3e61d.headers
│ │ │ ├── b32671b71119c7fc156f3aa050d1cb12
│ │ │ ├── b32671b71119c7fc156f3aa050d1cb12.headers
│ │ │ ├── c23e4492fdaefde5fe60524fab308532
│ │ │ ├── c23e4492fdaefde5fe60524fab308532.headers
│ │ │ ├── c389930a56f1bad9257ed1490fc32c9b
│ │ │ ├── c389930a56f1bad9257ed1490fc32c9b.headers
│ │ │ ├── cc88ef857a3a1492913c066047c5c033
│ │ │ ├── cc88ef857a3a1492913c066047c5c033.headers
│ │ │ ├── d3b7e3ec7ad3c841c45ff000fd77b711
│ │ │ ├── d3b7e3ec7ad3c841c45ff000fd77b711.headers
│ │ │ ├── d495e09987382290004f52a8fa39243b
│ │ │ ├── d495e09987382290004f52a8fa39243b.headers
│ │ │ ├── dadb0a8973a79e019a2a0affeb248deb
│ │ │ ├── dadb0a8973a79e019a2a0affeb248deb.headers
│ │ │ ├── ebd826ea6dcd2abd0dcc961dd0fe4176
│ │ │ ├── ebd826ea6dcd2abd0dcc961dd0fe4176.headers
│ │ │ ├── ede6348d94fd4d537bcc1e42b050d8ed
│ │ │ ├── ede6348d94fd4d537bcc1e42b050d8ed.headers
│ │ │ ├── ff4470aede5dba39b4736419dda38443
│ │ │ └── ff4470aede5dba39b4736419dda38443.headers
│ │ ├── index/
│ │ │ ├── 474829c21c2e69d2c0d889af2a714584
│ │ │ ├── 474829c21c2e69d2c0d889af2a714584.headers
│ │ │ ├── ae18615611dfb3f32feaf1c607df7bac
│ │ │ ├── ae18615611dfb3f32feaf1c607df7bac.headers
│ │ │ ├── d40cbb49129f01a9d5130a95f54d4f79
│ │ │ └── d40cbb49129f01a9d5130a95f54d4f79.headers
│ │ ├── npm/
│ │ │ ├── 278e742ec691d9647761d9e06a93c852
│ │ │ ├── 278e742ec691d9647761d9e06a93c852.headers
│ │ │ ├── 6da574a19e30e15a2628bc2a7ae7d5a4
│ │ │ └── 6da574a19e30e15a2628bc2a7ae7d5a4.headers
│ │ └── source/
│ │ ├── 0046732f19ee23072e08f55d2a400eca
│ │ ├── 0046732f19ee23072e08f55d2a400eca.headers
│ │ ├── 02777f766910df6791475f44c0e2b57b
│ │ ├── 02777f766910df6791475f44c0e2b57b.headers
│ │ ├── 0429ac4bdf217161d9a2772fdb7861e2
│ │ ├── 0429ac4bdf217161d9a2772fdb7861e2.headers
│ │ ├── 0c774ab0c1fa72fb9dbba94b96693973
│ │ ├── 0c774ab0c1fa72fb9dbba94b96693973.headers
│ │ ├── 1152654e8bfc034a9d043925c55fbe48
│ │ ├── 1152654e8bfc034a9d043925c55fbe48.headers
│ │ ├── 180995f905c69e6355ccbeb197109fb9
│ │ ├── 180995f905c69e6355ccbeb197109fb9.headers
│ │ ├── 1b4935ddf796087a37e45c313edddd4f
│ │ ├── 1b4935ddf796087a37e45c313edddd4f.headers
│ │ ├── 310c26a3622e22be3798b810bb056cd2
│ │ ├── 310c26a3622e22be3798b810bb056cd2.headers
│ │ ├── 32532aa076ea4d37a94def3f370e23fc
│ │ ├── 32532aa076ea4d37a94def3f370e23fc.headers
│ │ ├── 36d2b9e3113bb8656477a0866759fca3
│ │ ├── 36d2b9e3113bb8656477a0866759fca3.headers
│ │ ├── 3f5a8e5d9434bbd4ec976c36835cdc49
│ │ ├── 3f5a8e5d9434bbd4ec976c36835cdc49.headers
│ │ ├── 40971c651c26e9ab81128eb0c10f37af
│ │ ├── 40971c651c26e9ab81128eb0c10f37af.headers
│ │ ├── 44c08d748b72275a181e7820e9c258e8
│ │ ├── 44c08d748b72275a181e7820e9c258e8.headers
│ │ ├── 4d7966a6e7249722abdff0a7555a2527
│ │ ├── 4d7966a6e7249722abdff0a7555a2527.headers
│ │ ├── 4ece6fa3645c526294eaf1b270113e6d
│ │ ├── 4ece6fa3645c526294eaf1b270113e6d.headers
│ │ ├── 50ec845be323513dc05d4ce6eeb56639
│ │ ├── 50ec845be323513dc05d4ce6eeb56639.headers
│ │ ├── 586e879d6364ca5313dd5f956d47dbd4
│ │ ├── 586e879d6364ca5313dd5f956d47dbd4.headers
│ │ ├── 5fcfa736ff5a936752226c33baab7ce5
│ │ ├── 5fcfa736ff5a936752226c33baab7ce5.headers
│ │ ├── 6566d9e3adefb1ed8fe60a17bbf13133
│ │ ├── 6566d9e3adefb1ed8fe60a17bbf13133.headers
│ │ ├── 6632454a445cfcd8152c30b4b8c64783
│ │ ├── 6632454a445cfcd8152c30b4b8c64783.headers
│ │ ├── 672a67ca0e6ca2125e9601f4e532dc2e
│ │ ├── 672a67ca0e6ca2125e9601f4e532dc2e.headers
│ │ ├── 6ef5fd0be1ac70c0cc78c63dc72f97da
│ │ ├── 6ef5fd0be1ac70c0cc78c63dc72f97da.headers
│ │ ├── 6f6f60463501ffe7964700fdb5262ea7
│ │ ├── 6f6f60463501ffe7964700fdb5262ea7.headers
│ │ ├── 77c0ab2484ae068cadf90515cd2bb6d4
│ │ ├── 77c0ab2484ae068cadf90515cd2bb6d4.headers
│ │ ├── 7fb20e04d9d456482d62b369a1c268c0
│ │ ├── 7fb20e04d9d456482d62b369a1c268c0.headers
│ │ ├── 833f1d757cdb7cfa152b54680d0d2d73
│ │ ├── 833f1d757cdb7cfa152b54680d0d2d73.headers
│ │ ├── 8881281d102f7688a2b0e5d7ffb48299
│ │ ├── 8881281d102f7688a2b0e5d7ffb48299.headers
│ │ ├── 8acf97225f349b2c99978025ba5b8e92
│ │ ├── 8acf97225f349b2c99978025ba5b8e92.headers
│ │ ├── 8dcbc0b25ce4f37fd5c0bc06d633eb52
│ │ ├── 8dcbc0b25ce4f37fd5c0bc06d633eb52.headers
│ │ ├── 92d4837e4094c3e096f398bc89aabb0f
│ │ ├── 92d4837e4094c3e096f398bc89aabb0f.headers
│ │ ├── 981d42650123f10d1600074d65aa4f43
│ │ ├── 981d42650123f10d1600074d65aa4f43.headers
│ │ ├── 9a7bfe12b0910e8bd69ed06184030276
│ │ ├── 9a7bfe12b0910e8bd69ed06184030276.headers
│ │ ├── a27623f8433973cd8a3c9ac32782dd9b
│ │ ├── a27623f8433973cd8a3c9ac32782dd9b.headers
│ │ ├── a9be16a47e9b4e3762bfbcfbec14effd
│ │ ├── a9be16a47e9b4e3762bfbcfbec14effd.headers
│ │ ├── acd2d3271b8cd130cd75e572ada409c1
│ │ ├── acd2d3271b8cd130cd75e572ada409c1.headers
│ │ ├── afd6397af26789ff7e024c758e094e02
│ │ ├── afd6397af26789ff7e024c758e094e02.headers
│ │ ├── b00ce31e18a32896ac83d6819e9816fe
│ │ ├── b00ce31e18a32896ac83d6819e9816fe.headers
│ │ ├── b6d2435e45a7f8f3b88152e577c55b84
│ │ ├── b6d2435e45a7f8f3b88152e577c55b84.headers
│ │ ├── bb391b38e8f2529e20e3e313a7875566
│ │ ├── bb391b38e8f2529e20e3e313a7875566.headers
│ │ ├── bbd9372c326ea4fb4acc82bd30e9491c
│ │ ├── bbd9372c326ea4fb4acc82bd30e9491c.headers
│ │ ├── c2a263604b39c741dcba960ab7bdf64e
│ │ ├── c2a263604b39c741dcba960ab7bdf64e.headers
│ │ ├── ca1b5b9f76e662b613be43d64e92c4b4
│ │ ├── ca1b5b9f76e662b613be43d64e92c4b4.headers
│ │ ├── d4bf4a43cf4e7b6c27e40c732c9d8bfa
│ │ ├── d4bf4a43cf4e7b6c27e40c732c9d8bfa.headers
│ │ ├── daaa6494600d82aa3e21e56452a8702a
│ │ ├── daaa6494600d82aa3e21e56452a8702a.headers
│ │ ├── e2d10b245b6bce60976eb41c755c5333
│ │ ├── e2d10b245b6bce60976eb41c755c5333.headers
│ │ ├── e66bf57e7754e3a75c0b3da3c7d3b894
│ │ ├── e66bf57e7754e3a75c0b3da3c7d3b894.headers
│ │ ├── f13ef1d336f7343f21a8f1e755ee4a1d
│ │ ├── f13ef1d336f7343f21a8f1e755ee4a1d.headers
│ │ ├── f4fbc0e6ea0806cdebfe05e95480858f
│ │ ├── f4fbc0e6ea0806cdebfe05e95480858f.headers
│ │ ├── fba7a93bdb3f483048d32ccc0a105e2b
│ │ ├── fba7a93bdb3f483048d32ccc0a105e2b.headers
│ │ ├── febf07de22a7d2d7ffe02742c6b81857
│ │ └── febf07de22a7d2d7ffe02742c6b81857.headers
│ └── download/
│ ├── mocked/
│ │ ├── broken-archive.tgz
│ │ └── non-gzip-archive.tgz
│ └── recorded/
│ ├── github/
│ │ ├── 0ca08d9404d3be6b0f4b710e7dce325c
│ │ ├── 0ca08d9404d3be6b0f4b710e7dce325c.headers
│ │ ├── 0d6bf2e4d590ee9d6ece01c851500563
│ │ ├── 0d6bf2e4d590ee9d6ece01c851500563.headers
│ │ ├── 2197e3675f7b1189860675d45228c712
│ │ ├── 2197e3675f7b1189860675d45228c712.headers
│ │ ├── 324da49a49b1bf9799ad0af735d42175
│ │ ├── 324da49a49b1bf9799ad0af735d42175.headers
│ │ ├── 347ad9c22b702976e2e6304bee584cd2
│ │ ├── 347ad9c22b702976e2e6304bee584cd2.headers
│ │ ├── 39c4db447b629049bd6c82625ecbb182
│ │ ├── 39c4db447b629049bd6c82625ecbb182.headers
│ │ ├── 598868e39b0c5f898b243dc6a7799590
│ │ ├── 598868e39b0c5f898b243dc6a7799590.headers
│ │ ├── 6b4c11cf7f1c30a3c0f45d6dee2c98c8
│ │ ├── 6b4c11cf7f1c30a3c0f45d6dee2c98c8.headers
│ │ ├── ce4fa3241f0364d1ea4e654f3cf13cb9
│ │ ├── ce4fa3241f0364d1ea4e654f3cf13cb9.headers
│ │ ├── d7c449b25de454b1b362bc5af32cc777
│ │ └── d7c449b25de454b1b362bc5af32cc777.headers
│ └── npm/
│ ├── 57f54040bdda5ac6ffd196cac24be2d8
│ ├── 57f54040bdda5ac6ffd196cac24be2d8.headers
│ ├── afed24ca0f8e9f12344ae6a851b46159
│ └── afed24ca0f8e9f12344ae6a851b46159.headers
├── mocha.opts
├── spec/
│ └── analyze/
│ ├── collect/
│ │ ├── github.js
│ │ ├── index.js
│ │ ├── metadata.js
│ │ ├── npm.js
│ │ ├── source.js
│ │ └── util/
│ │ ├── fileContents.js
│ │ ├── fileSize.js
│ │ ├── pointsToRanges.js
│ │ └── promisePropsSettled.js
│ ├── download/
│ │ ├── git.js
│ │ ├── github.js
│ │ ├── index.js
│ │ ├── npm.js
│ │ └── util/
│ │ ├── findPackageDir.js
│ │ ├── mergePackageJson.js
│ │ └── untar.js
│ ├── evaluate/
│ │ ├── index.js
│ │ ├── maintenance.js
│ │ ├── popularity.js
│ │ └── quality.js
│ └── util/
│ ├── exec.js
│ ├── gotRetry.js
│ ├── hostedGitInfo.js
│ ├── normalizePackageJson.js
│ └── packageJsonFromData.js
├── test.js
└── util/
└── sepia.js
================================================
FILE CONTENTS
================================================
================================================
FILE: .editorconfig
================================================
root = true
[*]
indent_style = space
indent_size = 4
end_of_line = lf
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
[*.md]
trim_trailing_whitespace = false
[package.json]
indent_size = 2
================================================
FILE: .eslintignore
================================================
/test/coverage
/dev
/cli.js
================================================
FILE: .eslintrc.json
================================================
{
"root": true,
"extends": [
"eslint-config-moxy/es8",
"eslint-config-moxy/addons/node"
],
"globals": {
"logger": false
}
}
================================================
FILE: .gitignore
================================================
node_modules
npm-debug.*
/config/local.*
/test/coverage
/test/tmp
/test/fixtures/**/downloaded
/dev
================================================
FILE: .travis.yml
================================================
language: node_js
node_js:
- node
- lts/*
script: "npm run test-travis"
before_install:
- sudo apt-get install -y bsdtar
after_success:
- "npm i codecov"
- "node_modules/.bin/codecov"
================================================
FILE: LICENSE
================================================
Copyright (c) 2016 npms
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is furnished
to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
================================================
FILE: README.md
================================================
# npms-analyzer
[![Build status][travis-image]][travis-url] [![Coverage status][codecov-image]][codecov-url] [![Dependency status][david-dm-image]][david-dm-url] [![Dev dependency status][david-dm-dev-image]][david-dm-dev-url]
The npms-analyzer analyzes the npm ecosystem, collecting info, evaluating and scoring each package.
## Usage
This project offers all its functionality through a CLI.
*(output might be outdated)*
Note that you must [setup](./docs/setup.md) the project before using the CLI. The most important commands will be described below. To discover the other ones run `$ npms-analyzer -h`.
### npms-analyzer observe
The `observe` command starts observing changes that occur in the `npm` registry as well as packages that were not analyzed for a while. Each reported package will be pushed into a queue to be processed by the queue consumers.
```bash
$ npms-analyzer observe --log-level debug | pino
```
For more information about the command, run `$ npms-analyzer observe -h`
### npms-analyzer consume
The `consume` command starts consuming the queue, running the analysis process for each queued package.
```bash
$ npms-analyzer consume --log-level debug --concurrency 5 | pino
```
For more information about the command, run `$ npms-analyzer consume -h`
### npms-analyzer scoring
The `scoring` command, continuously iterates over the analysis results and calculates a score for all the `npm` packages, storing its result in `elasticsearch`.
```bash
$ npms-analyzer scoring
```
For more information about the command, run `$ npms-analyzer scoring -h`
## Architecture
There's a separate document that explains the architecture, you may read it [here](./docs/architecture.md).
## Setup
There's a separate document that explains the setup procedure, you may read it [here](./docs/setup.md).
## Deploys
There's a separate document that explains the deployment procedure, you may read it [here](./docs/deploys.md).
## Tests
Before running the tests, you must have read through the setup guide.
```bash
$ npm test
$ npm test-cov # to get coverage report
```
[codecov-url]:https://codecov.io/gh/npms-io/npms-analyzer
[codecov-image]:https://img.shields.io/codecov/c/github/npms-io/npms-analyzer/master.svg
[david-dm-dev-image]: https://img.shields.io/david/dev/npms-io/npms-analyzer.svg
[david-dm-dev-url]: https://david-dm.org/npms-io/npms-analyzer#info=devDependencies
[david-dm-image]: https://img.shields.io/david/npms-io/npms-analyzer.svg
[david-dm-url]: https://david-dm.org/npms-io/npms-analyzer
[travis-image]: http://img.shields.io/travis/npms-io/npms-analyzer/master.svg
[travis-url]: https://travis-ci.org/npms-io/npms-analyzer
================================================
FILE: cli.js
================================================
#!/bin/sh
':' //; exec "$(command -v node)" --max-old-space-size=4192 "$0" "$@"
'use strict';
// require('heapdump');
require('./lib/configure');
const yargs = require('yargs');
yargs
.strict()
.wrap(Math.min(120, yargs.terminalWidth()))
.version()
.alias('version', 'v')
.help()
.alias('help', 'h')
.usage('npms-analyzer command line, choose one of the available commands.\n\nUsage: $0 <command> .. [options]')
.option('log-level', {
type: 'string',
alias: 'll',
choices: ['fatal', 'error', 'warn', 'info', 'debug', 'trace'],
describe: 'The log level to use',
global: true,
})
.commandDir('./cmd')
.demandCommand(1, 'Please supply a valid command')
.argv;
================================================
FILE: cmd/consume.js
================================================
'use strict';
const assert = require('assert');
const config = require('config');
const analyze = require('../lib/analyze');
const score = require('../lib/scoring/score');
const bootstrap = require('./util/bootstrap');
const stats = require('./util/stats');
// Need JSON.parse & JSON stringify because of config reserved words
// See: https://github.com/lorenwest/node-config/issues/223
const blacklist = JSON.parse(JSON.stringify(config.get('blacklist')));
const githubTokens = config.get('githubTokens');
const log = logger.child({ module: 'cli/consume' });
/**
* Handles a message.
*
* @param {Object} msg - The message.
* @param {Nano} npmNano - The npm nano instance.
* @param {Nano} npmsNano - The npms nano instance.
* @param {Elastic} esClient - The Elasticsearch instance.
*
* @returns {Promise} A promise that fulfills when consumed.
*/
function onMessage(msg, npmNano, npmsNano, esClient) {
const name = msg.data;
// Check if this package is blacklisted
const blacklisted = blacklist[name];
if (blacklisted) {
const err = Object.assign(new Error(`Package ${name} is blacklisted`), { code: 'BLACKLISTED', unrecoverable: true });
return onFailedAnalysis(name, err, npmsNano, esClient)
.catch(() => {});
}
log.info(`Processing package ${name}`);
// Check if the package has been analyzed after it has been pushed to the queue
return analyze.get(name, npmsNano)
.catch({ code: 'ANALYSIS_NOT_FOUND' }, () => {})
.then((analysis) => {
if (analysis && Date.parse(analysis.startedAt) >= Date.parse(msg.pushedAt)) {
log.info(`Skipping analysis of ${name} because it was already analyzed meanwhile`);
return;
}
// If not, analyze it! :D
return analyze(name, npmNano, npmsNano, {
githubTokens,
waitRateLimit: true,
rev: analysis && analysis._rev,
})
// Score it to get a "real-time" feeling, ignoring any errors
.then((analysis) => score(analysis, npmsNano, esClient).catch(() => {}))
.catch({ code: 'PACKAGE_NOT_FOUND' }, () => score.remove(name, esClient))
// Ignore unrecoverable errors, so that these are not re-queued
.catch({ unrecoverable: true }, (err) => (
onFailedAnalysis(name, err, npmsNano, esClient)
.catch(() => {})
));
});
}
function onFailedAnalysis(name, err, npmsNano, esClient) {
// Save the failed analysis, by generating an empty analysis object with the associated error
return analyze.saveFailed(name, err, npmsNano)
// Score it to get a "real-time" feeling, ignoring any errors
.then((analysis) => score(analysis, npmsNano, esClient).catch(() => {}));
}
// ----------------------------------------------------------------------------
exports.command = 'consume [options]';
exports.describe = 'Starts observing module changes and pushes them into the queue';
exports.builder = (yargs) =>
yargs
.usage('Usage: $0 consume [options]\n\n\
Consumes packages that are queued, triggering the analysis process for each package.')
.default('log-level', 'error')
.option('concurrency', {
type: 'number',
default: 5,
alias: 'c',
describe: 'Number of packages to consume concurrently',
})
.check((argv) => {
assert(argv.concurrency > 0, 'Invalid argument: --concurrency must be a number greater than 0');
return true;
});
exports.handler = (argv) => {
process.title = 'npms-analyzer-consume';
logger.level = argv.logLevel || 'error';
// Bootstrap dependencies on external services
bootstrap(['couchdbNpm', 'couchdbNpms', 'queue', 'elasticsearch'], { wait: true })
.spread((npmNano, npmsNano, queue, esClient) => {
// Stats
stats.process();
stats.queue(queue);
stats.progress(npmNano, npmsNano);
stats.tokens(githubTokens, 'github');
// Clean old packages from the download directory
return analyze.cleanTmpDir()
// Start consuming
.then(() => (
queue.consume((message) => onMessage(message, npmNano, npmsNano, esClient), {
concurrency: argv.concurrency,
onRetriesExceeded: (message, err) => onFailedAnalysis(message.data, err, npmsNano, esClient),
})
));
})
.done();
};
================================================
FILE: cmd/observe.js
================================================
'use strict';
const assert = require('assert');
const config = require('config');
const promiseRetry = require('promise-retry');
const realtime = require('../lib/observers/realtime');
const stale = require('../lib/observers/stale');
const bootstrap = require('./util/bootstrap');
const stats = require('./util/stats');
const log = logger.child({ module: 'cli/observe' });
/**
* Pushes a package into the queue, retrying several times on error.
* If all retries are used, there isn't much we can do, therefore the process will gracefully exit.
*
* @param {Array} name - The package name.
* @param {Number} priority - The priority to assign to this package when pushing into the queue.
* @param {Queue} queue - The analysis queue instance.
*
* @returns {Promise} The promise that fulfills once done.
*/
function onPackage(name, priority, queue) {
return promiseRetry((retry) => (
queue.push(name, priority)
.catch(retry)
))
.catch((err) => {
log.fatal({ err, name }, 'Too many failed attempts while trying to push the package into the queue, exiting..');
process.exit(1);
});
}
// ----------------------------------------------------------------------------
exports.command = 'observe [options]';
exports.describe = 'Consumes modules from the queue, analyzing them';
exports.builder = (yargs) =>
yargs
.usage('Usage: $0 observe [options]\n\n\
Starts the observing process, enqueueing packages that need to be analyzed into the queue.')
.default('log-level', 'error')
.option('default-seq', {
type: 'number',
default: 0,
alias: 'ds',
describe: 'The default seq to be used on first run',
})
.check((argv) => {
assert(argv.defaultSeq >= 0, 'Invalid argument: --default-seq must be a number greater or equal to 0');
return true;
});
exports.handler = (argv) => {
process.title = 'npms-analyzer-observe';
logger.level = argv.logLevel || 'error';
// Bootstrap dependencies on external services
bootstrap(['couchdbNpm', 'couchdbNpms', 'queue'], { wait: true })
.spread((npmNano, npmsNano, queue) => {
// Stats
stats.process();
stats.queue(queue);
// Start observing..
config.observers.realtime &&
realtime(npmNano, npmsNano, { defaultSeq: argv.defaultSeq }, (name) => onPackage(name, 1, queue));
config.observers.stale &&
stale(npmsNano, (name) => onPackage(name, 0, queue));
})
.done();
};
================================================
FILE: cmd/scoring.js
================================================
'use strict';
const assert = require('assert');
const humanizeDuration = require('humanize-duration');
const prepare = require('../lib/scoring/prepare');
const aggregate = require('../lib/scoring/aggregate');
const score = require('../lib/scoring/score');
const finalize = require('../lib/scoring/finalize');
const bootstrap = require('./util/bootstrap');
const stats = require('./util/stats');
const log = logger.child({ module: 'cli/scoring' });
/**
* Waits the time needed before running the first cycle.
*
* @param {Number} delay - The delay between each cycle.
* @param {Elastic} esClient - The Elasticsearch instance.
*
* @returns {Promise} The promise to be waited.
*/
function waitRemaining(delay, esClient) {
// Need to use Promise.resolve() because Elasticsearch doesn't use the global promise
return Promise.resolve(esClient.indices.getAlias({ name: 'npms-current' }))
.then((response) => {
const index = Object.keys(response)[0];
const timestamp = Number(index.replace(/^npms-/, ''));
const wait = timestamp ? Math.max(0, timestamp + delay - Date.now()) : 0;
const waitStr = humanizeDuration(Math.round(wait / 1000) * 1000, { largest: 2 });
wait && log.info({ now: (new Date()).toISOString() }, `Waiting ${waitStr} before running the first cycle..`);
return Promise.delay(wait);
})
.catch((err) => err.status === 404, () => {});
}
/**
* Runs a scoring cycle.
* When it finishes, another cycle will be automatically run after a certain delay.
*
* @param {Number} delay - The delay between each cycle.
* @param {Nano} npmsNano - The npm nano instance.
* @param {Elastic} esClient - The Elasticsearch instance.
*/
function cycle(delay, npmsNano, esClient) {
const startedAt = Date.now();
log.info('Starting scoring cycle');
// Prepare
prepare(esClient)
// Aggregate + score all packages
.tap(() => (
aggregate(npmsNano)
.then((aggregation) => aggregation && score.all(aggregation, npmsNano, esClient)) // If aggregation is null, there were no evaluations
))
// Finalize
.then((esInfo) => finalize(esInfo, esClient))
// We are done!
.then(() => {
const durationStr = humanizeDuration(Math.round((Date.now() - startedAt) / 1000) * 1000, { largest: 2 });
log.info(`Scoring cycle successful, took ${durationStr}`);
return delay;
}, (err) => {
log.fatal({ err }, 'Scoring cycle failed');
return 10 * 60 * 1000;
})
// Start all over again after a short delay
.then((wait) => {
const waitStr = humanizeDuration(Math.round(wait / 1000) * 1000, { largest: 2 });
log.info({ now: (new Date()).toISOString() }, `Waiting ${waitStr} before running the next cycle..`);
Promise.delay(wait)
.then(() => cycle(delay, npmsNano, esClient));
})
.done();
}
// ----------------------------------------------------------------------------
exports.command = 'scoring [options]';
exports.describe = 'Continuously iterate over the analyzed modules, scoring them';
exports.builder = (yargs) =>
yargs
.usage('Usage: $0 scoring [options]\n\n\
Continuously iterate over the analyzed packages, scoring them.')
.default('log-level', 'error')
.option('cycle-delay', {
type: 'number',
default: 3 * 60 * 60 * 1000, // 3 hours
alias: 'd',
describe: 'The time to wait between each scoring cycle (in ms)',
})
.check((argv) => {
assert(argv.cycleDelay >= 0, 'Invalid argument: --cycle-delay must be a number greater or equal to 0');
return true;
});
exports.handler = (argv) => {
// Disable long stack traces specifically for this command since we create them in bursts
// This improves performance by a great margin
Promise.config({ longStackTraces: false });
process.title = 'npms-analyzer-scoring';
logger.level = argv.logLevel || 'error';
// Bootstrap dependencies on external services
bootstrap(['couchdbNpms', 'elasticsearch'], { wait: true })
.spread((npmsNano, esClient) => {
// Stats
stats.process();
// Wait for the previous cycle delay if necessary
return waitRemaining(argv.cycleDelay, esClient)
// Start the continuous process of scoring!
.then(() => cycle(argv.cycleDelay, npmsNano, esClient));
})
.done();
};
================================================
FILE: cmd/tasks/check-gh-tokens.js
================================================
'use strict';
const config = require('config');
const got = require('got');
const githubTokens = config.get('githubTokens');
const log = logger.child({ module: 'cli/check-gh-tokens' });
exports.command = 'check-gh-tokens [options]';
exports.describe = 'Checks the status of each GitHub token';
exports.builder = (yargs) =>
yargs
.usage('Usage: $0 check-gh-tokens [options]\n\n\
Checks the status of each GitHub token.');
exports.handler = (argv) => {
process.title = 'npms-analyzer-check-gh-tokens';
logger.level = argv.logLevel;
const valid = [];
const invalid = [];
Promise.map(githubTokens, (token) => (
got.get('https://api.github.com/user', {
json: true,
headers: {
accept: 'application/vnd.github.v3+json',
authorization: `token ${token}`,
},
})
.then(() => valid.push(token))
.catch((err) => err.statusCode === 401 || err.statusCode === 403, (err) => {
log.debug({ err }, `Token ${token} seems invalid`);
invalid.push(token);
})
), { concurrency: 5 })
.then(() => {
log.info({ valid }, `${valid.length} valid tokens`);
invalid.length && log.error({ invalid }, `${invalid.length} invalid tokens`);
})
.then(() => process.exit(invalid.length ? 1 : 0))
.done();
};
================================================
FILE: cmd/tasks/clean-extraneous.js
================================================
'use strict';
const stats = require('../util/stats');
const bootstrap = require('../util/bootstrap');
const log = logger.child({ module: 'cli/clean-extraneous' });
/**
* Fetches the npm packages.
*
* @param {Nano} npmNano - The npm nano instance.
*
* @returns {Promise} The promise that fulfills when done.
*/
function fetchNpmPackages(npmNano) {
log.info('Fetching npm packages, this might take a while..');
return npmNano.listAsync()
.then((response) => (
response.rows
.map((row) => row.id)
.filter((id) => id.indexOf('_design/') !== 0)
));
}
/**
* Fetches the npms packages.
*
* @param {Nano} npmsNano - The npms nano instance.
*
* @returns {Promise} The promise that fulfills when done.
*/
function fetchNpmsPackages(npmsNano) {
log.info('Fetching npms packages, this might take a while..');
return npmsNano.listAsync({ startkey: 'package!', endkey: 'package!\ufff0' })
.then((response) =>
response.rows.map((row) =>
row.id
.split('!')
.slice(1)
.join('!')
));
}
/**
* Fetches the npms packages.
*
* @param {Nano} npmsNano - The npms nano instance.
*
* @returns {Promise} The promise that fulfills when done.
*/
function fetchNpmsObservedPackages(npmsNano) {
log.info('Fetching npms observed packages, this might take a while..');
return npmsNano.listAsync({ startkey: 'observer!package!', endkey: 'observer!package!\ufff0' })
.then((response) =>
response.rows.map((row) =>
row.id
.split('!')
.slice(2)
.join('!')
));
}
/**
* Calculates which npms packages are considered extraneous and removes them.
*
* @param {Array} npmPackages - All npm packages.
* @param {Array} npmsPackages - All npms packages.
* @param {Nano} npmsNano - The npms nano instance.
* @param {Boolean} dryRun - True to do a dry-run, false otherwise.
*
* @returns {Promise} The promise that fulfills when done.
*/
function cleanExtraneousNpmsPackages(npmPackages, npmsPackages, npmsNano, dryRun) {
log.info(
{ npmPackagesCount: npmPackages.length, npmsPackagesCount: npmsPackages.length },
'Calculating extraneous packages, this might take a while..'
);
const npmPackagesSet = new Set(npmPackages);
const extraneousPackages = npmsPackages.filter((name) => !npmPackagesSet.has(name));
log.info(`There's a total of ${extraneousPackages.length} extraneous packages`);
extraneousPackages.forEach((name) => log.debug(name));
if (!extraneousPackages.length) {
return;
}
if (dryRun) {
log.info('This is a dry-run, skipping..');
return;
}
let count = 0;
return Promise.map(extraneousPackages, (name) => {
count += 1;
count % 100 === 0 && log.info(`Removed ${count} packages`);
const key = `package!${name}`;
return npmsNano.getAsync(key)
.then((doc) => npmsNano.destroyAsync(key, doc._rev));
}, { concurrency: 15 })
.then(() => log.info('Extraneous packages were removed!'));
}
/**
* Calculates which npms observed packages are considered extraneous and removes them.
*
* @param {Array} npmPackages - All npm packages.
* @param {Array} npmsObservedPackages - All npms observed packages.
* @param {Nano} npmsNano - The npms nano instance.
* @param {Boolean} dryRun - True to do a dry-run, false otherwise.
*
* @returns {Promise} The promise that fulfills when done.
*/
function cleanExtraneousNpmsObservedPackages(npmPackages, npmsObservedPackages, npmsNano, dryRun) {
log.info(
{ npmPackagesCount: npmPackages.length, npmsObservedPackagesCount: npmsObservedPackages.length },
'Calculating extraneous observed packages, this might take a while..'
);
const npmPackagesSet = new Set(npmPackages);
const extraneousPackages = npmsObservedPackages.filter((name) => !npmPackagesSet.has(name));
log.info(`There's a total of ${extraneousPackages.length} extraneous observed packages`);
extraneousPackages.forEach((name) => log.debug(name));
if (!extraneousPackages.length || dryRun) {
log.info('This is a dry-run, skipping..');
return;
}
let count = 0;
return Promise.map(extraneousPackages, (name) => {
count += 1;
count % 100 === 0 && log.info(`Removed ${count} observed packages`);
const key = `observer!package!${name}`;
return npmsNano.getAsync(key)
.then((doc) => npmsNano.destroyAsync(key, doc._rev));
}, { concurrency: 15 })
.then(() => log.info('Extraneous observed packages were removed!'));
}
// --------------------------------------------------
exports.command = 'clean-extraneous [options]';
exports.describe = 'Finds packages that are analyzed but no longer exist in npm';
exports.builder = (yargs) =>
yargs
.usage('Usage: $0 tasks clean-extraneous [options]\n\n\
Finds packages that are analyzed but no longer exist in npm.\nThis command is useful if operations were lost due to repeated \
errors, e.g.: RabbitMQ or CouchDB were down or unstable.')
.option('dry-run', {
alias: 'dr',
type: 'boolean',
default: false,
describe: 'Enables dry-run',
});
exports.handler = (argv) => {
process.title = 'npms-analyzer-clean-extraneous';
logger.level = argv.logLevel;
bootstrap(['couchdbNpm', 'couchdbNpms'])
.spread((npmNano, npmsNano) => {
// Stats
stats.process();
// The strategy below loads all packages in memory.. we can do this because the total packages is around ~250k
// which fit well in memory and is much faster than doing manual iteration (~20sec vs ~3min)
// Fetch npm packages
return fetchNpmPackages(npmNano)
// Fetch npms packages & clean extraneous
.tap((npmPackages) => (
fetchNpmsPackages(npmsNano)
.then((npmsPackages) => cleanExtraneousNpmsPackages(npmPackages, npmsPackages, npmsNano, argv.dryRun))
))
// Fetch npms observed packages & clean extraneous
.then((npmPackages) => (
fetchNpmsObservedPackages(npmsNano)
.then((npmsObservedPackages) => cleanExtraneousNpmsObservedPackages(npmPackages, npmsObservedPackages, npmsNano, argv.dryRun))
));
})
.then(() => process.exit())
.done();
};
================================================
FILE: cmd/tasks/enqueue-missing.js
================================================
'use strict';
const difference = require('lodash/difference');
const bootstrap = require('../util/bootstrap');
const stats = require('../util/stats');
const log = logger.child({ module: 'cli/enqueue-missing' });
/**
* Fetches the npm packages.
*
* @param {Nano} npmNano - The npm nano instance.
*
* @returns {Promise} The promise that fulfills when done.
*/
function fetchNpmPackages(npmNano) {
log.info('Fetching npm packages, this might take a while..');
return npmNano.listAsync()
.then((response) => (
response.rows
.map((row) => row.id)
.filter((id) => id.indexOf('_design/') !== 0)
));
}
/**
* Fetches the npms packages.
*
* @param {Nano} npmsNano - The npms nano instance.
*
* @returns {Promise} The promise that fulfills when done.
*/
function fetchNpmsPackages(npmsNano) {
log.info('Fetching npms packages, this might take a while..');
return npmsNano.listAsync({ startkey: 'package!', endkey: 'package!\ufff0' })
.then((response) => response.rows.map((row) => row.id.split('!')[1]));
}
/**
* Calculates which packages are missing and enqueues them.
*
* @param {Array} npmPackages - All npm packages.
* @param {Array} npmsPackages - All npms packages.
* @param {Queue} queue - The analysis queue instance.
* @param {Boolean} dryRun - True to do a dry-run, false otherwise.
*
* @returns {Promise} The promise that fulfills when done.
*/
function enqueueMissingPackages(npmPackages, npmsPackages, queue, dryRun) {
const missingPackages = difference(npmPackages, npmsPackages);
log.info(`There's a total of ${missingPackages.length} missing packages`);
missingPackages.forEach((name) => log.debug(name));
if (!missingPackages.length) {
return;
}
if (dryRun) {
log.info('This is a dry-run, skipping..');
return;
}
let count = 0;
return Promise.map(missingPackages, (name) => {
count += 1;
count % 1000 === 0 && log.info(`Enqueued ${count} packages`);
return queue.push(name);
}, { concurrency: 15 })
.then(() => log.info('Missing packages were enqueued!'));
}
// --------------------------------------------------
exports.command = 'enqueue-missing [options]';
exports.describe = 'Finds packages that were not analyzed and enqueues them';
exports.builder = (yargs) =>
yargs
.usage('Usage: $0 tasks enqueue-missing [options]\n\n\
Finds packages that were not analyzed and enqueues them.\nThis command is useful if packages were lost due to repeated transient \
errors, e.g.: internet connection was lot or GitHub was down.')
.option('dry-run', {
alias: 'dr',
type: 'boolean',
default: false,
describe: 'Enables dry-run',
});
exports.handler = (argv) => {
process.title = 'npms-analyzer-enqueue-missing';
logger.level = argv.logLevel;
// Bootstrap dependencies on external services
bootstrap(['couchdbNpm', 'couchdbNpms', 'queue'])
.spread((npmNano, npmsNano, queue) => {
// Stats
stats.process();
// The strategy below loads all packages in memory.. we can do this because the total packages is around ~250k
// which fit well in memory and is much faster than doing manual iteration (~20sec vs ~3min)
return Promise.all([
fetchNpmPackages(npmNano),
fetchNpmsPackages(npmsNano),
])
.spread((npmPackages, npmsPackages) => enqueueMissingPackages(npmPackages, npmsPackages, queue, argv.dryRun));
})
.then(() => process.exit())
.done();
};
================================================
FILE: cmd/tasks/enqueue-outdated.js
================================================
'use strict';
const bootstrap = require('../util/bootstrap');
const stats = require('../util/stats');
const log = logger.child({ module: 'cli/enqueue-outdated' });
/**
* Fetches the npm packages.
*
* @param {Nano} npmNano - The npm nano instance.
*
* @returns {Promise} The promise that fulfills when done.
*/
function fetchNpmPackages(npmNano) {
log.info('Fetching npm packages, this might take a while..');
return npmNano.viewAsync('npms-analyzer', 'packages-version')
.then((response) => (
response.rows
.map((row) => ({ name: row.key, version: row.value }))
));
}
/**
* Fetches the npms packages.
*
* @param {Nano} npmsNano - The npms nano instance.
*
* @returns {Promise} The promise that fulfills when done.
*/
function fetchNpmsPackages(npmsNano) {
log.info('Fetching npms packages, this might take a while..');
return npmsNano.viewAsync('npms-analyzer', 'packages-version')
.then((response) => (
response.rows
.map((row) => ({ name: row.key, version: row.value }))
));
}
/**
* Calculates which packages are outdated (missing or version mismatch) and enqueues them.
*
* @param {Array} npmPackages - All npm packages.
* @param {Array} npmsPackages - All npms packages.
* @param {Queue} queue - The analysis queue instance.
* @param {Boolean} dryRun - True to do a dry-run, false otherwise.
*
* @returns {Promise} The promise that fulfills when done.
*/
function enqueueOutdated(npmPackages, npmsPackages, queue, dryRun) {
log.info(
{ npmPackagesCount: npmPackages.length, npmsPackagesCount: npmsPackages.length },
'Calculating outdated packages, this might take a while..'
);
const npmsPackagesMap = npmsPackages.reduce((npmsPackagesMap, pkg) => npmsPackagesMap.set(pkg.name, pkg.version), new Map());
const outdatedPackages = npmPackages.filter((pkg) => npmsPackagesMap.get(pkg.name) !== pkg.version);
log.info(`There's a total of ${outdatedPackages.length} outdated packages`);
outdatedPackages.forEach((pkg) => log.debug(pkg.name));
if (!outdatedPackages.length) {
return;
}
if (dryRun) {
log.info('This is a dry-run, skipping..');
return;
}
let count = 0;
return Promise.map(outdatedPackages, (pkg) => {
count += 1;
count % 1000 === 0 && log.info(`Enqueued ${count} packages`);
return queue.push(pkg.name);
}, { concurrency: 15 })
.then(() => log.info('Outdated packages were enqueued!'));
}
// --------------------------------------------------
exports.command = 'enqueue-outdated [options]';
exports.describe = 'Finds packages that are outdated and enqueues them';
exports.builder = (yargs) =>
yargs
.usage('Usage: $0 tasks enqueue-outdated [options]\n\n\
Finds packages that are outdated and enqueues them.\nThis command is useful if packages were lost due to repeated transient \
errors, e.g.: internet connection was lot or GitHub was down.')
.option('dry-run', {
alias: 'dr',
type: 'boolean',
default: false,
describe: 'Enables dry-run',
});
exports.handler = (argv) => {
process.title = 'npms-analyzer-enqueue-outdated';
logger.level = argv.logLevel;
// Bootstrap dependencies on external services
bootstrap(['couchdbNpm', 'couchdbNpms', 'queue'])
.spread((npmNano, npmsNano, queue) => {
// Stats
stats.process();
// The strategy below loads all packages in memory.. we can do this because the total packages is around ~250k
// which fit well in memory and is much faster than doing manual iteration (~20sec vs ~3min)
return Promise.all([
fetchNpmPackages(npmNano),
fetchNpmsPackages(npmsNano),
])
.spread((npmPackages, npmsPackages) => enqueueOutdated(npmPackages, npmsPackages, queue, argv.dryRun));
})
.then(() => process.exit())
.done();
};
================================================
FILE: cmd/tasks/enqueue-view.js
================================================
'use strict';
const assert = require('assert');
const bootstrap = require('../util/bootstrap');
const stats = require('../util/stats');
const log = logger.child({ module: 'cli/enqueue-view' });
/**
* Fetches packages of a view.
*
* @param {String} view - The view in the form of design-doc/view-name.
* @param {Nano} npmNano - The npm nano instance.
*
* @returns {Promise} The promise that fulfills when done.
*/
function fetchView(view, npmNano) {
log.info(`Fetching view ${view}`);
const split = view.split('/');
return npmNano.viewAsync(split[0], split[1])
.then((response) => (
response.rows
.map((row) => row.key.replace(/^package!/, ''))
));
}
/**
* Enqueues packages to be analyzed.
*
* @param {Array} packages - The package names to be enqueued.
* @param {Queue} queue - The analysis queue instance.
* @param {Boolean} dryRun - True to do a dry-run, false otherwise.
*
* @returns {Promise} The promise that fulfills when done.
*/
function enqueueViewPackages(packages, queue, dryRun) {
log.info(`There's a total of ${packages.length} packages in the view`);
packages.forEach((name) => log.debug(name));
if (!packages.length) {
return;
}
if (dryRun) {
log.info('This is a dry-run, skipping..');
return;
}
let count = 0;
return Promise.map(packages, (name) => {
count += 1;
count % 5000 === 0 && log.info(`Enqueued ${count} packages`);
return queue.push(name);
}, { concurrency: 15 })
.then(() => log.info('View packages were enqueued!'));
}
// --------------------------------------------------
exports.command = 'enqueue-view <view> [options]';
exports.describe = 'Enqueues all packages contained in a npms view';
exports.builder = (yargs) =>
yargs
.usage('Usage: $0 tasks enqueue-view <design-doc/view-name> [options]\n\n\
Enqueues all packages contained in the npms database view.\n\nNOTE: The view must be in the npms database and the key must be the package \
name (may be prefixed with `package!`)')
.example('$0 tasks enqueue-view npms-analyzer/docs-to-be-fixed')
.option('dry-run', {
alias: 'dr',
type: 'boolean',
default: false,
describe: 'Enables dry-run',
})
.check((argv) => {
assert(/^[a-z0-9_-]+\/[a-z0-9_-]+$/.test(argv.view), 'The view argument must match the following format: <design-doc/view-name>');
return true;
});
exports.handler = (argv) => {
process.title = 'npms-analyzer-enqueue-view';
logger.level = argv.logLevel;
const view = argv.view;
// Bootstrap dependencies on external services
bootstrap(['couchdbNpm', 'couchdbNpms', 'queue'])
.spread((npmNano, npmsNano, queue) => {
// Stats
stats.process();
// The strategy below loads all packages in memory.. we can do this because the total packages is around ~250k
// which fit well in memory and is much faster than doing manual iteration (~20sec vs ~3min)
return fetchView(view, npmsNano)
.then((packages) => enqueueViewPackages(packages, queue, argv.dryRun));
})
.then(() => process.exit())
.done();
};
================================================
FILE: cmd/tasks/migrate.js
================================================
'use strict';
const couchdbIterator = require('couchdb-iterator');
const analyze = require('../../lib/analyze');
const bootstrap = require('../util/bootstrap');
const stats = require('../util/stats');
const log = logger.child({ module: 'cli/migrate' });
function extractScope(name) {
const match = name.match(/^@([^/]+)\/.+$/);
return match ? match[1] : 'unscoped';
}
// --------------------------------------------------------------
exports.command = 'migrate [options]';
exports.describe = 'Run the latest migration';
exports.builder = (yargs) =>
yargs
.usage('Usage: $0 tasks migrate [options]\n\n\
Run the latest migration.');
exports.handler = (argv) => {
process.title = 'npms-analyzer-migrate';
logger.level = argv.logLevel;
// Bootstrap dependencies on external services
bootstrap(['couchdbNpm', 'couchdbNpms'])
.spread((npmNano, npmsNano) => {
log.info('Starting migration');
// Stats
stats.process();
// Iterate over all packages
return couchdbIterator(npmsNano, (row) => {
row.index && row.index % 2500 === 0 && log.info(`Processed ${row.index} rows`);
if (!row.doc) {
return;
}
const name = row.doc.collected.metadata.name;
row.doc.collected.metadata.scope = extractScope(name);
return analyze.save(row.doc, npmsNano)
.catch((err) => {
log.error({ err }, `Failed to process ${name}`);
throw err;
});
}, {
startkey: 'package!',
endkey: 'package!\ufff0',
concurrency: 25,
limit: 2500,
includeDocs: true,
})
.then((count) => log.info(`Completed, processed a total of ${count} rows`));
})
.then(() => process.exit())
.done();
};
================================================
FILE: cmd/tasks/process-package.js
================================================
'use strict';
const config = require('config');
const analyze = require('../../lib/analyze');
const score = require('../../lib/scoring/score');
const bootstrap = require('../util/bootstrap');
const log = logger.child({ module: 'cli/process-package' });
exports.command = 'process-package <package> [options]';
exports.describe = 'Processes a single package, analyzing and scoring it';
exports.builder = (yargs) =>
yargs
.usage('Usage: $0 tasks process-package <package> [options]\n\n\
Processes a single package, analyzing and scoring it.')
.example('$0 tasks process-package analyze cross-spawn')
.example('$0 tasks process-package analyze cross-spawn --no-analyze', 'Just score the package, do not analyze')
.option('analyze', {
type: 'boolean',
default: true,
describe: 'Either to analyze and score or just score',
});
exports.handler = (argv) => {
process.title = 'npms-analyzer-process-package';
logger.level = argv.logLevel;
const name = argv.package.toString(); // Package 0 evaluates to number so we must cast to a string
// Bootstrap dependencies on external services
bootstrap(['couchdbNpm', 'couchdbNpms', 'elasticsearch'])
.spread((npmNano, npmsNano, esClient) => (
// Analyze the package
Promise.try(() => {
if (!argv.analyze) {
return analyze.get(name, npmsNano);
}
return analyze(name, npmNano, npmsNano, {
githubTokens: config.get('githubTokens'),
});
})
.tap((analysis) => log.info({ analysis }, 'Analyze data'))
// Score the package
.then((analysis) => (
score(analysis, npmsNano, esClient)
.tap((score) => log.info({ score }, 'Score data'))
.catch({ code: 'SCORE_INDEX_NOT_FOUND' }, () => {})
))
.catch({ code: 'PACKAGE_NOT_FOUND' }, (err) => score.remove(name, esClient).finally(() => { throw err; }))
))
.then(() => process.exit())
.done();
};
================================================
FILE: cmd/tasks/re-evaluate.js
================================================
'use strict';
const couchdbIterator = require('couchdb-iterator');
const evaluate = require('../../lib/analyze/evaluate');
const save = require('../../lib/analyze').save;
const bootstrap = require('../util/bootstrap');
const stats = require('../util/stats');
const log = logger.child({ module: 'cli/re-evaluate' });
exports.command = 're-evaluate [options]';
exports.describe = 'Iterates over all analyzed packages, evaluating them again';
exports.builder = (yargs) =>
yargs
.usage('Usage: $0 tasks re-evaluate [options]\n\n\
Iterates over all analyzed packages, evaluating them again.\nThis command is useful if the evaluation algorithm has changed and \
the evaluation needs to be re-calculated for all packages. Note that the packages score won\'t be updated.');
exports.handler = (argv) => {
process.title = 'npms-analyzer-re-evaluate';
logger.level = argv.logLevel;
// Bootstrap dependencies on external services
bootstrap(['couchdbNpms'])
.spread((npmsNano) => {
log.info('Starting packages re-evaluation');
// Stats
stats.process();
// Iterate over all packages, re-evaluating them
return couchdbIterator(npmsNano, (row) => {
row.index && row.index % 10000 === 0 && log.info(`Processed ${row.index} rows`);
const doc = row.doc;
if (!doc) {
return;
}
const name = doc.collected.metadata.name;
log.debug(`Evaluating ${name}..`);
return Promise.try(() => {
doc.evaluation = evaluate(doc.collected);
return save(doc, npmsNano);
})
.catch((err) => {
log.error({ err }, `Failed to evaluate ${name}`);
throw err;
});
}, {
startkey: 'package!',
endkey: 'package!\ufff0',
concurrency: 25,
limit: 2500,
includeDocs: true,
})
.then((count) => log.info(`Completed, processed a total of ${count} rows`));
})
.then(() => process.exit())
.done();
};
================================================
FILE: cmd/tasks/re-metadata.js
================================================
'use strict';
const couchdbIterator = require('couchdb-iterator');
const metadata = require('../../lib/analyze/collect/metadata');
const packageJsonFromData = require('../../lib/analyze/util/packageJsonFromData');
const analyze = require('../../lib/analyze');
const bootstrap = require('../util/bootstrap');
const stats = require('../util/stats');
const log = logger.child({ module: 'cli/re-metadata' });
exports.command = 're-metadata [options]';
exports.describe = 'Iterates over all analyzed packages, running the metadata collector again';
exports.builder = (yargs) =>
yargs
.usage('Usage: $0 tasks re-metadata [options]\n\n\
Iterates over all analyzed packages, running the metadata collector again.\nThis command is useful if there was a bug in the \
metadata collector. Note that the packages score won\'t be updated.');
exports.handler = (argv) => {
process.title = 'npms-analyzer-re-metadata';
logger.level = argv.logLevel;
// Bootstrap dependencies on external services
bootstrap(['couchdbNpm', 'couchdbNpms'])
.spread((npmNano, npmsNano) => {
log.info('Starting packages re-metadata');
// Stats
stats.process();
// Iterate over all packages
return couchdbIterator(npmsNano, (row) => {
row.index && row.index % 2500 === 0 && log.info(`Processed ${row.index} rows`);
if (!row.doc) {
return;
}
const name = row.id
.split('!')
.slice(1)
.join('!');
// Grab package data
return npmNano.getAsync(name)
.then((data) => {
let packageJson;
// Extract package json
try {
packageJson = packageJsonFromData(name, data);
} catch (err) {
if (!err.unrecoverable) {
throw err;
}
// Remove the package if an unrecoverable error happened
// We do this to prevent old metadata to stay around, which will probably cause issues further ahead
return analyze.remove(name, npmsNano);
}
// Re-run metadata
return metadata(data, packageJson)
// Save it!
.then((metadata) => {
row.doc.collected.metadata = metadata;
return analyze.save(row.doc, npmsNano);
})
.catch((err) => {
log.error({ err }, `Failed to process ${name}`);
throw err;
});
})
// Delete the analisis if the package does not exist in npm (e.g.: was deleted)
.catch({ error: 'not_found' }, () => analyze.remove(name, npmsNano));
}, {
startkey: 'package!',
endkey: 'package!\ufff0',
concurrency: 25,
limit: 2500,
includeDocs: true,
})
.then((count) => log.info(`Completed, processed a total of ${count} rows`));
})
.then(() => process.exit())
.done();
};
================================================
FILE: cmd/tasks.js
================================================
'use strict';
exports.command = 'tasks';
exports.describe = 'Execute a task';
exports.builder = (yargs) =>
yargs
.usage('Group of task commands, choose one of the available commands.\n\nUsage: $0 tasks <command> [options]')
.default('log-level', 'info')
.commandDir('./tasks')
.demandCommand(1, 'Please supply a valid command');
================================================
FILE: cmd/util/bootstrap.js
================================================
'use strict';
const config = require('config');
const nano = require('nano');
const elasticsearch = require('elasticsearch');
const promiseRetry = require('promise-retry');
const get = require('lodash/get');
const queue = require('../../lib/queue');
const retriesOption = { minTimeout: 2500, retries: 5 };
const log = logger.child({ module: 'bootstrap' });
/**
* Bootstrap several dependencies, waiting for them to be ready: CouchDB, Elasticsearch and Queue.
* Tries several times before failing.
*
* @param {Object} deps - The dependencies to setup.
* @param {Object} [options] - The options; read bellow to get to know each available option.
*
* @returns {Promise} The promise that resolves when they are ready.
*/
function bootstrap(deps, options) {
options = Object.assign({
wait: false, // True to wait for the dependencies to be ready (in case they are unavailable)
}, options);
// Log uncaught exceptions
process.on('uncaughtException', (err) => {
log.fatal({ err }, `Uncaught exception: ${err.message}`);
throw err;
});
return Promise.map(deps, (dep) => {
switch (dep) {
case 'couchdbNpm':
case 'couchdbNpms':
return bootstrapCouchdb(config.get(dep), options);
case 'elasticsearch':
return bootstrapElasticsearch(config.get('elasticsearch'), options);
case 'queue':
return bootstrapQueue(config.get('queue'), options);
default:
throw new Error(`Unknown dependency: ${dep}`);
}
});
}
// ----------------------------------------------------------------------------
/**
* Bootstraps a CouchDB database client, returning a nano instance.
*
* @param {Object} config - The CouchDB config.
* @param {Object} options - The options inferred from bootstrap().
*
* @returns {Promise} The promise that resolves when done.
*/
function bootstrapCouchdb(config, options) {
const nanoClient = Promise.promisifyAll(nano(config));
if (!nanoClient.config.db) {
throw new Error('Expected CouchDB URL to point to a DB');
}
nanoClient.serverScope = Promise.promisifyAll(nano(Object.assign({}, config, { url: nanoClient.config.url })));
return promiseRetry((retry) => (
nanoClient.getAsync('somedocthatwillneverexist')
.catch({ error: 'not_found' }, () => {})
.catch((err) => {
log.warn({ err }, `Check of ${nanoClient.config.db} failed`);
retry(err);
})
), options.wait ? retriesOption : { retries: 0 })
.then(() => log.debug(`CouchDB for ${nanoClient.config.db} is ready`))
.return(nanoClient);
}
/**
* Bootstraps a Elasticsearch client.
*
* @param {Object} config - The Elasticsearch config.
* @param {Object} options - The options inferred from bootstrap().
*
* @returns {Promise} The promise that resolves when done.
*/
function bootstrapElasticsearch(config, options) {
const esClient = new elasticsearch.Client(config);
return promiseRetry((retry) => (
Promise.resolve(esClient.get({
index: 'someindexthatwillneverexist',
type: 'sometypethatwillneverexist',
id: 'someidthatwillneverexist',
maxRetries: 0,
}))
.catch((err) => get(err, 'body.error.type') === 'index_not_found_exception', () => {})
.catch((err) => {
log.warn({ err }, 'Check of Elasticsearch failed');
retry(err);
})
), options.wait ? retriesOption : { retries: 0 })
.then(() => log.debug('Elasticsearch is ready'))
.return(esClient);
}
/**
* Bootstraps the analysis queue.
*
* @param {Object} config - The queue config.
* @param {Object} options - The options inferred from bootstrap().
*
* @returns {Promise} The promise that resolves when done.
*/
function bootstrapQueue(config, options) {
const analysisQueue = queue(config.name, config.addr, config.options);
return promiseRetry((retry) => (
analysisQueue.stat()
.catch((err) => {
log.warn({ err }, 'Check of Queue failed');
retry(err);
})
), options.wait ? retriesOption : { retries: 0 })
.then(() => log.debug('Queue is ready'))
.return(analysisQueue);
}
module.exports = bootstrap;
================================================
FILE: cmd/util/stats/index.js
================================================
'use strict';
module.exports = require('require-directory')(module, './', { recurse: false });
================================================
FILE: cmd/util/stats/process.js
================================================
'use strict';
const pino = require('pino');
const humanizeDuration = require('humanize-duration');
const log = logger.child({ module: 'stats/process' });
/**
* Continuously monitor the process, printing metrics such as the memory and uptime.
*/
function statProcess() {
// Do nothing if loglevel is higher than info
if (log.levelVal > pino.levels.values.info) {
return;
}
const pid = process.pid;
setInterval(() => {
const memoryUsage = (process.memoryUsage().heapUsed / 1024 / 1024).toFixed(2);
const uptime = humanizeDuration(Math.round(process.uptime()) * 1000, { largest: 1 });
log.info(`pid: ${pid}; memory: ${memoryUsage} MB; uptime: ${uptime}`);
}, 15000)
.unref();
}
module.exports = statProcess;
================================================
FILE: cmd/util/stats/progress.js
================================================
'use strict';
const pino = require('pino');
const log = logger.child({ module: 'stats/progress' });
// TODO: Add status for replication and other stuff?
/**
* Continuously monitor the analyzer progress, printing information such as the analysis %.
*
* @param {Nano} npmNano - The npm nano client instance.
* @param {Nano} npmsNano - The npms nano client instance.
*/
function statProgress(npmNano, npmsNano) {
// Do nothing if loglevel is higher than info
if (log.levelVal > pino.levels.values.info) {
return;
}
let pending = false;
setInterval(() => {
if (pending) {
log.info('Progress stat is still being retrieved..');
return;
}
pending = true;
Promise.props({
npmDocsCount: npmNano.infoAsync().then((res) => res.doc_count),
npmDesignDocsCount: npmNano.listAsync({ startkey: '_design/', endkey: '_design0' }).then((res) => res.rows.length),
npmsPackagesCount: npmsNano.viewAsync('npms-analyzer', 'packages-evaluation', { reduce: true })
.then((res) => res.rows[0] ? res.rows[0].value : 0),
})
.finally(() => { pending = false; })
.then((result) => {
const analysis = `${(result.npmsPackagesCount / (result.npmDocsCount - result.npmDesignDocsCount) * 100).toFixed(4)}%`;
log.info({ analysis }, 'Progress stat');
}, (err) => {
log.error({ err }, 'Progress stat failed');
})
.done();
}, 15000)
.unref();
}
module.exports = statProgress;
================================================
FILE: cmd/util/stats/queue.js
================================================
'use strict';
const pino = require('pino');
const log = logger.child({ module: 'stats/queue' });
/**
* Continuously monitor the queue, printing information such as the number of enqueued messages.
*
* @param {Queue} queue - The queue instance.
*/
function statQueue(queue) {
// Do nothing if loglevel is higher than info
if (log.levelVal > pino.levels.values.info) {
return;
}
let pending = false;
setInterval(() => {
if (pending) {
log.info('Queue stat is still being retrieved..');
return;
}
pending = true;
queue.stat()
.finally(() => { pending = false; })
.then((stat) => {
log.info({ stat }, 'Queue stat');
}, (err) => {
log.error({ err }, 'Queue stat failed');
})
.done();
}, 15000)
.unref();
}
module.exports = statQueue;
================================================
FILE: cmd/util/stats/tokens.js
================================================
'use strict';
const pino = require('pino');
const tokenDealer = require('token-dealer');
const minBy = require('lodash/minBy');
const log = logger.child({ module: 'stats/tokens' });
/**
* Monitors the API tokens managed by token-dealer of a given group.
*
* @param {Array} tokens - The array of tokens.
* @param {String} [group] - The token's group (e.g.: Github).
*/
function statTokens(tokens, group) {
// Do nothing if loglevel is higher than info
if (log.levelVal > pino.levels.values.info) {
return;
}
setInterval(() => {
const tokensUsage = Object.values(tokenDealer.getTokensUsage(tokens, { group }));
const usableTokensUsage = tokensUsage.filter((entry) => !entry.exhausted);
if (usableTokensUsage.length) {
log.info(`${usableTokensUsage.length} out of ${tokensUsage.length} tokens are usable (${group})`);
return;
}
if (tokensUsage.length < 1) {
log.info(`No tokens (${group})`);
return;
}
const nextResettingToken = minBy(tokensUsage, 'reset');
const remainingMins = Math.ceil((nextResettingToken.reset - Date.now()) / 1000 / 60);
log.info(`All tokens are exhausted, next one will reset in ${remainingMins} minutes (${group})`);
}, 15000)
.unref();
}
module.exports = statTokens;
================================================
FILE: config/couchdb/npms-analyzer-npm.json
================================================
{
"_id": "_design/npms-analyzer",
"language": "javascript",
"views": {
"packages-version": {
"map": "function (doc) {\n if (doc['dist-tags'] && doc['dist-tags'].latest) {\n emit(doc._id, doc['dist-tags'].latest);\n }\n}"
}
}
}
================================================
FILE: config/couchdb/npms-analyzer-npms.json
================================================
{
"_id": "_design/npms-analyzer",
"language": "javascript",
"views": {
"packages-evaluation": {
"map": "function (doc) {\n if (doc._id.indexOf('package!') === 0) {\n emit(doc._id.split('!').slice(1).join('!'), doc.evaluation);\n }\n}",
"reduce": "_count"
},
"packages-stale": {
"map": "function (doc) {\n if (doc._id.indexOf('package!') === 0) {\n if (doc.error) {\n if (!doc.error.unrecoverable) {\n emit(['failed', Date.parse(doc.error.caughtAt), doc._id.split('!').slice(1).join('!')]);\n }\n } else {\n emit(['normal', Date.parse(doc.finishedAt), doc._id.split('!').slice(1).join('!')]);\n }\n }\n}"
},
"packages-version": {
"map": "function (doc) {\n if (doc._id.indexOf('package!') === 0) {\n emit(doc._id.split('!').slice(1).join('!'), doc.collected.metadata.version);\n }\n}"
}
}
}
================================================
FILE: config/default.json5
================================================
{
// Databases & similar stuff
couchdbNpm: {
url: 'http://admin:admin@127.0.0.1:5984/npm',
requestDefaults: { timeout: 15000 },
},
couchdbNpms: {
url: 'http://admin:admin@127.0.0.1:5984/npms',
requestDefaults: { timeout: 15000 },
},
elasticsearch: {
host: 'http://127.0.0.1:9200',
requestTimeout: 15000,
apiVersion: '6.3',
log: null,
},
queue: {
name: 'npms',
addr: 'amqp://guest:guest@127.0.0.1',
options: { maxPriority: 1 },
},
// List of packages that will be ignored by the CLI consume command (analysis process)
blacklist: {
'hownpm': 'Invalid version: 1.01',
'zachtestproject1': 'Test project that makes registry return 500 internal',
'zachtestproject2': 'Test project that makes registry return 500 internal',
'zachtestproject3': 'Test project that makes registry return 500 internal',
'zachtestproject4': 'Test project that makes registry return 500 internal',
'broken-package-truncated-tar-header': 'Broken tarball',
},
// Github tokens to be used by token-dealer
githubTokens: [],
// Enabled observers.
observers: {
realtime: true,
stale: false,
}
}
================================================
FILE: config/elasticsearch/npms.json5
================================================
{
// ------------------------------------------------------------------------------
// Index settings
// ------------------------------------------------------------------------------
"settings" : {
"number_of_shards" : 1,
"number_of_replicas": 0,
"analysis": {
// Custom tokenizers
"tokenizer": {
// Exclusive tokenizer used for autocompletion highlight so that it correctly highlights partial words..
// See: https://github.com/elastic/elasticsearch/issues/3137#issuecomment-22116469
"autocomplete": {
"type": "edgeNGram",
"min_gram": "1",
"max_gram": "15",
"token_chars": ["letter", "digit"],
},
},
// Custom filters
"filter": {
"non_alfanum_to_space": {
"type": "pattern_replace",
"pattern": "(?i)[^a-z0-9]+",
"replacement": " ",
},
// Split word filter, which takes tokens, such as es6_promise or lodash.foreach, and splits them
// into several other tokens
"split_word": {
"type": "word_delimiter",
"generate_word_parts": true,
"generate_number_parts": true,
"catenate_words": false,
"catenate_numbers": false,
"catenate_all": false,
"split_on_case_change": true,
"preserve_original": true,
"split_on_numerics": true,
"stem_english_possessive": true,
},
// Edge ngram to provide fallback to stemming
"edge_ngram": {
"type": "edgeNGram",
"min_gram": "4",
"max_gram": "15",
},
// Dedicated filter for autocompletion
"autocomplete": {
"type": "edgeNGram",
"min_gram": "1",
"max_gram": "15",
},
// Remove duplicate tokens
"unique_on_same_position": {
"type": "unique",
"only_on_same_position": false,
},
},
// Custom analyzers
"analyzer": {
// The packages_* series produces good results for the `name` and `keywords` fields
"package": {
"tokenizer": "standard",
"filter": [
"asciifolding",
"split_word",
"lowercase",
"unique_on_same_position",
],
},
"package_english": {
"tokenizer": "standard",
"filter": [
"asciifolding",
"split_word",
"lowercase",
"kstem", // Non-aggressive english stemmer
"unique_on_same_position",
],
},
"package_english_aggressive": {
"tokenizer": "standard",
"filter": [
"asciifolding",
"split_word",
"lowercase",
"porter_stem", // Aggressive english stemmer
"unique_on_same_position",
],
},
"package_edge_ngram": {
// This analyzer provides fallbacks in which the stemmer is not efficient, e.g.: searching for "glob" should match "globby"
"tokenizer": "standard",
"filter": [
"asciifolding",
"split_word",
"lowercase",
"edge_ngram",
"unique_on_same_position",
],
},
// The package_autocomplete_* series produces good results for autocompletion
"package_autocomplete": {
"tokenizer": "standard",
"filter": [
"asciifolding",
"split_word",
"lowercase",
"autocomplete",
"unique_on_same_position",
],
},
"package_autocomplete_keyword": {
// This analyzer emits the whole string but replaces non-alfanum with spaces
// so that we can use it boost exact prefix matches higher
"tokenizer": "keyword",
"filter": [
"asciifolding",
"non_alfanum_to_space",
"lowercase",
"autocomplete",
"trim",
"unique_on_same_position",
],
},
"package_autocomplete_keyword_search": {
// This analyzer is the "search_analyzer" for "package_autocomplete_keyword"
"tokenizer": "keyword",
"filter": [
"asciifolding",
"non_alfanum_to_space",
"lowercase",
"trim",
],
},
"package_autocomplete_highlight": {
// This analyzer is necessary to perform proper highlighting
// See: https://github.com/elastic/elasticsearch/issues/3137#issuecomment-22116469
// Can't use split_word which may lead to "strange" highlighting in some edge cases :(
"tokenizer": "autocomplete",
"filter": [
"asciifolding",
"non_alfanum_to_space",
"lowercase",
"trim",
],
},
// The raw analyzer does very small normalizations
"raw": {
"tokenizer": "keyword",
"filter": [
"asciifolding",
"lowercase",
"trim",
],
},
},
// Custom normalizers
"normalizer": {
// The raw analyzer does very small normalizations
"raw": {
"type": "custom",
"filter": [
"asciifolding",
"lowercase",
"trim",
],
},
},
},
},
// ------------------------------------------------------------------------------
// Mappings
// ------------------------------------------------------------------------------
"mappings" : {
"score" : {
"dynamic": false,
"_all": {
"enabled": false,
},
"properties" : {
"package": {
"type": "object",
"properties": {
// The `name`, `description` and `keywords` fields all have `norms` disabled because we don't care about the fields length
// Also they have additional fields beyond the `standard` analyzer, such as `package_english`, `package_english` etc
"name": {
"type": "text",
"fields": {
"standard": {
"type": "text",
"analyzer": "standard",
},
"english": {
"type": "text",
"analyzer": "package_english",
},
"english_aggressive": {
"type": "text",
"analyzer": "package_english_aggressive",
},
"edge_ngram": {
"type": "text",
"analyzer": "package_edge_ngram",
"search_analyzer": "package",
},
"autocomplete": {
"type": "text",
"analyzer": "package_autocomplete",
"search_analyzer": "package",
},
"autocomplete_highlight": {
"type": "text",
"analyzer": "package_autocomplete_highlight",
"search_analyzer": "package",
"index_options": "offsets",
},
"autocomplete_keyword": {
"type": "text",
"analyzer": "package_autocomplete_keyword",
"search_analyzer": "package_autocomplete_keyword_search",
},
"raw": {
"type": "keyword",
"normalizer": "raw",
},
},
},
"description": {
"type": "text",
"fields": {
"standard": {
"type": "text",
"analyzer": "standard",
},
"english": {
"type": "text",
"analyzer": "package_english",
},
"english_aggressive": {
"type": "text",
"analyzer": "package_english_aggressive",
},
"edge_ngram": {
"type": "text",
"analyzer": "package_edge_ngram",
"search_analyzer": "package",
},
},
},
"keywords": {
"type": "text",
"fields": {
"standard": {
"type": "text",
"analyzer": "standard",
},
"english": {
"type": "text",
"analyzer": "package_english",
},
"english_aggressive": {
"type": "text",
"analyzer": "package_english_aggressive",
},
"edge_ngram": {
"type": "text",
"analyzer": "package_edge_ngram",
"search_analyzer": "package",
},
"raw": {
"type": "text",
"analyzer": "raw",
}
},
},
"version": {
"type": "text",
"index": false,
},
"date": {
"type": "date",
"index": false,
},
"links": {
"type": "object",
"properties": {
"npm": { "type": "text", "index": false },
"homepage": { "type": "text", "index": false },
"repository": { "type": "text", "index": false },
"bugs": { "type": "text", "index": false },
},
},
"author": {
"type": "object",
"properties": {
"name": {
"type": "keyword",
"normalizer": "raw",
},
"username": {
"type": "keyword",
"normalizer": "raw",
},
"email": {
"type": "keyword",
"normalizer": "raw",
},
"url": { "type": "text", "index": false },
},
},
"publisher": {
"type": "object",
"properties": {
"username": { "type": "text", "index": false },
"email": { "type": "text", "index": false },
},
},
"maintainers": {
"type": "object",
"properties": {
"username": {
"type": "keyword",
"normalizer": "raw",
},
"email": {
"type": "keyword",
"normalizer": "raw",
},
},
},
"scope": {
"type": "keyword",
"normalizer": "raw",
},
},
},
"flags": {
"type": "object",
"properties": {
"deprecated": { "type": "text", "index_options": "docs" },
"insecure": { "type": "integer" },
"unstable": { "type": "boolean" },
},
},
"evaluation": {
"type": "object",
"properties": {
"quality": {
"type": "object",
"properties": {
"carefulness": { "type": "double", "index": false },
"tests": { "type": "double", "index": false },
"health": { "type": "double", "index": false },
"branding": { "type": "double", "index": false },
},
},
"popularity": {
"type": "object",
"properties": {
"communityInterest": { "type": "double", "index": false },
"downloadsCount": { "type": "double", "index": false },
"downloadsAcceleration": { "type": "double", "index": false },
"dependentsCount": { "type": "double", "index": false },
},
},
"maintenance": {
"type": "object",
"properties": {
"releasesFrequency": { "type": "double", "index": false },
"commitsFrequency": { "type": "double", "index": false },
"openIssues": { "type": "double", "index": false },
"issuesDistribution": { "type": "double", "index": false },
},
},
},
},
"score": {
"type": "object",
"properties": {
"final": { "type": "double" },
"detail": {
"type": "object",
"properties": {
"quality": { "type": "double" },
"popularity": { "type": "double" },
"maintenance": { "type": "double" },
},
},
},
},
},
},
},
}
================================================
FILE: docs/architecture.md
================================================
# Architecture
The `npms-analyzer` runs two continuous and distinct processes. One is the `analysis` process where each package gets inspected and evaluated. The other one is the `continuous scoring` process where all packages get a score based on the aggregated evaluation results.
- [Analysis](#analysis)
- [Continuous scoring](#continuous-scoring)
## Analysis
The analysis process analyzes the `npm` packages, producing a result and a score.
By looking at the diagram above, you get an idea of how the analysis process works. Below you may find a more detailed description for the most complex components. The `grey` components are present in `lib`.
### Observers
Observers continuously push packages to the queue whenever they see fit.
- realtime - Observes the replicated `npm` registry for changes, pushing new or updated packages into the analyze queue.
- stale - Fetches packages that were not analyzed for some time, pushing them to the queue.
The packages reported by the `realtime` have priority over the other observers, so that recently published packages are analyzed first.
### Queue
The queue holds all packages that are waiting to be analyzed. This component gives us:
- Burst protection
- No loss of packages on crashes or failures
- Automatic retries
### Analyze
The analyze is a simple pipeline that produces an analysis result:
1. Fetches the package data
2. Downloads the source code
3. Runs the collectors
4. Runs the evaluators
5. Stores the result in CouchDB and Elasticsearch
Below you may find additional information on the collectors and evaluators.
#### Collectors
The collectors are responsible for gathering useful information about each package from a variety of sources:
- metadata
- source
- github
- npm
##### metadata
The metadata collector extracts basic data and attributes of a package.
- Extract package name, scope, description and keywords
- Extract package author, maintainers and contributors
- Extract the license
- Get releases timing information
- Extract repository and homepage
- Extract README
- Extract the package dependencies
- Check if the package is deprecated
- Check if the package has a test script
##### source
The source collector digs into the source code.
- Check certain files: `.npmignore`, `.gitignore`, `.gitattributes`, README size, tests size, etc
- Detect linters, such as `eslint`, `jshint`, `jslint` and `jscs`
- Detect badges in the README
- Compute code complexity *
- Grab the code coverage %
- Get repository file size
- Get dependencies insight, including if they are outdated
- Search for tech debts: TODOs, FIXMEs, etc *
- Get security insight with node security project
Items signaled with * are not yet done.
##### github
The github collector uses GitHub to collect useful data and statistics present there.
- Get number of stars, subscribers and forks
- Fetch the repository activity in terms of commits
- Fetch the number of issues and their distribution over time
- Extract the homepage
- Fetch contributors
- Check the build status
This collector is susceptible to the GitHub [rate limit](https://developer.github.com/v3/rate_limit/) policy. To fight against this limit, you may define several GitHub keys in the config to be used in a round-robin fashion.
##### npm
The npm collector uses the replicated CouchDB views and the npm [download-counts](https://github.com/npm/download-counts) API to extract useful information present there.
- Get number of stars
- Get number of downloads over time
- Get number of dependents
#### Evaluators
The evaluators take the information that was previously collected and evaluate different aspects of the package. These aspects are divide in four categories:
- quality
- popularity
- maintenance
- personalities
Evaluators may further divide each of these aspects into more granular ones, but their values are always scalars.
##### quality
Quality attributes are easy to calculate because they are self contained. These are the kind of attributes that a person looks first when looking at the package.
Below are some of the points taken into consideration:
- Has README? Has license? Has .gitignore and friends?
- Is the version stable (> 1.x.x)? Is it deprecated?
- Has tests? Whats their coverage %? Is build passing?
- Has outdated dependencies? Do they have vulnerabilities?
- Has custom website? Has badges?
- Does the project have linters configured?
- What's the code complexity score?
##### maintenance
Maintenance attributes allows us to understand if the package is active & healthy or if it is abandoned. These are typically the second kind of attributes that a person looks when looking at the package.
Below follows some of the points taken into consideration:
- Ratio of open issues vs the total issues
- The time it takes to close issues
- Most recent commit
- Commit frequency
##### popularity
Popularity attributes allows us to understand the package adoption and community size. These are the kind of attributes that a person looks when they are undecided on the package choice.
Below follows some of the points taken into consideration:
- Number of stars
- Number of forks
- Number of subscribers
- Number of contributors
- Number of dependents
- Number of downloads
- Downloads acceleration
##### personalities
If two packages are similar, one tend to choose the one in which the author is well known in the community. While this doesn't directly translate to quality, it's still a strong factor that we should account.
Relationships between people are also important. When an user follows another, there's a bound between them. We can infer that people prefer packages from the users they follow.
I will not elaborate on this because this evaluator will NOT be developed nor used in the initial release.
### Scoring
Calculates the package score based on the current aggregation if any. If there's no aggregation, the package won't be scored at the moment, but it will be later in the `continuous scoring` process.
## Continuous scoring
The continuous scoring process runs once in a while to score all `npm` packages, indexing the score data in `Elasticsearch` to be searchable.
By looking at the diagram above, you get an idea of how the continuous scoring process works. Below you may find a more detailed description for each component. The `grey` components are present in `lib`.
One important detail is that the continuous scoring process creates and maintains two [aliases](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-aliases.html):
- `npms-current`: The index with the full data from the last completed scoring process
- `npms-new`: The index that the current scoring process is writing to
### Prepare
The prepare step creates a new index and updates the `npms-new` alias to point to that index. It also removes extraneous indices from previous failed cycles (if any).
### Aggregate
The aggregation step iterates all the packages evaluations, calculating the `min`, `max` and `mean` values for each evaluation. The aggregation is stored in CouchDB to also be used by the `analysis` process.
### Score packages
After having the aggregation done, all packages are iterated again to produce a score based on the previously calculated aggregation.
The package evaluation and aggregation `mean` are normalized ([0, 1]), using the aggregation `min` and `max` values, and a Bezier Curve is computed using 4 control points: (0, 0), (normalizedAggregationMean, 0.75), (normalizedAggregationMean, 0.75), (1, 1). The package score is the Y value that corresponds, in this curve, to the package evaluation (X axis).
The score data for each package are stored in `Elasticsearch` into both `npms-current` and `npms-new` indices.
### Finalize
The finalize step updates the `npms-current` alias to point to the newly populated index and deletes the `npms-new` alias and previous index.
================================================
FILE: docs/deploys.md
================================================
# Deploys
We use `pm2` to deploy `npms-analyzer`, install it by running `$ npm install -g pm2`. You may find the pm2 configuration file in `ecosystem.json5`.
## Setting up
Before doing the first deploy, you need to setup the server. All commands executed in the server are expected to be run with `analyzer` user.
- Create the `analyzer` user on server
- Add `analyzer` user to the list of sudoers
- Install pm2 in the server
- Setup the deploy environment by running `$ pm2 deploy ecosystem.json5 production setup` in your local machine
- Create `~/npms-analyzer/local.json5` in the server with the custom configuration (databases, GitHub API tokens, etc)
- Do your first deploy by running `$ pm2 deploy ecosystem.json5 production` in your local machine
- Setup logrotate by running `$ sudo pm2 logrotate -u analyzer` on the server and then edit `/etc/logrotate.d/pm2-www` to change change `/root` to `/home/analyzer`, weekly to daily, and from 12 days to 14 days)
- Setup pm2 to run at start by running `$ sudo pm2 startup -u analyzer --hp "/home/analyzer"` on the server
- Finally run `$ pm2 save` to store the running processes
## Deploying
Deployment is easy, just run `$ pm2 deploy ecosystem.json5 production` in your local machine.
================================================
FILE: docs/diagrams/analysis.xml
================================================
<mxfile type="device" userAgent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36" version="5.3.4.1" editor="www.draw.io"><diagram>7Vxbc+K4Ev41qZrzkBRgLsljmElmHnZrZzcPe/bplMAC+0RYHtmEZH79tOxuX2UwQSY7E0MV2EKS5e5PfTcXzsfN82fFQu936XJxMRq4zxfOp4vRaDaZwKdueEkbxsObtGGtfDdtGuYND/53jo0DbN36Lo9KHWMpReyH5calDAK+jEttTCm5K3dbSVG+asjWdMW84WHJRL31b9+NvbT1ejTN279wf+3RlYdTvL8FWz6uldwGeL2LkbNKXunPG0ZzJTfq3AENlZQwjT7aPH/kQtORaJRS477h12yRige4kAMDHBzyxMQW7zRtiOIXuvWd58f8IWRLfb4D7l44cy/eCDgbwmEUK/nIP0ohFbQEMoBu85UvBDXBHd990m/dLoMYmQtLS88L/ab349E1LDbth0ugJXIVc4ST4UaTJrzLz1xueKxeoAsOmA7xvhB+zhRxtcuZOURiDbwiIwfYyBBA62zunK5wgKRtIPPo+qchc42mBso3ktlBYiGVafMWiDw2EpnGnUbk2bskskPnBSpPJiYq31igMsq7Eo2nAq4wd/0nOFzrQ2paqGoLTG/oV2jigkWxv2zdP+JMLb2m7hXuRx4L9eHyRfiBy4FR8wOASAQ31/euOQzjXbnDkzWsVBMy+aGEG2D+8Fa/6+C5T156tGKuDxioYg2wcs82vtDM/cLFEwdisDLYhk4dbGPYWGPAzJwJfx3oW4S5kxu0ALgZ7WPa1gbhOTMAzgbehuMaF7kLmhlPpYo9uZYBE3d5a4VrBXYCDdTLf7E9OflHn1xNqszlgXurrQY4XQi5fEyb7oGZONP/eRy/IEPYNpbQlK/lNykBRwaJUkRG+gsZE7pvWx43sjSSW5UAWRMfjSSm1px6oTzUBNzLdwW7MPafyqaPiYvJUCAU06OoQyj9II5qTM7mb8d3tBfbCZpC04fIh+3DQPKAgdhajAThRl88WMn/7B9zQMLAjtLtRQnCI/87WyQdNLKQOtB7Mr+YaD1i2rV6c8LGF7f4w8Z33QTagi24mGc2pUF8VBHUKE5aQgs3oMH8Qlsbb65kwprEyOXgajBDtpIXkJ4dh7fN81dNw8LE0/Ksl6T9aAq5WkWwCU5DJG6oAiDBydnCrYNTw2J2rKnRaxZtu6BdnDmFBqucDPCiZhlb0CxkyljSLM9+rBULHed65R3pHFIwJaWDLLasdGpCwBmi+U9YqvoU6aJwVA6TbrQXEqIgLD7JXSAkc3OSaQHqwlcvN46WG8OKxD+r4CBEF5gL9y1AE0qVAuddsdOTyv8OnZm+Jb1WC/wd3VTCNaTRC/wl3VFycckzOYm/dU0Pc7Beyb9WyVcDFgZmkqtoXcnX45uJqd+z8WQ2jsnlPwcb6/p07cfedtFz0gYnDRHErjjpOI1+fRSyoMTM6betToTMIVjHL2kht9BleKUH4a9ZJBCtqsxPT+drcNR7nLxCcBMozoCTGWr3Pu6Xj2zlg5FULvpgaNCcLe5n9tAqUWSCSZab3N+dhFSOn/T6r3XRiE5NAUaDHNLBvUvcaFoM4V6rSaIs/UENf245XCOXTNkPLaOKsHLIc2uEdyS2BslrT67LstjKIF0TW/vjmzZcx2scQjAkN/FQJvjaglS7oQ1rWarlgaZErv2CUs2QzqAdbE+stVZOdQ/xL86gGGWjozpyEXEFgKxnQT8IP4p5ADISClB0hiLgO10DksSRobVNfiMZB7NHvkwmcn0FkQfYgDCnkjqLsZTbpecuTktknFnQdGUfNQsaG/ZROeI4QSelKEhMaVErgqRe2nCCIPnFhAVJ+H+HsKDVFITFAwTLTJKimMYMOX/U+zsTDoPYY9o82nGlxwZgcsDCgOQvEfCUJAqEqAawdwVv3P79Rj8+HDlDr/UtdrrJYa7WtdT2U3lzJ0Uvm2cgeehdrYTcLT2m4isWAIgA7DL4X9GFFHylkdKS8+fmrwV2jq/L2QNDPNJUP5UVXp0UXB7YNQENaceSaZid0C+/gKzvPJV4g6lhinvMUNHTFKmyqaUSj5+oISf5mowjZaX6gEk+snF3wPFXrsBcT1TE/qxVOcGN0riU4aaY5hvHV6bkyZojJsf2L0dY6qOnZT86K+w9dpPMKsvIZG8Hm2RQd9zslq9aimXXV6B4KEBRxompR9VqubPXGPZueyd9pawNy2JK1XJkWaC8P0ehLMVLO8M2oE67IRGUOeiHZXogdQikCRUyvEXJtaEA4k4fs77AxVaBy5iU3J6aCEqj2S9wIVVdYPC3Ldx3nE7T50mPZCbtuj3MpFCA9WI0Q4VLCNFhbV37sa8DRz1HT+aoqdalO47Wi102DIxfHrBAFzj0/Dydn4aKl+742VnJSyjDLTzqksjtvuylM3FuKHvpDiwYmjxTeLCSOf65AoRHhXG6DhpWLfba8wdt4yGTm3J0ust4SP8MjJVnYA5iiyQLDWjCyGGYVR0He4+51C91aLUNiH4VEA8E5lpUYyVpq5qK/ANTrFl1xdEPcB78TwBsqqTPGp/gNKlgCb11Xg5aPOjHYUDr/NupCTXrf09QznmQr1l6MgejZKXsmg3tSQi1DqNbnWcHWvYgehMQOWRbnQdFdQfMDooelhBKCtY9itqj6LR//SE4ZPnXOozIbLOPInpwfk/hRsfG1psZ91n1z6nZ/4pldThBS7nYYoI2kSVnqPTKLl4s9YItr8u10jKu9+qPn7iLy9VXXT7sC6f5v6+lhmz+d3bO3Q8=</diagram></mxfile>
================================================
FILE: docs/diagrams/continuous-scoring.xml
================================================
<mxfile type="device" userAgent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36" version="5.3.4.1" editor="www.draw.io"><diagram>7VpZb+M2EP41fkxgHT7yaGfj3YcWWCAF2j4VtERLbCjRpehrf/0OqaF1+zqSAhsbMMTRkCLn++Yg5Z73nGy/SrKMfxch5T23H2573pee645GffjVgl0uGHgoiCQLc5FTCF7ZD4pCq7ZiIc0qikoIrtiyKgxEmtJAVWRESrGpqi0Erz51SSL7xELwGhDelP7JQhXn0rE7LOTfKIti+2Rn+JTfmZPgLZJileLzeq63MJ/8dkLsWGah3gvYUAoBw+irZPtMubajtVFujVnH3f0kJU1xIkc6uKO8y5rwFa40F2RqZ5e+iZmir0sS6PYG0O1501glHFoOXGZKijf6LLiQIElFCmrTBePcimDFL1/0V8tFqhBcmFreLukNZ747hsnmejiFllWhaE2losgwI8JVfqUioUruQAXvWrYh+zzb3hRY+k8oi8s49lFIkD/RfujCrHCBlm23MnKkYuMhhydMQ7aGy0hfWtFc1iUwfIteSUQ5yRQLTtbPKJFB3KVeQz+LyVJfBjvO0pACUNMjhDBkp3rtGmHoH4oNNiKYqTakuVHhDYDvTPS3SZ6Z+ejekoQMOFDnGnBlRhLGNbjfKF9TMAapks3xmmTzXf0FOeEsSvUSYWyzwBsQzn0aVxjn+E3G2ZhYJpwl4TV8c/wGijSEaIZNIVUsIpES/lJIa6iV4AQbyN1fKDeNv3XjcVAHl6bhREdaaM65CN5y0QzAxJH+pUrtEBCyUgJExVx+EwJ41BJRyszI79gArHVPxbgT0kyspCGyNj4mFiIjarUwHmoDHsRdghcqtq6mizYUTVcwFNG9rMJSsFRlDZD345+EO06/FGe+S6qXfG5I//RgnSOqDuyNmw68zw5lD7aOfo0H25RxIw/eMqUd2F4X/vsL+bZ15Ipze2jnG3t3sv2u/bmcDfDpSCa/Xlbks8JeBU/uEiasJUpxYhJFkkZEfUaKSyKFMxx8XKhw+pbsd4sVlSpg37B3/vdxYt+zE+p7u7/3hHHGEmSE2y47RB6oGu5//kAdceSSKGF33nfbtqTLBEIZ8FZIvfn+3I7cM0SNvI/bjtjNdheTSvgO/1vps48pLEs9oCEmoMHpAght71p2vAZCsjQ6kTp6zDOPLlBkETGz8Kba7AApn6A4YWFoomobEwVoL7ihXAx6FDqcSslLmVQw99anKLXo00Iip4+bzzKL8BTqShbhwIeOqu5SJL9Tguubz1UJrj1J56tozXTHq2YPE0y5bDYsuHmSHLnVEOXbsHF2kvSr5ZgzwvbFxXb7cwaW/rWivmtedX1nnM+rcIB8BhdnbA8PAH6BXeMNnOVSh8AYWN1HGi+5wiVOj4LNw2SdBaF86iciXHF4R/KLbuEaaawFywPHtbUXBO+6hbNBtgTqjIEHaGN8onk+mr7du3wImjb9HCpUmmGrXJbiHifZgtGX8aMuH4OYSPVI0lQoiCgi/adsO6xLTy0r3wfhPaMkThnED+Bmt/FXH13G5tJhE+FBC8D7F4HXIdz2Rq9jo2pBnC+T9JGblHYLfz2UADvevd4L3xoLb+G/9WMOu4Eooevi8XwFXSyArgJ30DweNbDpp8PDYXT9NwZI7DDsFcchRzenkmbsB5kbBQ00lqOgPZj2BhpQXTGBTr67O7pLVYZ6GWQO2DL/YUqrB78D5k4+dBdS1wHer52g2uBcDtg2qJcRv8CdoVn8wSIvs4t/rHgvPwE=</diagram></mxfile>
================================================
FILE: docs/setup.md
================================================
# Setup
Below you will find a list of items that you must do to get the project working on your local machine. The production setup document is not present in the repository for security reasons.
## Config file
This project uses [config](https://www.npmjs.com/package/config) for configuration. You may create `config/local.json5` file to override the configuration as necessary, especially to define `githubTokens`.
## Programs & utilities
- `node` must be installed and available in `$PATH` (`>= v8`)
- `git` must be installed and available in the `$PATH`.
- GNU coreutils (`rm`, `mkdir`, `chmod`, `wc`) must be available.
- `tar` or `bsdtar` must be available (BSD version is preferred.. on Debian install with `$ aptitude install bsdtar`)
- Install the `pino` CLI to prettify logging output by running `$ npm install -g pino-pretty`
## CouchDB
- Install [CouchDB](http://couchdb.apache.org/) and run it (tested with `v2.2`).
- Create database named `npms` by executing `curl -X PUT http://admin:admin@localhost:5984/npms`
- Setup npm replication from `https://replicate.npmjs.com/registry` to `npm` database in `continuous` mode.
- Setup the necessary views by creating the document `_design/npms-analyzer-npm` in the `npm` database with the contents of `https://github.com/npms-io/npms-analyzer/blob/master/config/couchdb/npm-analyzer.json`
- Setup the necessary views by creating the document `_design/npms-analyzer-npms` in the `npms` database with the contents of `https://github.com/npms-io/npms-analyzer/blob/master/config/couchdb/npms-analyzer.json`
Note: for the replication to work, you might need to [tweak](https://github.com/apache/couchdb/issues/1550#issuecomment-411751809) `auth-plugins` in the CouchDB config:
```
[replicator]
auth_plugins = couch_replicator_auth_noop
```
## RabbitMQ
**NOTE**: You may put `RabbitMQ standalone` into the gitignored `dev` folder while developing!
- Install [RabbitMQ](https://www.rabbitmq.com/download.html) and run it (tested with `v3.6.1`).
- Install the [management](https://www.rabbitmq.com/management.html) plugin which is very useful by running `rabbitmq-plugins enable rabbitmq_management`
- Head to `http://localhost:15672` and login with `guest/guest` and see if everything is ok.
## Elasticsearch
**NOTE**: You may put the `Elasticsearch` app into the gitignored `dev` folder while developing!
- Install [Elasticsearch](https://www.elastic.co/downloads/elasticsearch) (tested with `v6.4`)
- Install the [head](https://github.com/mobz/elasticsearch-head) to perform various manual operations in a web GUI
- Add these configurations to the `elasticsearch.yml`:
- `action.auto_create_index: -npms-current,-npms-new,+*`
## Crontab
If you plan to run this in production, you should add `$ npms-analyzer tasks enqueue-missing` and `$ npms-analyzer tasks clean-extraneous` to crontab. These tasks ensure that, in case of errors, the `npms` packages are in sync with the packages from the `npm` registry.
================================================
FILE: ecosystem.json5
================================================
// This is the pm2 configuration file for npms-analyzer
{
apps: [
{
name: 'npms-analyzer-observe',
script: './cli.js',
args: 'observe',
instances: 1,
env : { NODE_ENV: 'production' },
max_memory_restart: '2018M', // Restart if it's getting close to the limit
node_args: '--max_old_space_size=2048',
},
{
name: 'npms-analyzer-consume',
script: './cli.js',
args: 'consume --concurrency 5',
instances: 2,
env : { NODE_ENV: 'production' },
max_memory_restart: '2018M', // Restart if it's getting close to the limit
node_args: '--max_old_space_size=2048',
},
{
name: 'npms-analyzer-scoring',
script: './cli.js',
args: 'scoring',
instances: 1,
env : { NODE_ENV: 'production' },
max_memory_restart: '4066M', // Restart if it's getting close to the limit
node_args: '--max_old_space_size=4096',
},
],
deploy: {
production: {
user: 'analyzer',
host: '212.47.252.55',
ref: 'origin/master',
repo: 'https://github.com/npms-io/npms-analyzer.git',
path: '/home/analyzer/npms-analyzer',
'post-deploy': '\
cp ../local.json5 ./config/ && \
npm update --loglevel http --production && \
pm2 startOrRestart ecosystem.json5 --env production',
},
},
}
================================================
FILE: lib/analyze/collect/bin/david-json
================================================
#!/usr/bin/env node
'use strict';
global.Promise = require('bluebird');
// Some packages have a .npmrc to integrate with CI systems which require NPM_TOKEN env var to be defined
// See: https://github.com/alanshaw/david/issues/109
process.env.NPM_TOKEN = '';
process.env.AUTH_TOKEN = '';
process.env.NPM_AUTH_TOKEN = '';
process.env.SOCIALTABLES_NPM_TOKEN = ''; // https://github.com/socialtables/react-table-sorter/blob/master/.npmrc
const path = require('path');
const yargs = require('yargs');
const loadJsonFile = require('load-json-file');
const david = Promise.promisifyAll(require('david'));
const argv = yargs
.strict()
.wrap(Math.min(120, yargs.terminalWidth()))
.help()
.alias('help', 'h')
.usage('Usage: $0 --registry [registry-url]')
.option('registry', {
type: 'string',
alias: 'r',
describe: 'The registry URL',
default: 'https://registry.npmjs.org',
})
.argv;
// ---------------------------------------
const packageJsonPath = path.join(process.cwd(), 'package.json');
loadJsonFile(packageJsonPath)
.then((packageJson) => (
david.getUpdatedDependenciesAsync(packageJson, {
npm: {
'fetch-retries': 0, // No need for retries, the registry is local
registry: argv.registry,
},
loose: true, // Enable loose semver, there's some really strange versions that got into the registry somehow
stable: true,
})
))
.then((deps) => {
process.stdout.write(JSON.stringify(deps, null, 2));
process.stdout.write('\n');
})
.done();
================================================
FILE: lib/analyze/collect/github.js
================================================
'use strict';
const got = require('got');
const moment = require('moment');
const ghIssuesStats = require('gh-issues-stats');
const tokenDealer = require('token-dealer');
const deepCompact = require('deep-compact');
const promiseRetry = require('promise-retry');
const uniqBy = require('lodash/uniqBy');
const pick = require('lodash/pick');
const promisePropsSettled = require('./util/promisePropsSettled');
const pointsToRanges = require('./util/pointsToRanges');
const hostedGitInfo = require('../util/hostedGitInfo');
const gotRetry = require('../util/gotRetry');
const unavailableStatusCodes = [404, 400, 403, 451]; // 404 - not found; 400 - invalid repo name; 403/451 - dmca takedown
const log = logger.child({ module: 'collect/github' });
/**
* Extract commits frequency based on the /stats/commit_activity response.
*
* @param {Object} commitActivity - The commit activity response.
*
* @returns {Array} The commits.
*/
function extractCommits(commitActivity) {
// Aggregate the commit activity into ranges
const points = commitActivity.map((entry) => ({ date: moment.unix(entry.week).utc(), count: entry.total }));
const ranges = pointsToRanges(points, pointsToRanges.bucketsFromBreakpoints([7, 30, 90, 180, 365]));
// Finally map to a prettier array based on the ranges
return ranges.map((range) => ({
from: range.from,
to: range.to,
count: range.points.reduce((sum, point) => sum + point.count, 0),
}));
}
/**
* Utility function to do a request to the GitHub API.
*
* @param {String} resource - The resource path.
* @param {Object} options - The options inferred from github() options.
*
* @returns {Promise} The promise for GitHub response.
*/
function githubRequest(resource, options) {
const url = `https://api.github.com${resource}`;
return promiseRetry((retry) => (
// Use token dealer to circumvent rate limit issues
tokenDealer(options.tokens, (token, exhaust) => {
const handleRateLimit = (response, err) => {
if (response.headers['x-ratelimit-remaining'] === '0') {
const isRateLimitError = err && err.statusCode === 403 && /rate limit/i.test(response.body.message);
exhaust(Number(response.headers['x-ratelimit-reset']) * 1000, isRateLimitError);
}
};
return got(url, {
json: true,
timeout: 30000,
headers: Object.assign({ accept: 'application/vnd.github.v3+json' }, token ? { authorization: `token ${token}` } : null),
retry: gotRetry,
})
.then((response) => {
handleRateLimit(response);
return response;
}, (err) => {
err.response && handleRateLimit(err.response, err);
throw err;
});
}, {
group: 'github',
wait: options.waitRateLimit,
onExhausted: (token, reset) => log.error(`Token ${token ? token.substr(0, 10) : '<empty>'}.. exhausted`, { reset }),
})
.then((response) => {
// If response is 202, it means that there's no cached result so we must wait a bit and try again
if (response.statusCode === 202) {
log.debug(`Got 202 response for ${url} (not cached), retrying..`);
retry(Object.assign(new Error(`Empty response for ${url}`), { code: 'NO_CACHED_RESPONSE' }));
}
// If response is 204, it means that there's no content.. e.g.: there's no commits yet
if (response.statusCode === 204) {
return null;
}
return response.body;
})
), { minTimeout: 2500, retries: 5 })
// If after all the retries there's still no content, return an empty array
.catch({ code: 'NO_CACHED_RESPONSE' }, (err) => {
log.warn({ err }, err.message);
return [];
})
// Check if the repository is unavailable
.catch((err) => unavailableStatusCodes.indexOf(err.statusCode) !== -1, (err) => {
log.info({ err }, `GitHub request to ${url} failed with ${err.statusCode}`);
return null;
})
.catch((err) => {
/* istanbul ignore next */
log.error({ err }, `GitHub request to ${url} failed`);
/* istanbul ignore next */
throw err;
});
}
/**
* Fetches statistical information for a repository.
*
* @param {String} repository - The {user}/{project}.
* @param {Object} options - The options inferred from github() options.
*
* @returns {Promise} The promise for the stats.
*/
function fetchIssuesStats(repository, options) {
return ghIssuesStats(repository, {
tokens: options.tokens,
concurrency: 5,
got: { retry: gotRetry },
tokenDealer: {
wait: options.waitRateLimit,
lru: tokenDealer.defaultLru,
onExhausted: (token, reset) => log.error(`Token ${token ? token.substr(0, 10) : '<empty>'}.. exhausted`, { reset }),
},
})
// Sum up the issues with the pull requests
.then((stats) => ({
count: stats.issues.count + stats.pullRequests.count,
openCount: stats.issues.openCount + stats.pullRequests.openCount,
distribution: Object.keys(stats.issues.distribution).reduce((accumulated, range) => {
accumulated[range] = stats.issues.distribution[range] + stats.pullRequests.distribution[range];
return accumulated;
}, {}),
}))
// Check if the repository is unavailable
.catch((err) => unavailableStatusCodes.indexOf(err.statusCode) !== -1, (err) => {
log.warn({ err }, `Fetch of issues stats for ${repository} failed with ${err.statusCode}`);
return null;
})
.catch((err) => {
/* istanbul ignore next */
log.error({ err }, `Fetch of issues stats for ${repository} failed`);
/* istanbul ignore next */
throw err;
});
}
// ----------------------------------------------------------------------------
/**
* Runs the github analyzer.
* If the repository is not hosted in GitHub, the promise resolves to `null`.
*
* @param {Object} packageJson - The latest package.json data (normalized).
* @param {Object} downloaded - The downloaded info (`dir`, `packageJson`, ...).
* @param {Object} [options] - The options; read below to get to know each available option.
*
* @returns {Promise} The promise that fulfills when done.
*/
function github(packageJson, downloaded, options) {
let repository = packageJson.repository;
if (!repository) {
log.debug(`No repository field present for ${packageJson.name}, ignoring..`);
return Promise.resolve(null);
}
const gitInfo = hostedGitInfo(repository.url);
if (!gitInfo || gitInfo.type !== 'github') {
log.debug({ repository }, `Repository for ${packageJson.name} is not hosted on GitHub, ignoring..`);
return Promise.resolve(null);
}
options = Object.assign({
tokens: null, // The GitHub API tokens to use
waitRateLimit: false, // True to wait if rate limit for all tokens were exceeded,
}, options);
repository = `${gitInfo.user}/${gitInfo.project}`;
return promisePropsSettled({
info: githubRequest(`/repos/${repository}`, options),
contributors: githubRequest(`/repos/${repository}/stats/contributors`, options),
commitActivity: githubRequest(`/repos/${repository}/stats/commit_activity`, options),
issuesStats: fetchIssuesStats(repository, options),
statuses: githubRequest(`/repos/${repository}/commits/${downloaded.gitRef || 'master'}/statuses`, options),
})
.then((props) => {
if (!props.info) {
log.info(`The GitHub repository ${repository} is unavailable`);
return null;
}
if (!props.contributors || !props.commitActivity || !props.issuesStats) {
log.info(`It seems that the GitHub repository ${repository} is empty`);
return null;
}
return deepCompact({
homepage: props.info.homepage,
forkOf: (props.info.fork && props.info.parent && props.info.parent.full_name) || null,
starsCount: props.info.stargazers_count,
forksCount: props.info.forks_count,
subscribersCount: props.info.subscribers_count,
issues: Object.assign(props.issuesStats, { isDisabled: !props.info.has_issues }),
// Contributors (top 100)
contributors: props.contributors
.map((contributor) => {
const author = contributor.author;
// Empty entries will be stripped by deepCompact
return author && { username: contributor.author.login, commitsCount: contributor.total };
})
.reverse(),
// Commit activity
commits: extractCommits(props.commitActivity),
// Statuses
statuses: uniqBy(props.statuses, (status) => status.context)
.map((status) => pick(status, 'context', 'state')),
});
})
.tap(() => log.debug(`The github collector for ${packageJson.name} completed successfully`));
}
module.exports = github;
================================================
FILE: lib/analyze/collect/index.js
================================================
'use strict';
const pickBy = require('lodash/pickBy');
const intersectionWith = require('lodash/intersectionWith');
const isEmpty = require('lodash/isEmpty');
const collectors = require('require-directory')(module, './', { recurse: false });
const promisePropsSettled = require('./util/promisePropsSettled');
const log = logger.child({ module: 'collect' });
/**
* Checks if a package matches the downloaded repository.
*
* Unfortunately many people try to trick the system by pointing their repositories to popular repositories,
* such as `jQuery`.
*
* @param {String} data - The package data.
* @param {Object} packageJson - The latest package.json data (normalized).
* @param {Object} downloaded - The downloaded info (`dir`, `packageJson`, ...).
* @param {Nano} npmNano - The npm nano client instance.
*
* @returns {Promise} A promise that resolves to true if publisher is the owner, false if in doubt.
*/
function checkRepositoryOwnership(data, packageJson, downloaded, npmNano) {
// If name is equal, then the publisher is the owner.. no further checks required
if (packageJson.name === downloaded.packageJson.name) {
return Promise.resolve(true);
}
const repositoryUrl = packageJson.repository && packageJson.repository.url;
const downloadedRepositoryUrl = downloaded.packageJson.repository && downloaded.packageJson.repository.url;
// Check if both have no repository
if (!repositoryUrl && !downloadedRepositoryUrl) {
return Promise.resolve(true);
}
// Check if download actually failed (e.g.: does not exist)
if (isEmpty(downloaded.packageJson)) {
return Promise.resolve(false);
}
// Do a final check against the maintainers of the downloaded package
return npmNano.getAsync(downloaded.packageJson.name)
.then((downloadedData) => (
intersectionWith(data.maintainers, downloadedData.maintainers, (maintainer, downloadedMaintainer) =>
maintainer.name === downloadedMaintainer.name || maintainer.email === downloadedMaintainer.email).length > 0)
)
.tap((isMaintainer) => {
!isMaintainer && log.warn({ packageJson, downloaded },
`Publisher of package ${packageJson.name} does not own the repository`);
})
.catch({ error: 'not_found' }, () => false);
}
// ----------------------------------------------------------------------------
/**
* Generates an empty collected data.
*
* @param {name} name - The package name.
*
* @returns {Object} The empty collected data.
*/
function empty(name) {
return {
metadata: collectors.metadata.empty(name),
};
}
/**
* Runs all the collectors.
*
* @param {String} data - The package data.
* @param {Object} packageJson - The latest package.json data (normalized).
* @param {Object} downloaded - The downloaded info (`dir`, `packageJson`).
* @param {Nano} npmNano - The npm nano client instance.
* @param {Object} [options] - The options; read below to get to know each available option.
*
* @returns {Promise} The promise that fulfills when done.
*/
function collect(data, packageJson, downloaded, npmNano, options) {
options = Object.assign({
githubTokens: null, // The GitHub API tokens to use
waitRateLimit: false, // True to wait if rate limit for all tokens were exceeded
}, options);
return checkRepositoryOwnership(data, packageJson, downloaded, npmNano)
.then((isRepositoryOwner) => {
const isSourceOwner = downloaded.downloader === 'npm' || isRepositoryOwner;
return promisePropsSettled({
metadata: collectors.metadata(data, packageJson),
npm: collectors.npm(data, packageJson, npmNano),
github: isRepositoryOwner && collectors.github(packageJson, downloaded, {
tokens: options.githubTokens,
waitRateLimit: options.waitRateLimit,
}),
source: isSourceOwner && collectors.source(data, packageJson, downloaded, {
npmRegistry: `${npmNano.config.url}/${npmNano.config.db}`,
}),
})
.then((collected) => pickBy(collected));
});
}
module.exports = collect;
module.exports.empty = empty;
module.exports.collectors = collectors;
================================================
FILE: lib/analyze/collect/metadata.js
================================================
'use strict';
const moment = require('moment');
const spdx = require('spdx');
const spdxCorrect = require('spdx-correct');
const deepCompact = require('deep-compact');
const isLinkWorking = require('is-link-working');
const get = require('lodash/get');
const find = require('lodash/find');
const pickBy = require('lodash/pickBy');
const mapValues = require('lodash/mapValues');
const size = require('lodash/size');
const hostedGitInfo = require('../util/hostedGitInfo');
const pointsToRanges = require('./util/pointsToRanges');
const promisePropsSettled = require('./util/promisePropsSettled');
const log = logger.child({ module: 'collect/metadata' });
/**
* Extracts the releases frequency.
*
* @param {Object} data - The package data.
*
* @returns {Array} An array of ranges with the release count for each entry.
*/
function extractReleases(data) {
// Aggregate the releases into ranges
const time = data.time || {};
const points = Object.keys(time).map((version) => ({ date: moment.utc(time[version]), version }));
const ranges = pointsToRanges(points, pointsToRanges.bucketsFromBreakpoints([30, 90, 180, 365, 730]));
// Build the releases frequency array based on the releases ranges
return ranges.map((range) => ({
from: range.from,
to: range.to,
count: range.points.length,
}));
}
/**
* Normalizes a single license value to a SPDX identifier.
*
* @param {String} name - The package name.
* @param {String|Object} license - The license value, which can be a string or an object (deprecated).
*
* @returns {String} The normalized license, which is a SPDX identifier.
*/
function normalizeLicense(name, license) {
// Handle { type: 'MIT', url: 'http://..' }
if (license && license.type) {
license = license.type;
}
// Ensure that the license is a non-empty string
// Note that `spdx-correct` throws on strings with empty chars, so we must use trim()
// e.g.: webjs-cli
if (typeof license !== 'string' || !license.trim()) {
log.warn({ license }, `Invalid license for package ${name} was found`);
return null;
}
// Try to correct licenses that are not valid SPDX identifiers
if (!spdx.valid(license)) {
const correctedLicense = spdxCorrect(license);
if (correctedLicense) {
log.debug(`Package ${name} license was corrected from ${license} to ${correctedLicense}`);
license = correctedLicense;
} else {
log.warn({ license }, `License for package ${name} is not a valid SPDX indentifier`);
license = null;
}
}
return license;
}
/**
* Extracts the license from the package data.
* Attempts to normalize any license to valid SPDX identifiers.
*
* @param {Object} packageJson - The latest package.json object (normalized).
*
* @returns {String} The license or null if unable to extract it.
*/
function extractLicense(packageJson) {
const originalLicense = packageJson.license || packageJson.licenses;
let license = originalLicense;
// Short-circuit for packages without a license
if (license == null) {
log.trace(`No license for package ${packageJson.name} is set`);
return null;
}
// Some old packages used objects or an array of objects to specify licenses
// We do some effort to normalize them into SPDX license expressions
if (Array.isArray(license)) {
license = license
.map((license) => normalizeLicense(packageJson.name, license))
.reduce((str, license) => str + (str ? ' OR ' : '') + license, '');
} else {
license = normalizeLicense(packageJson.name, license);
}
return license;
}
/**
* Extracts useful links of the package (homepage, repository, etc).
*
* @param {Object} packageJson - The latest package.json object (normalized).
*
* @returns {Object} The links.
*/
function extractLinks(packageJson) {
const gitInfo = hostedGitInfo(packageJson.repository && packageJson.repository.url);
const links = pickBy({
npm: `https://www.npmjs.com/package/${encodeURIComponent(packageJson.name)}`,
homepage: packageJson.homepage,
repository: gitInfo && gitInfo.browse(),
bugs: (packageJson.bugs && packageJson.bugs.url) || (gitInfo && gitInfo.bugs()),
});
// Filter only good links, removing broken ones
// Avoid checking the npm link because we are sure it works..
const linksBeingChecked = [];
const isLinkWorkingCache = { [links.npm]: true };
const areLinksWorking = mapValues(links, (link) => {
const normalizedLink = link.split('#')[0]; // Remove trailing # (e.g.: #readme)
if (!isLinkWorkingCache[normalizedLink]) {
isLinkWorkingCache[normalizedLink] = isLinkWorking(normalizedLink);
linksBeingChecked.push(normalizedLink);
}
return isLinkWorkingCache[normalizedLink];
});
log.debug({ linksBeingChecked }, 'Checking for broken links');
return Promise.props(areLinksWorking)
.then((result) => {
let finalLinks = mapValues(links, (link, name) => result[name] ? link : null);
// If the homepage is broken, fallback to the repository docs
if (!finalLinks.homepage && finalLinks.repository) {
finalLinks.homepage = gitInfo.docs();
}
finalLinks = pickBy(finalLinks);
// Log the broken links
const brokenLinks = pickBy(links, (link, name) => !result[name]);
const brokenLinksCount = size(brokenLinks);
brokenLinksCount && log.info({ links, brokenLinks, finalLinks },
`Detected ${brokenLinksCount} broken links on ${packageJson.name}`);
return finalLinks;
});
}
/**
* Extracts the author of the the package.
* Tries to match against the maintainers to guess its `npm` username.
*
* @param {Object} packageJson - The latest package.json object (normalized).
* @param {Array} maintainers - The package maintainers.
*
* @returns {Object} The author (name + [username] + [email] + [url]) or null if unable to extract it.
*/
function extractAuthor(packageJson, maintainers) {
if (!packageJson.author) {
return null;
}
const author = Object.assign({}, packageJson.author);
const maintainer = maintainers && find(maintainers, (maintainer) => maintainer.email === packageJson.author.email);
if (maintainer) {
author.username = maintainer.name;
}
return author;
}
/**
* Extracts the person who published the package.
* For older packages, it might be unavailable so a best-effort to guess it is made.
*
* @param {Object} packageJson - The latest package.json object (normalized).
* @param {Array} maintainers - The package maintainers.
*
* @returns {Object} The publisher (username + email) or null if unable to extract it.
*/
function extractPublisher(packageJson, maintainers) {
let npmUser;
// Assume the _npmUser if exists
npmUser = packageJson._npmUser;
// Fallback to find the author within the maintainers
// If it doesn't exist, fallback to the first maintainer
if (!npmUser && maintainers) {
npmUser = packageJson.author && find(maintainers, (maintainer) => maintainer.email === packageJson.author.email);
npmUser = npmUser || maintainers[0];
}
return npmUser ? { username: npmUser.name, email: npmUser.email } : null;
}
/**
* Extract the scope from the package name.
*
* @param {Object} packageJson - The latest package.json data (normalized).
*
* @returns {String} The scope, or "unscoped" if no scope is found.
*/
function extractScope(packageJson) {
const match = packageJson.name.match(/^@([^/]+)\/.+$/);
return match ? match[1] : 'unscoped';
}
/**
* Extracts the package maintainers.
*
* This solves various issues with data consistency:
* - Some packages have the maintainers in the data, others in the package.json.
* - The top-level maintainers were empty but the package.json ones were correct, e.g.: `graphql-shorthand-parser`.
* - The maintainers was not an array but a string e.g.: `connect-composer-stats.`.
*
* @param {Object} data - The package data.
* @param {Object} packageJson - The latest package.json data (normalized).
*
* @returns {Array} The maintainers or null if unable to extract them.
*/
function extractMaintainers(data, packageJson) {
if (Array.isArray(data.maintainers) && data.maintainers.length) {
return data.maintainers;
}
if (Array.isArray(packageJson.maintainers) && packageJson.maintainers.length) {
return packageJson.maintainers;
}
log.warn({ packageJsonMaintainers: packageJson.maintainers, dataMaintainers: data.maintainers },
`Failed to extract maintainers of ${packageJson.name}`);
return null;
}
// ----------------------------------------------------------------------------
/**
* Generates an empty metadata objet.
*
* @param {name} name - The package name.
*
* @returns {Object} The empty collected data.
*/
function empty(name) {
return {
name,
version: '0.0.0',
links: {
npm: `https://www.npmjs.com/package/${encodeURIComponent(name)}`,
},
};
}
/**
* Runs the metadata analyzer.
*
* @param {Object} data - The package data.
* @param {Object} packageJson - The latest package.json data (normalized).
*
* @returns {Promise} The promise that fulfills when done.
*/
function metadata(data, packageJson) {
return promisePropsSettled({
links: extractLinks(packageJson),
})
.then((props) => {
const maintainers = extractMaintainers(data, packageJson);
return deepCompact({
name: packageJson.name,
scope: extractScope(packageJson),
version: packageJson.version,
description: packageJson.description,
keywords: packageJson.keywords,
date: data.time && (data.time[packageJson.version] || data.time.modified),
author: extractAuthor(packageJson, maintainers),
publisher: extractPublisher(packageJson, maintainers),
maintainers: maintainers && maintainers.map((maintainer) =>
({ username: maintainer.name, email: maintainer.email })
),
contributors: packageJson.contributors,
repository: packageJson.repository,
links: props.links,
license: extractLicense(packageJson),
dependencies: packageJson.dependencies,
devDependencies: packageJson.devDependencies,
peerDependencies: packageJson.peerDependencies,
bundledDependencies: packageJson.bundledDependencies || packageJson.bundleDependencies,
optionalDependencies: packageJson.optionalDependencies,
releases: extractReleases(data),
// There's some packages such as oh-flex that have an invalid deprecated field
// that was manually written in the package.json...
deprecated: typeof packageJson.deprecated === 'string' ? packageJson.deprecated : null,
hasTestScript: get(packageJson, 'scripts.test', 'no test specified').indexOf('no test specified') === -1 ? true : null,
hasSelectiveFiles: Array.isArray(packageJson.files) && packageJson.files.length > 0 ? true : null,
// Need to use typeof because there's some old packages in which the README is an object, e.g.: `flatsite`
readme: (typeof data.readme === 'string' && data.readme) ? data.readme : null,
});
})
.tap(() => log.debug(`The metadata collector for ${packageJson.name} completed successfully`));
}
module.exports = metadata;
module.exports.empty = empty;
================================================
FILE: lib/analyze/collect/npm.js
================================================
'use strict';
const got = require('got');
const moment = require('moment');
const size = require('lodash/size');
const pointsToRanges = require('./util/pointsToRanges');
const promisePropsSettled = require('./util/promisePropsSettled');
const gotRetry = require('../util/gotRetry');
const log = logger.child({ module: 'collect/npm' });
/**
* Fetches the download count from https://api.npmsjs.org/downloads.
*
* @see https://github.com/npm/download-counts
*
* @param {String} name - The package name.
*
* @returns {Promise} The promise for the downloads object.
*/
function fetchDownloads(name) {
const requestRange = {
from:
moment.utc()
.subtract(1, 'd')
.startOf('day')
.subtract(365, 'd')
.format('YYYY-MM-DD'),
to:
moment.utc()
.subtract(1, 'd')
.startOf('day')
.format('YYYY-MM-DD'),
};
const url = `https://api.npmjs.org/downloads/range/${requestRange.from}:${requestRange.to}/${encodeURIComponent(name)}`;
return got(url, {
json: true,
timeout: 30000,
retry: gotRetry,
})
.then((res) => res.body.downloads)
// Check if there is no stats yet
.catch({ statusCode: 404 }, () => [])
.then((downloads) => {
// Aggregate the data into ranges
const points = downloads.map((entry) => ({ date: moment.utc(entry.day), count: entry.downloads }));
const ranges = pointsToRanges(points, pointsToRanges.bucketsFromBreakpoints([1, 7, 30, 90, 180, 365]));
// Finally map to a prettier array based on the ranges, calculating the mean and count for each range
return ranges.map((range) => {
const downloadsCount = range.points.reduce((sum, point) => sum + point.count, 0);
return {
from: range.from,
to: range.to,
count: downloadsCount,
};
});
})
.catch((err) => {
log.error({ err, url }, `Failed to fetch ${name} downloads`);
throw err;
});
}
/**
* Fetches the dependents count.
*
* @param {String} name - The package name.
* @param {Nano} npmNano - The client nano instance for npm.
*
* @returns {Promise} The promise for the dependents count.
*/
// function fetchDependentsCount(name, npmNano) {
// return npmNano.viewAsync('app', 'dependedUpon', {
// startkey: [name],
// endkey: [name, '\ufff0'],
// limit: 1,
// reduce: true,
// stale: 'update_after',
// })
// .then((response) => !response.rows.length ? 0 : response.rows[0].value)
// .catch((err) => {
// /* istanbul ignore next */
// log.error({ err }, `Failed to fetch ${name} dependents count from CouchDB`);
// /* istanbul ignore next */
// throw err;
// });
// }
/**
* Extract the stars count.
*
* @param {Object} data - The package data.
*
* @returns {Number} The number of stars.
*/
function extractStarsCount(data) {
// The users that starred are stored in the package data itself under the `users` property
return size(data.users);
}
// ----------------------------------------------------------------------------
/**
* Runs the npm analyzer.
*
* @param {Object} data - The package data.
* @param {Object} packageJson - The latest package.json data (normalized).
*
* @returns {Promise} The promise that fulfills when done.
*/
function npm(data, packageJson) {
return promisePropsSettled({
downloads: fetchDownloads(packageJson.name, { timeout: 30000 }),
// TODO: The couchdb view was deprecated, find an alternative
// dependentsCount: fetchDependentsCount(packageJson.name, npmNano),
starsCount: extractStarsCount(data),
})
.tap(() => log.debug(`The npm collector for ${packageJson.name} completed successfully`));
}
module.exports = npm;
================================================
FILE: lib/analyze/collect/source.js
================================================
'use strict';
const path = require('path');
const detectRepoLinters = require('detect-repo-linters');
const detectRepoTestFiles = require('detect-repo-test-files');
const detectReadmeBadges = require('detect-readme-badges');
const detectRepoChangelog = require('detect-repo-changelog');
const fetchCoverage = require('fetch-coverage');
const loadJsonFile = require('load-json-file');
const deepCompact = require('deep-compact');
const isRegularFile = require('is-regular-file');
const got = require('got');
const fileSize = require('./util/fileSize');
const promisePropsSettled = require('./util/promisePropsSettled');
const exec = require('../util/exec');
const gotRetry = require('../util/gotRetry');
const fileContents = require('./util/fileContents');
const uniqWith = require('lodash/uniqWith');
const isEqual = require('lodash/isEqual');
const { uniq } = require('lodash');
const davidBin = path.normalize(`${__dirname}/bin/david-json`);
const log = logger.child({ module: 'collect/source' });
/**
* Inspects important files, such as the tests and README file sizes.
*
* @param {Object} data - The package data.
* @param {Object} downloaded - The downloaded info (`dir`, `packageDir`, ...).
*
* @returns {Promise} The promise for the inspection result.
*/
function inspectFiles(data, downloaded) {
// Sum readme file sizes of the one in the package dir with the root
const readmeSize = fileSize([
`${downloaded.packageDir}/${data.readmeFilename || 'README.md'}`,
downloaded.dir !== downloaded.packageDir ? `${downloaded.DIR}/${data.readmeFilename || 'README.md'}` : '',
].filter((path) => path));
// Prefer tests located in the package dir and fallback to the root
const testsSize = (
detectRepoTestFiles(downloaded.packageDir)
.then((files) => !files.length && downloaded.dir !== downloaded.packageDir ? detectRepoTestFiles(downloaded.dir) : files)
.then((files) => fileSize(files))
);
// .npmignore must be located inside the package dir
// TODO: Improve npmignore detection because it can be in sub-directories too
const hasNpmIgnore = isRegularFile(`${downloaded.packageDir}/.npmignore`).then((is) => is || null);
// npm-shrinkwrap must be located inside the package dir
const hasShrinkwrap = isRegularFile(`${downloaded.packageDir}/npm-shrinkwrap.json`).then((is) => is || null);
// Usually changelogs are at the root directory, still we prefer the package dir one if it exists
const hasChangelog = detectRepoChangelog(downloaded.packageDir)
.then((file) => !file && downloaded.dir !== downloaded.packageDir ? detectRepoChangelog(downloaded.dir) : file)
.then((file) => file ? true : null);
return Promise.props({
readmeSize,
testsSize,
hasNpmIgnore,
hasShrinkwrap,
hasChangelog,
});
}
/**
* Gets the readme badges.
* @param {Object} data - The package data.
* @param {Object} downloaded - The downloaded info (`dir`, `packageDir`, ...).
*
* @returns {Promise} The promise for the badges result.
*/
function getReadmeBadges(data, downloaded) {
// Use badges from both the package dir and root README
return Promise.props({
onPackageDir: typeof data.readme === 'string' && data.readme ? data.readme : fileContents(`${downloaded.packageDir}/${data.readmeFilename || 'README.md'}`),
onRoot: downloaded.dir !== downloaded.packageDir ? fileContents(`${downloaded.dir}/${data.readmeFilename || 'README.md'}`) : '',
})
.then((readmes) => Promise.props({
onRoot: detectReadmeBadges(readmes.onRoot || ''),
onPackageDir: detectReadmeBadges(readmes.onPackageDir || ''),
}))
// Cleanup duplicates
.then((badges) => uniqWith([...badges.onPackageDir, ...badges.onRoot], isEqual));
}
/**
* Gets the repository linters.
*
* @param {Object} downloaded - The downloaded info (`dir`, `packageDir`, ...).
*
* @returns {Promise} The promise for the linters result.
*/
function getRepoLinters(downloaded) {
// Linters usually are at the root but we consider both just in case..
return Promise.props({
onPackageDir: detectRepoLinters(downloaded.packageDir),
onRootDir: downloaded.dir !== downloaded.packageDir ?
detectRepoLinters(downloaded.dir)
// A JSON error might occur if `detect-repo-linters`fails to parse `package.json` as JSON
// Since the `package.json` at the root was not validated, it can have errors
// If that's the case, we want to skip them here
.catch({ name: 'JSONError' }, () => {
log.warn({ dir: downloaded.dir }, 'Error reading downloaded package.json when scanning for linters');
return [];
}) :
[],
})
.then((linters) => uniq([...linters.onPackageDir, ...linters.onRootDir]));
}
/**
* Fetches the code coverage.
*
* @param {Object} packageJson - The latest package.json data (normalized).
* @param {Array} badges - The badges detected by the detect-readme-badges package to speed up the process.
*
* @returns {Promise} The promise for the code coverage, a number from 0 to 1.
*/
function fetchCodeCoverage(packageJson, badges) {
const repository = packageJson.repository;
if (!repository) {
return Promise.resolve();
}
return fetchCoverage(repository.url, {
badges,
got: { retry: gotRetry },
})
.catch((err) => {
const name = packageJson.name;
/* istanbul ignore next */
if (err.errors) {
err.errors.forEach((err, index) => log.warn({ err }, `Error #${index} while fetching ${name} code coverage of`));
}
/* istanbul ignore next */
log.error({ err }, `Failed to fetch ${name} code coverage`);
/* istanbul ignore next */
throw err;
});
}
/**
* Checks the package looking for known vulnerabilities.
* Uses https://github.com/nodesecurity/nsp under the hood.
*
* @param {Object} packageJson - The latest package.json data (normalized).
*
* @returns {Promise} The promise for the vulnerabilities or false if package is totally broken.
*/
function checkVulnerabilities(packageJson) {
const url = 'https://registry.npmjs.org/-/npm/v1/security/advisories/search';
return got(url, {
json: true,
retry: gotRetry,
query: {
module: packageJson.name,
version: packageJson.version,
},
})
.then((res) => res.body.objects)
.map((vulnerability) => ({
id: vulnerability.id,
title: vulnerability.title,
overview: vulnerability.overview,
recommendation: vulnerability.recommendation,
createdAt: vulnerability.created,
updatedAt: vulnerability.updated,
severity: vulnerability.severity,
module: vulnerability.module_name,
vulnerableVersions: vulnerability.vulnerable_versions,
patchedVersions: vulnerability.patched_versions,
advisory: vulnerability.url,
}))
.catch({ statusCode: 404 }, () => {
log.warn({ url }, `The npm security API returned 404 when fetching ${packageJson.name}@${packageJson.version} vulnerabilities`);
return [];
})
.catch((err) => {
/* istanbul ignore next */
log.error({ err }, `Failed to scan ${packageJson.name}@${packageJson.version} vulnerabilities`);
/* istanbul ignore next */
throw err;
});
}
/**
* Checks the package dependencies looking for outdated versions.
* Uses https://github.com/alanshaw/david under the hood, the package that powers https://david-dm.org/.
*
* @param {String} name - The package name.
* @param {String} dir - The package directory.
* @param {Object} options - The options inferred from source() options.
*
* @returns {Promise} The promise for the outdated dependencies, indexed by name or false if deps are totally broken.
*/
function checkOutdatedDeps(name, dir, options) {
const jsonFile = `${dir}/.npms-david.json`;
// Need to pipe stdout to a file due to a NodeJS bug where the output was being truncated
// See: https://github.com/nodejs/node/issues/784
// We also run a binary wrapper around the david package because it had memory leaks, causing the memory of this
// process to grow over time
return exec(exec.escape`${davidBin} --registry ${options.npmRegistry} > ${jsonFile}`, {
cwd: dir,
timeout: 60 * 1000,
})
.then(() => loadJsonFile(jsonFile))
// Ignore broken deps (e.g.: ccbuild@1.8.1)
.catch((err) => /failed to get versions/i.test(err.stderr), (err) => {
log.warn({ err }, `Some of ${name} dependencies are broken, skipping check outdated..`);
return false;
})
// Ignore broken package data (e.g.: gqformemail)
.catch((err) => /versions.sort is not a function/i.test(err.stderr), (err) => {
log.warn({ err }, `The package data of ${name} is broken, skipping check outdated..`);
return false;
})
// Ignore broken package name (e.g.: @~lisfan/vue-image-placeholder)
.catch((err) => /\[ERR_ASSERTION\]/i.test(err.stderr), (err) => {
log.warn({ err }, `The package data of ${name} is broken (probably the name), skipping check outdated..`);
return false;
})
// Many packages have a `.npmrc` with a custom registry that require authentication
// Those packages are broken, in the sense that they can't be installed by the outside world (e.g.: webhint-hint-chisel)
.catch((err) => /requires auth credentials/i.test(err.message), (err) => {
err.unrecoverable = true;
throw err;
})
.catch((err) => {
/* istanbul ignore next */
log.error({ err }, `Failed to check outdated dependencies of ${name}`);
/* istanbul ignore next */
throw err;
});
}
// ----------------------------------------------------------------------------
// TODO: code complexity? https://www.npmjs.com/package/escomplex
// TODO: technical debts, such as TODO's and FIXME's?
/**
* Runs the source analyzer.
*
* @param {String} data - The package data.
* @param {Object} packageJson - The latest package.json data (normalized).
* @param {Object} downloaded - The downloaded info (`dir`, `packageJson`, ...).
* @param {Object} [options] - The options; read below to get to know each available option.
*
* @returns {Promise} The promise that fulfills when done.
*/
function source(data, packageJson, downloaded, options) {
options = Object.assign({
npmRegistry: 'https://registry.npmjs.org', // The registry url to be used
}, options);
// Analyze source first because the external cli tools add files to the directory
return Promise.try(() => (
promisePropsSettled({
files: inspectFiles(data, downloaded),
badges: getReadmeBadges(data, downloaded),
linters: getRepoLinters(downloaded),
})
.tap((props) =>
// Only now we got badges..
fetchCodeCoverage(packageJson, props.badges)
.then((coverage) => { props.coverage = coverage; })
)
))
// Finally use external cli tools
.then((props) => (
promisePropsSettled({
outdatedDependencies: checkOutdatedDeps(packageJson.name, downloaded.packageDir, options),
vulnerabilities: checkVulnerabilities(packageJson),
})
.then((props_) => Object.assign(props, props_))
))
.then((result) => deepCompact(result))
.tap(() => log.debug(`The source collector for ${packageJson.name} completed successfully`));
}
module.exports = source;
================================================
FILE: lib/analyze/collect/util/fileContents.js
================================================
'use strict';
const readFile = Promise.promisify(require('fs').readFile);
const log = logger.child({ module: 'util/file-contents' });
/**
* Gets the file contents of a file.
*
* @param {String} path - The path.
*
* @returns {Promise} A promise that fulfills when done.
*/
function fileContents(path) {
return readFile(path)
.then((buffer) => buffer.toString())
// Return 0 if path does not exist
.catch({ code: 'ENOENT' }, () => null)
// Return 0 if path is directory
.catch({ code: 'EISDIR' }, () => null)
// Return 0 if too many symlinks are being followed, e.g.: `condensation`
.catch({ code: 'ELOOP' }, (err) => {
log.warn({ err }, `ELOOP while getting file size of ${path}, returning 0..`);
return null;
})
// Ignore errors of packages that have large nested paths.. e.g.: `cordova-plugin-forcetouch`
.catch({ code: 'ENAMETOOLONG' }, (err) => {
/* istanbul ignore next */
log.warn({ err }, `ENAMETOOLONG while getting file size of ${path}, returning 0..`);
/* istanbul ignore next */
return null;
});
}
module.exports = fileContents;
================================================
FILE: lib/analyze/collect/util/fileSize.js
================================================
'use strict';
const stat = Promise.promisify(require('fs').stat);
const globby = require('globby');
const log = logger.child({ module: 'util/file-size' });
/**
* Gets the size of a regular file(s).
*
* @param {String|Array} path - The path(s).
*
* @returns {Promise} A promise that fulfills when done.
*/
function fileSize(path) {
const paths = Array.isArray(path) ? path : [path];
return Promise.map(paths, (path) => (
stat(path)
.then((stat) => stat.isFile() ? stat.size : 0)
// Return 0 if path does not exist
.catch({ code: 'ENOENT' }, () => 0)
// Return 0 if too many symlinks are being followed, e.g.: `condensation`
.catch({ code: 'ELOOP' }, (err) => {
log.warn({ err }, `ELOOP while getting file size of ${path}, returning 0..`);
return 0;
})
// Ignore errors of packages that have large nested paths.. e.g.: `cordova-plugin-forcetouch`
.catch({ code: 'ENAMETOOLONG' }, (err) => {
/* istanbul ignore next */
log.warn({ err }, `ENAMETOOLONG while getting file size of ${path}, returning 0..`);
/* istanbul ignore next */
return 0;
})
), { concurrency: 50 })
.then((sizes) => sizes.reduce((sum, size) => sum + size, 0));
}
/**
* Gets the size of a directory.
*
* @param {String} dir - The directory path.
*
* @returns {Promise} A promise that fulfills when done.
*/
function fileSizeDir(dir) {
return globby('**/*', {
cwd: dir,
// Only return files
onlyFiles: true,
// Include files starting with a dot
dot: true,
// Ignore symlinks to avoid loops
followSymlinkedDirectories: false,
// Return absolute paths
absolute: true,
})
.then((paths) => fileSize(paths));
}
module.exports = fileSize;
module.exports.dir = fileSizeDir;
================================================
FILE: lib/analyze/collect/util/pointsToRanges.js
================================================
'use strict';
const moment = require('moment');
/**
* Aggregates an array of points into buckets of date ranges.
*
* The array of points must have a `date` property with a moment's date.
*
* @param {Array} points - The array of points.
* @param {Array} buckets - An array of buckets, see bucketsFromBreakpoints().
*
* @returns {Array} An array of ranges, which are objects with `from`, `to` and `points` properties.
*/
function pointsToRanges(points, buckets) {
return buckets.map((bucket) => {
const filteredPoints = points.filter((point) => moment.utc(point.date).isBetween(bucket.start, bucket.end, null, '[)'));
return {
from: moment.utc(bucket.start).toISOString(),
to: moment.utc(bucket.end).toISOString(),
points: filteredPoints,
};
});
}
/**
* Utility function that builds a buckets array from breakpoints expressed in days.
* Useful to use in conjunction with `pointsToRanges()`.
*
* @param {Array} breakpoints - The breakpoints (order must be ASC).
*
* @returns {Array} An array of objects containaing `start` and `end` moment dates to be used in pointsToRanges.
*/
function bucketsFromBreakpoints(breakpoints) {
const referenceDate = moment.utc().startOf('day');
return breakpoints
.map((breakpoint) => ({
start: referenceDate.clone().subtract(breakpoint, 'd'),
end: referenceDate,
}));
}
module.exports = pointsToRanges;
module.exports.bucketsFromBreakpoints = bucketsFromBreakpoints;
================================================
FILE: lib/analyze/collect/util/promisePropsSettled.js
================================================
'use strict';
const mapValues = require('lodash/mapValues');
/**
* Promise utility similar to bluebird's .props() but only fulfills when all promises are fulfilled,
* even rejections.
*
* @param {Object} object - The object that contain the promises.
*
* @returns {Promise} A promise that only fulfills after all the promises have fulfilled.
*/
function promisePropsSettled(object) {
object = mapValues(object, (promise) => Promise.resolve(promise).reflect());
return Promise.props(object)
.then((results) => (
mapValues(results, (inspection) => {
if (inspection.isRejected()) {
throw inspection.reason();
}
return inspection.value();
})
));
}
module.exports = promisePropsSettled;
================================================
FILE: lib/analyze/download/git.js
================================================
'use strict';
const urlLib = require('url');
const exec = require('../util/exec');
const hostedGitInfo = require('../util/hostedGitInfo');
const findPackageDir = require('./util/findPackageDir');
const mergePackageJson = require('./util/mergePackageJson');
const assertFilesCount = require('./util/assertFilesCount');
const log = logger.child({ module: 'download/git' });
/**
* Downloads the package using git.
*
* @param {String} url - The repository clone URL.
* @param {String} ref - The ref to download (null to download the default branch).
* @param {String} tmpDir - The temporary dir path to download to.
* @param {Object} options - The options inferred from github() options.
*
* @returns {Promise} The promise that resolves with the downloaded ref.
*/
function download(url, ref, tmpDir, options) {
let downloadedRef = ref;
log.debug(`Will now clone ${url}`);
// Clone repository
return exec(exec.escape`git clone -q ${url} .`, { cwd: tmpDir })
// Checkout the ref if any
.then(() => ref && exec(exec.escape`git checkout -q ${ref}`, { cwd: tmpDir }))
// Wait a maximum of X time
.timeout(options.maxTime)
// Clear out the ref if any error occurred
.catch((err) => {
downloadedRef = null;
throw err;
})
// Finally remove the .git folder if it exists
.finally(() => exec(exec.escape`rm -rf ${tmpDir}/.git`))
// Repository does not exist, is invalid, or we have no permission?
// https://foo:bar@github.com/something/thatwillneverexist.git -> authentication failed
// https://foo:bar@github.com/some/privaterepo.git -> authentication failed
// https://foo:bar@github.com/org/foo+foo.git -> not found
// https://foo:bar@github.com/org/foo%foo.git -> unable to access (400)
// https://foo:bar@bitbucket.org/something/thatwillneverexist.git -> not found
// https://foo:bar@bitbucket.org/some/privaterepo.git -> authentication failed
// https://foo:bar@bitbucket.org/org/foo+foo.git -> not found
// https://foo:bar@bitbucket.org/org/foo%foo.git -> unable to access (400)
// https://foo:bar@gitlab.com/something/thatwillneverexist.git -> authenticated failed
// https://foo:bar@gitlab.com/some/privaterepo.git -> authentication failed
// https://foo:bar@gitlab.com/org/foo+foo.git -> unable to access (500)
// https://foo:bar@gitlab.com/org/foo%foo.git -> unable to access (400)
// https://foo:bar@gitlab.com/upe-consulting/npm%2Flogger.git/info/refs -> not valid: is this a git repository?
.catch((err) => /not found|authentication failed/i.test(err.stderr), (err) => {
log.warn({ err }, `Repository ${url} does not exist or is private`);
})
.catch((err) => /unable to access|is this a git repository/i.test(err.stderr), (err) => {
log.warn({ err }, `Repository ${url} seems to be invalid`);
})
// Check if ref no longer exists
// did not match any file -> if branch or tag does not exist
// reference is not a tree -> if sha does not exist
.catch((err) => /did not match any file|reference is not a tree/i.test(err.stderr), (err) => {
log.warn({ err }, `Failed to checkout ref ${ref} for ${url}, using default branch..`);
})
// Finally return the ref
.then(() => downloadedRef);
}
/**
* Gets the clone URL from `gitInfo` (https).
*
* @param {Object} gitInfo - The git info object.
*
* @returns {String} The https clone URL.
*/
function getCloneUrl(gitInfo) {
let url;
// Use https:// protocol to avoid having to setup ssh keys in GitHub, Bitbucket and GitLab
// Also, foo@bar is added as username & password to prevent git clone from prompting for credentials
// Even if foo@bar does not exist or is invalid, public repositories are still cloned correctly
url = gitInfo.https().substr(4);
url = Object.assign(urlLib.parse(url), { auth: 'foo:bar' });
url = urlLib.format(url);
return url;
}
// ------------------------------------------------------------------
/**
* Checks if this package should be downloaded using git.
*
* If it does, the promise results with a function that will download the package.
* If it does not, the promise will resolve to null.
*
* @param {Object} packageJson - The latest package.json data (normalized).
* @param {Object} [options] - The options; read below to get to know each available option.
*
* @returns {Function} The download function or null.
*/
function git(packageJson, options) {
const repository = packageJson.repository;
if (!repository) {
return null;
}
const gitInfo = hostedGitInfo(repository.url);
if (!gitInfo) {
return null;
}
options = Object.assign({
maxTime: 600000, // Max allowed download time (10m)
maxFiles: 32000, // Max allowed files to download
}, options);
return (tmpDir) => {
const url = getCloneUrl(gitInfo);
const ref = packageJson.gitHead || null;
return download(url, ref, tmpDir, options)
.then((gitRef) => ({
downloader: 'git',
dir: tmpDir,
gitRef,
}))
// Assert that the number of downloaded files is not too big to be processed
.tap(() => assertFilesCount(tmpDir, options.maxFiles))
// Find package dir within the repository
// The package is usually in the root for regular repositories, but not for mono-repositories
.tap((downloaded) => (
findPackageDir(packageJson, tmpDir)
.then((packageDir) => { downloaded.packageDir = packageDir; })
))
// Merge the downloaded repository package.json with the one from the registry
// See mergePackageJson() to know why we do this
.tap((downloaded) => (
mergePackageJson(packageJson, downloaded.packageDir)
.then((downloadedPackageJson) => { downloaded.packageJson = downloadedPackageJson; })
))
// Remove package-lock.json file if any.
// People often forgot to update the package-lock, which is not published, and messes up
// with collectors, such as `nodesecurity`.
.tap((downloaded) => exec(exec.escape`rm -rf ${downloaded.packageDir}/package-lock.json`));
};
}
module.exports = git;
================================================
FILE: lib/analyze/download/github.js
================================================
'use strict';
const fs = require('fs');
const tokenDealer = require('token-dealer');
const got = require('got');
const untar = require('./util/untar');
const hostedGitInfo = require('../util/hostedGitInfo');
const findPackageDir = require('./util/findPackageDir');
const gotRetry = require('../util/gotRetry');
const exec = require('../util/exec');
const mergePackageJson = require('./util/mergePackageJson');
const log = logger.child({ module: 'download/github' });
const unavailableStatusCodes = [404, 400, 403, 451]; // 404 - not found; 400 - invalid repo name; 403/451 - dmca takedown
/**
* Downloads the package from GitHub.
*
* @param {String} shorthand - The <org>/<repo>.
* @param {String} ref - The ref to download (null to download the default branch).
* @param {String} tmpDir - The temporary dir path to download to.
* @param {Object} options - The options inferred from github() options.
*
* @returns {Promise} The promise that resolves with the downloaded ref.
*/
function download(shorthand, ref, tmpDir, options) {
const url = `https://api.github.com/repos/${shorthand}/tarball/${ref || ''}`;
const tarballFile = `${tmpDir}/tarball.tar.gz`;
let downloadedRef = ref;
log.trace(`Will download tarball of ${shorthand}@${ref || 'default'}..`);
// Download tarball
// Use token dealer to circumvent rate limit issues
return tokenDealer(options.tokens, (token, exhaust) => (
new Promise((resolve, reject) => {
let request;
const handleRateLimit = (response, err) => {
if (response.headers['x-ratelimit-remaining'] === '0') {
const isRateLimitError = err && err.statusCode === 403;
exhaust(Number(response.headers['x-ratelimit-reset']) * 1000, isRateLimitError);
}
};
got.stream(url, {
timeout: 30000,
headers: Object.assign({ accept: 'application/vnd.github.v3+json' }, token ? { authorization: `token ${token}` } : null),
retry: gotRetry,
})
.on('request', (request_) => { request = request_; })
.on('response', (response) => {
// Handle rate limit stuff
handleRateLimit(response);
// Check if the file is too big..
const contentLength = Number(response.headers['content-length']);
if (contentLength > options.maxSize) {
request.abort();
reject(Object.assign(new Error(`${shorthand} tarball is too large (~${Math.round(contentLength / 1024 / 1024)}MB)`), {
unrecoverable: true,
}));
}
})
.on('error', (err, details, response) => {
// Handle rate limit stuff
try {
response && handleRateLimit(response, err);
} catch (exhaustedErr) {
err = exhaustedErr || err;
}
reject(err);
})
.pipe(fs.createWriteStream(tarballFile))
.on('error', reject)
.on('finish', resolve);
})
), {
group: 'github',
wait: options.waitRateLimit,
onExhausted: (token, reset) => log.error(`Token ${token ? token.substr(0, 10) : '<empty>'}.. exhausted`, { reset }),
})
// Extract tarball
.then(() => {
log.debug({ tarballFile }, `Successfully downloaded ${shorthand} tarball, will now extract ..`);
return untar(tarballFile, { maxFiles: options.maxFiles });
})
// Clear out the ref if any error occurred; also delete downloaded archive if any
.catch((err) => {
downloadedRef = null;
return exec(exec.escape`rm -rf ${tarballFile}`)
.finally(() => { throw err; });
})
// If we got a 404 either the repository or the specified ref does not exist (devs usually forget to push or do push -f)
// If a specific ref was requested, attempt to download the default branch
.catch((err) => err.statusCode === 404 && ref, () => {
log.warn(`Download of ${shorthand}@${ref} tarball failed with 404, trying default branch..`);
return download(shorthand, null, tmpDir, options);
})
// Check if the repository is unavailable
.catch((err) => unavailableStatusCodes.indexOf(err.statusCode) !== -1, (err) => {
log.warn({ err }, `Download of ${shorthand} tarball failed with ${err.statusCode}`);
})
// Check if the ref is malformed to mark this error as unrecoverable
// e.g.: generator-hold
.catch((err) => /request path contains unescaped characters/i.test(err.message), (err) => {
err.unrecoverable = true;
throw err;
})
.catch((err) => {
log.error({ err }, `Download of ${shorthand} tarball failed`);
throw err;
})
// Finally return the ref
.then(() => downloadedRef);
}
// ------------------------------------------------------------------
/**
* Checks if this package should be downloaded from GitHub.
*
* If it does, the promise results with a function that will download the package.
* If it does not, the promise will resolve to null.
*
* @param {Object} packageJson - The latest package.json data (normalized).
* @param {Object} [options] - The options; read below to get to know each available option.
*
* @returns {Function} The download function or null.
*/
function github(packageJson, options) {
const repository = packageJson.repository;
if (!repository) {
return null;
}
const gitInfo = hostedGitInfo(repository.url);
if (!gitInfo || gitInfo.type !== 'github') {
return null;
}
options = Object.assign({
tokens: null, // The GitHub API tokens to use
waitRateLimit: false, // True to wait if rate limit for all tokens were exceeded
maxSize: 262144000, // Max allowed download size (250MB)
maxFiles: 32000, // Max allowed files to download (extract)
}, options);
return (tmpDir) => {
const shorthand = `${gitInfo.user}/${gitInfo.project}`;
const ref = packageJson.gitHead || null;
return download(shorthand, ref, tmpDir, options)
.then((gitRef) => ({
downloader: 'github',
dir: tmpDir,
gitRef,
}))
// Find package dir within the repository
// The package is usually in the root for regular repositories, but not for mono-repositories
.tap((downloaded) => (
findPackageDir(packageJson, tmpDir)
.then((packageDir) => { downloaded.packageDir = packageDir; })
))
// Merge the downloaded repository package.json with the one from the registry
// See mergePackageJson() to know why we do this
.tap((downloaded) => (
mergePackageJson(packageJson, downloaded.packageDir)
.then((downloadedPackageJson) => { downloaded.packageJson = downloadedPackageJson; })
))
// Remove package-lock.json file if any.
// People often forgot to update the package-lock, which is not published, and messes up
// with collectors, such as `nodesecurity`.
.tap((downloaded) => exec(exec.escape`rm -rf ${downloaded.packageDir}/package-lock.json`));
};
}
module.exports = github;
================================================
FILE: lib/analyze/download/index.js
================================================
'use strict';
const os = require('os');
const stat = Promise.promisify(require('fs').stat);
const downloaders = require('require-directory')(module, './', { recurse: false });
const kebabCase = require('lodash/kebabCase');
const exec = require('../util/exec');
const downloadersOrder = [
(packageJson, options) => downloaders.github(packageJson, { tokens: options.githubTokens, waitRateLimit: options.waitRateLimit }),
(packageJson) => downloaders.git(packageJson),
(packageJson) => downloaders.npm(packageJson),
];
/**
* Generates a random string.
* This is a very fast but naive algorithm.
*
* @returns {String} The random string.
*/
function getRandomStr() {
return Math.random()
.toString(36)
.slice(2);
}
/**
* Creates a temporary folder for a package to be downloaded to.
*
* @param {String} name - The package name.
*
* @returns {Promise} The promise that resolves with the temporary folder path.
*/
function createTmpDir(name) {
// Suffix the folder with a random string to make it more unique
// Additionally, transform any special chars such as `/` to avoid having recursive directories
const dir = `${os.tmpdir()}/npms-analyzer/${kebabCase(name)}-${getRandomStr()}`;
return exec(exec.escape`rm -rf ${dir}`)
.then(() => exec(exec.escape`mkdir -p ${dir}`))
.then(() => dir);
}
/**
* Cleans old packages from the temporary folder.
*
* @returns {Promise} The promise that resolves when odne.
*/
function cleanTmpDir() {
const dir = `${os.tmpdir()}/npms-analyzer`;
return stat(dir)
.catch({ code: 'ENOENT' }, () => false)
.return(true)
.then((exists) => exists && exec(exec.escape`find ${dir} -mindepth 1 -maxdepth 1 -type d -mtime +1 -print0 | xargs -0 rm -rf`));
}
// -------------------------------------------------------------
/**
* Downloads a package into a temporary folder.
*
* @param {Object} packageJson - The latest package.json data (normalized).
* @param {Object} [options] - The options; read below to get to know each available option.
*
* @returns {Promise} A promise that resolves with the downloaded info (`dir`, `packageJson`, ...).
*/
function download(packageJson, options) {
let downloadFn;
options = Object.assign({
githubTokens: null, // The GitHub API tokens to use
waitRateLimit: false, // True to wait if rate limit for all tokens were exceeded
}, options);
downloadersOrder.some((downloader) => {
downloadFn = downloader(packageJson, options);
return !!downloadFn;
});
/* istanbul ignore if */
if (!downloadFn) {
return Promise.reject(Object.assign(new Error(`Could not find suitable downloader for ${packageJson.name}`),
{ unrecoverable: false }));
}
// Create temporary directory
return createTmpDir(packageJson.name)
// Download the package into the temporary directory
.then((tmpDir) => (
downloadFn(tmpDir)
// Cleanup the directory if download failed
.catch((err) => (
exec(exec.escape`rm -rf ${tmpDir}`)
.finally(() => { throw err; })
))
));
}
module.exports = download;
module.exports.downloaders = downloaders;
module.exports.cleanTmpDir = cleanTmpDir;
================================================
FILE: lib/analyze/download/npm.js
================================================
'use strict';
const fs = require('fs');
const got = require('got');
const untar = require('./util/untar');
const gotRetry = require('../util/gotRetry');
const exec = require('../util/exec');
const mergePackageJson = require('./util/mergePackageJson');
const log = logger.child({ module: 'download/npm' });
/**
* Downloads the package from the npm registry.
*
* @param {String} target - The <name>@<version> to download.
* @param {String} url - The tarball URL.
* @param {String} tmpDir - The temporary dir path to download to.
* @param {Object} options - The options inferred from npm() options.
*
* @returns {Promise} The promise that fulfills when done.
*/
function download(target, url, tmpDir, options) {
const tarballFile = `${tmpDir}/tarball.tar.gz`;
log.debug({ url }, `Will download tarball of ${target}..`);
// Download tarball
return new Promise((resolve, reject) => {
let request;
got.stream(url, { timeout: 30000, retry: gotRetry })
.on('error', reject)
.on('request', (request_) => { request = request_; })
.on('response', (response) => {
// Check if the file is too big..
const contentLength = Number(response.headers['content-length']);
if (contentLength > options.maxSize) {
request.abort();
reject(Object.assign(new Error(`Tarball is too large (~${Math.round(contentLength / 1024 / 1024)}MB)`), {
unrecoverable: true,
}));
}
})
.pipe(fs.createWriteStream(tarballFile))
.on('error', reject)
.on('finish', resolve);
})
// Extract tarball
.then(() => {
log.debug({ tarballFile }, `Successfully downloaded ${target} tarball, will now extract ..`);
return untar(tarballFile, { maxFiles: options.maxFiles });
})
// Check if the repository does not exist
.catch({ statusCode: 404 }, (err) => {
log.warn({ err }, `Download of ${target} tarball failed with ${err.statusCode}`);
return exec(exec.escape`rm -rf ${tarballFile}`);
})
.catch((err) => {
log.error({ err }, `Download of ${target} tarball failed`);
throw err;
});
}
// ------------------------------------------------------------------
/**
* Checks if this package should be downloaded from the npm registry.
*
* If it does, the promise results with a function that will download the package.
* If it does not, the promise will resolve to null.
*
* @param {Object} packageJson - The latest package.json data (normalized).
* @param {Object} [options] - The options; read below to get to know each available option.
*
* @returns {Function} The download function or null.
*/
function npm(packageJson, options) {
options = Object.assign({
maxSize: 262144000, // Max allowed download size (250MB)
maxFiles: 32000, // Max allowed files to download (extract)
}, options);
return (tmpDir) => {
const url = packageJson.dist && packageJson.dist.tarball;
const target = `${packageJson.name}@${packageJson.version}`;
return Promise.try(() => {
// Protect against packages that don't have tarballs, e.g.: `roost-mongo@0.1.0`
if (url) {
return download(target, url, tmpDir, options);
}
log.warn(`No tarball url for ${target}`);
})
.then(() => ({
downloader: 'npm',
dir: tmpDir,
packageDir: tmpDir,
}))
// Merge the downloaded repository package.json with the one from the registry
// See mergePackageJson() to know why we do this
.tap((downloaded) => (
mergePackageJson(packageJson, downloaded.packageDir)
.then((downloadedPackageJson) => { downloaded.packageJson = downloadedPackageJson; })
));
};
}
module.exports = npm;
================================================
FILE: lib/analyze/download/util/assertFilesCount.js
================================================
'use strict';
const exec = require('../../util/exec');
/**
* Asserts that the number of files in a directory is within a certain threshold.
*
* This should be used by downloaders if they can't pre-calculate the number of files
* without actually starting populating a directory with files.
*
* @param {String} dir - The dir.
* @param {Number} maxFiles - The total number of files.
*
* @returns {Promise} A promise that fulfills when done.
*/
function assertFilesCount(dir, maxFiles) {
return exec(exec.escape`find ${dir} | wc -l`)
.spread((stdout) => parseInt(stdout, 10))
.tap((filesCount) => {
if (isNaN(filesCount)) {
throw Object.assign(new Error('Unable to retrieve the number of files within the directory'),
{ unrecoverable: true, dir });
}
if (filesCount > maxFiles) {
throw Object.assign(new Error('Directory has too many files'), { unrecoverable: true, dir });
}
});
}
module.exports = assertFilesCount;
================================================
FILE: lib/analyze/download/util/findPackageDir.js
================================================
'use strict';
const path = require('path');
const loadJsonFile = require('load-json-file');
const globby = require('globby');
const log = logger.child({ module: 'util/find-package-dir' });
/**
* Searches for the real package dir after testing against the root one fails.
*
* @param {Object} packageJson - The package.json from the registry.
* @param {String} dir - The folder in which the package was downloaded.
*
* @returns {Promise} A promise that resolves with the package dir or null.
*/
function lookForPackageDir(packageJson, dir) {
// Gather all package json files
return globby('**/package.json', {
cwd: dir,
// Only return files
onlyFiles: true,
// Ignore symlinks to avoid loops
followSymlinkedDirectories: false,
})
// Transform them into directories, removing the root one
.then((files) => (
files
// Filter root one
.filter((file) => file !== 'package.json')
// Build dir arrays from matched files
.map((file) => path.join(dir, path.dirname(file)))
))
// Find the one that matches the package
.reduce((packageDir, possiblePackageDir) => {
if (packageDir) {
return packageDir;
}
return isSamePackage(packageJson, possiblePackageDir)
.then((isSame) => isSame ? possiblePackageDir : null);
}, null);
}
/**
* Tests if a directory matches the package we are looking for.
*
* @param {Object} packageJson - The package.json from the registry.
* @param {String} dir - The folder we are testing against.
*
* @returns {Promise} A promise that resolves with true if it matched, false otherwise.
*/
function isSamePackage(packageJson, dir) {
const file = `${dir}/package.json`;
return loadJsonFile(file)
// Ignore if the file doesn't exist
.catch({ code: 'ENOENT' }, () => ({}))
// Ignore any errors but log them
.catch((err) => {
log.debug({ err, file }, 'Error reading package.json');
return {};
})
.then((downloadedPackageJson) => packageJson.name === downloadedPackageJson.name);
}
// -----------------------------------------------------
/**
* Finds the real package directory.
*
* If the package.json file at the root matches, the `packageDir` will be the same as `dir`.
* If not, this function will do a deep search for a package.json that matches.
*
* For standard repositories, `packageDir` will be equal to the `dir`.
* For mono repositories, `packageDir` will be a sub-directory of `dir` pointing to where the package actually is.
* If we couldn't find the `packageDir`, `dir` will be returned.
*
* @param {Object} packageJson - The package.json from the registry.
* @param {String} dir - The folder in which the package was downloaded.
*
* @returns {Promise} A promise that resolves with the package directory.
*/
function findPackageDir(packageJson, dir) {
// Short-circuit to check against the root
return isSamePackage(packageJson, dir)
// Find using glob
.then((isSame) => isSame ? dir : lookForPackageDir(packageJson, dir))
// Fallback to using the root dir
.then((packageDir) => packageDir || dir);
}
module.exports = findPackageDir;
================================================
FILE: lib/analyze/download/util/mergePackageJson.js
================================================
'use strict';
const writeFile = Promise.promisify(require('fs').writeFile);
const loadJsonFile = require('load-json-file');
const assignWith = require('lodash/assignWith');
const notEmpty = require('deep-compact').notEmpty;
const normalizePackageJson = require('../../util/normalizePackageJson');
const log = logger.child({ module: 'util/merge-package-json' });
/**
* Merges the package json with the downloaded one.
*
* The published package.json has higher priority than the one downloaded from the source code since it's more exact.
* Though, some packages have pre-publish scripts that mutate the package.json hiding important stuff: One good example is
* `bower`, which bundles dependencies in the package itself to speed up installation.
*
* The passed in `packageJson` will be mutated and it will be written out to the downloaded folder, overwriting the downloaded one.
* The promise will be resolved with the original downloaded package json.
*
* @param {Object} packageJson - The package.json from the registry.
* @param {String} packageDir - The temporary folder in which the package was downloaded.
*
* @returns {Promise} A promise that resolves with the downloaded package json.
*/
function mergePackageJson(packageJson, packageDir) {
const file = `${packageDir}/package.json`;
// Read json file & normalize it
return loadJsonFile(file)
.then((downloadedPackageJson) => normalizePackageJson(downloadedPackageJson))
// Ignore if the file doesn't exist
.catch({ code: 'ENOENT' }, () => ({}))
// Ignore any errors but log them
.catch((err) => {
log.warn({ err, file }, 'Error reading downloaded package.json');
return {};
})
// Merge
.tap((downloadedPackageJson) => {
assignWith(packageJson, downloadedPackageJson,
(objValue, srcValue, key, obj) => notEmpty(objValue, key, obj) ? objValue : srcValue);
})
// Write to disk
.tap(() => writeFile(file, JSON.stringify(packageJson, null, 2)));
}
module.exports = mergePackageJson;
================================================
FILE: lib/analyze/download/util/untar.js
================================================
'use strict';
const path = require('path');
const which = require('which');
const unlink = Promise.promisify(require('fs').unlink);
const exec = require('../../util/exec');
// Prefer bsdtar over the installed tar.. bsdtar is more benevolent when dealing with certain errors
// See: http://comments.gmane.org/gmane.comp.gnu.mingw.msys/4816
/* eslint-disable max-statements-per-line, max-len */
const tarExec = (() => { try { return which.sync('bsdtar'); } catch (err) { return 'tar'; } })();
const malformedRegExp = /unrecognized archive format|does not look like a tar archive|not in gzip format|unknown compression format|remove already-existing dir/i;
/* eslint-enable max-statements-per-line, max-len */
/**
* Asserts that the number of files in the tar archive is below a certain threshold.
*
* @param {String} file - The file path.
* @param {Number} maxFiles - The total number of files.
*
* @returns {Promise} A promise that fulfills when done.
*/
function assertFilesCount(file, maxFiles) {
// Ignore "unknown extended header XX" because there might be a lot of them, e.g.: pickles2-contents-editor
// This only happens when using gnu tar, see: http://lee.greens.io/blog/2014/05/06/fix-tar-errors-on-os-x/
// e.g.: http://registry.npmjs.org/pickles2-contents-editor/-/pickles2-contents-editor-2.0.0-alpha.1.tgz
return exec(exec.escape`
listFiles() {
${tarExec} -ztf ${file}
}
filter() {
(grep -v "unknown extended header"; exit 0)
}
set -o pipefail
{ listFiles 2>&1 1>&3 | filter 1>&2; } 3>&1 | wc -l
`, { shell: '/bin/bash' })
.spread((stdout) => parseInt(stdout, 10))
.tap((filesCount) => {
if (isNaN(filesCount)) {
throw Object.assign(new Error('Unable to retrieve the number of files within the tarball'),
{ unrecoverable: true, tarballFile: file });
}
if (filesCount > maxFiles) {
throw Object.assign(new Error('Tarball has too many files'), { unrecoverable: true, tarballFile: file });
}
});
}
/**
* Decompresses a tar file to a directory.
*
* @param {String} file - The file path.
* @param {String} destDir - The destination directory.
*
* @returns {Promise} A promise that fulfills when done.
*/
function decompress(file, destDir) {
// Ignore "unknown extended header XX" because there might be a lot of them, e.g.: pickles2-contents-editor
// This only happens when using gnu tar, see: http://lee.greens.io/blog/2014/05/06/fix-tar-errors-on-os-x/
// e.g.: http://registry.npmjs.org/pickles2-contents-editor/-/pickles2-contents-editor-2.0.0-alpha.1.tgz
return exec(exec.escape`
decompress() {
${tarExec} -xf ${file} -C ${destDir} --strip-components=1
}
filter() {
(grep -v "unknown extended header"; exit 0)
}
set -o pipefail
{ decompress 2>&1 1>&3 | filter 1>&2; } 3>&1
`, { shell: '/bin/bash' })
.then(() => exec(exec.escape`chmod -R 0777 ${destDir}`));
}
// --------------------------------------------------
/**
* Small utility to untar a file.
* Malformed tar errors are ignored.
*
* @param {String} file - The file path.
* @param {Object} [options] - He options; read below to get to know each available option.
*
* @returns {Promise} A promise that fulfills when done.
*/
function untar(file, options) {
options = Object.assign({ maxFiles: 32000 }, options);
const destDir = path.dirname(file);
// Check the number of files
return assertFilesCount(file, options.maxFiles)
// Proceed with decompressing
.then(() => decompress(file, destDir))
// Delete tar file
.then(() => unlink(file))
// Ignore invalid tar files.. sometimes services respond with JSON
// e.g.: http://registry.npmjs.org/n-pubsub/-/n-pubsub-1.0.0.tgz
// e.g.: testing233 package, that somehow was able to set dist to http://example.com
// e.g.: cb-never-called
.catch((err) => malformedRegExp.test(err.stderr), (err) => {
throw Object.assign(new Error('Tarball is malformed'), { tarballFile: file, stderr: err.stderr });
})
.return(destDir);
}
module.exports = untar;
================================================
FILE: lib/analyze/evaluate/index.js
================================================
'use strict';
const evaluators = require('require-directory')(module, './', { recurse: false });
/**
* Runs all the evaluators.
*
* @param {Object} collected - The collected information.
*
* @returns {Object} The evaluation result.
*/
function evaluate(collected) {
return {
quality: evaluators.quality(collected),
popularity: evaluators.popularity(collected),
maintenance: evaluators.maintenance(collected),
};
}
module.exports = evaluate;
================================================
FILE: lib/analyze/evaluate/maintenance.js
================================================
'use strict';
const moment = require('moment');
const find = require('lodash/find');
const get = require('lodash/get');
const mapValues = require('lodash/mapValues');
const semver = require('semver');
const normalizeValue = require('normalize-value');
const log = logger.child({ module: 'evaluate/maintenance' });
/**
* Evaluates the releases frequency.
*
* @param {Object} collected - The collected information.
*
* @returns {Number} The releases frequency evaluation (from 0 to 1).
*/
function evaluateReleasesFrequency(collected) {
const releases = collected.metadata.releases;
if (!releases) {
return 0;
}
const range30 = find(releases, (range) => moment.utc(range.to).diff(range.from, 'd') === 30);
const range180 = find(releases, (range) => moment.utc(range.to).diff(range.from, 'd') === 180);
const range365 = find(releases, (range) => moment.utc(range.to).diff(range.from, 'd') === 365);
const range730 = find(releases, (range) => moment.utc(range.to).diff(range.from, 'd') === 730);
if (!range30 || !range180 || !range365 || !range730) {
throw new Error('Could not find entry in releases');
}
const mean30 = range30.count / (30 / 90);
const mean180 = range180.count / (180 / 90);
const mean365 = range365.count / (365 / 90);
const mean730 = range730.count / (730 / 90);
const quarterMean = (mean30 * 0.25) +
(mean180 * 0.45) +
(mean365 * 0.2) +
(mean730 * 0.1);
return normalizeValue(quarterMean, [
{ value: 0, norm: 0 },
{ value: 0.5, norm: 0.5 },
{ value: 1, norm: 0.75 },
{ value: 2, norm: 1 },
]);
}
/**
* Evaluates the commits frequency.
*
* @param {Object} collected - The collected information.
*
* @returns {Number} The commits frequency evaluation (from 0 to 1).
*/
function evaluateCommitsFrequency(collected) {
const commits = collected.github && collected.github.commits;
if (!commits) {
return 0;
}
const range30 = find(commits, (range) => moment.utc(range.to).diff(range.from, 'd') === 30);
const range180 = find(commits, (range) => moment.utc(range.to).diff(range.from, 'd') === 180);
const range365 = find(commits, (range) => moment.utc(range.to).diff(range.from, 'd') === 365);
if (!range30 || !range180 || !range365) {
throw new Error('Could not find entry in commits');
}
const mean30 = range30.count / (30 / 30);
const mean180 = range180.count / (180 / 30);
const mean365 = range365.count / (365 / 30);
const monthlyMean = (mean30 * 0.35) +
(mean180 * 0.45) +
(mean365 * 0.2);
return normalizeValue(monthlyMean, [
{ value: 0, norm: 0 },
{ value: 1, norm: 0.7 },
{ value: 5, norm: 0.9 },
{ value: 10, norm: 1 },
]);
}
/**
* Evaluates the open issues health.
*
* @param {Object} collected - The collected information.
*
* @returns {Number} The open issues health evaluation (from 0 to 1).
*/
function evaluateOpenIssues(collected) {
const issues = collected.github && collected.github.issues;
// If unable to get issues, evaluation is 0
if (!issues) {
return 0;
}
// If issues are disabled, return 0.5..
// We can't really evaluate something we don't know; if this value causes troubles find a better strategy
if (issues.isDisabled) {
return collected.github.forkOf ? 0.7 : 0.5; // Forks have issues disabled by default, don't be so harsh
}
// If the repository has 0 issues, evaluation is 0.7
if (!issues.count) {
return 0.7;
}
const openIssuesRatio = issues.openCount / issues.count;
return normalizeValue(openIssuesRatio, [
{ value: 0.2, norm: 1 },
{ value: 0.5, norm: 0.5 },
{ value: 1, norm: 0 },
]);
}
/**
* Evaluates the issues distribution evaluation.
*
* @param {Object} collected - The collected information.
*
* @returns {Number} The issues distribution evaluation (from 0 to 1).
*/
function evaluateIssuesDistribution(collected) {
const issues = collected.github && collected.github.issues;
// If unable to get issues, evaluation is 0
if (!issues) {
return 0;
}
// If issues are disabled, return 0.5..
// We can't really evaluate something we don't know; if this value causes troubles find a better strategy
if (issues.isDisabled) {
return collected.github.forkOf ? 0.7 : 0.5; // Forks have issues disabled by default, don't be so harsh
}
const ranges = Object.keys(issues.distribution).map(Number);
const totalCount = ranges.reduce((sum, range) => sum + issues.distribution[range], 0);
// If the repository has 0 issues, evaluation is 0.7
if (!totalCount) {
return 0.7;
}
const weights = ranges.map((range) => {
const weight = issues.distribution[range] / totalCount;
const conditioning = normalizeValue(range / 24 / 60 / 60, [
{ value: 29, norm: 1 },
{ value: 365, norm: 5 }, // An issue open for more than 1 year, weights 5x more than a normal one
]);
return weight * conditioning;
});
const mean = ranges.reduce((sum, range, index) => sum + (range * weights[index])) / (ranges.length || 1);
const issuesOpenMeanDays = mean / 60 / 60 / 24;
return normalizeValue(issuesOpenMeanDays, [
{ value: 5, norm: 1 },
{ value: 30, norm: 0.7 },
{ value: 90, norm: 0 },
]);
}
/**
* Checks if a package is finished, that is, it's stable enough that doesn't require a lot of maintenance.
*
* @param {Object} collected - The collected information.
*
* @returns {Boolean} True if finished, false otherwise.
*/
function isPackageFinished(collected) {
const isStable = semver.gte(collected.metadata.version, '1.0.0', true); // `true` = loose semver
const isNotDeprecated = !collected.metadata.deprecated;
const hasFewIssues = get(collected, 'github.issues.openCount', Infinity) < 15;
const hasREADME = !!collected.metadata.readme || get(collected, 'source.files.readmeSize', 0) > 0;
const hasTests = !!collected.metadata.hasTestScript;
const isFinished = isStable && isNotDeprecated && hasFewIssues && hasREADME && hasTests;
log.debug({ isStable, isNotDeprecated, hasFewIssues, hasREADME, hasTests },
`Package is considered ${isFinished ? 'finished' : 'unfinished'}`);
return isFinished;
}
// ----------------------------------------------------------------------------
/**
* Evaluates the package maintenance.
*
* @param {Object} collected - The collected information.
*
* @returns {Object} The evaluation result.
*/
function maintenance(collected) {
let evaluation = {
releasesFrequency: evaluateReleasesFrequency(collected),
commitsFrequency: evaluateCommitsFrequency(collected),
openIssues: evaluateOpenIssues(collected),
issuesDistribution: evaluateIssuesDistribution(collected),
};
// If the package is finished, it doesn't require a lot of maintenance
if (isPackageFinished(collected)) {
evaluation = mapValues(evaluation, (evaluation) => Math.max(evaluation, 0.9));
}
return evaluation;
}
module.exports = maintenance;
================================================
FILE: lib/analyze/evaluate/popularity.js
================================================
'use strict';
const moment = require('moment');
const find = require('lodash/find');
/**
* Evaluates the downloads count.
*
* @param {Object} collected - The collected information.
*
* @returns {Number} The monthly downloads mean (from 0 to Infinity).
*/
function evaluateDownloadsCount(collected) {
const downloads = collected.npm && collected.npm.downloads;
if (!downloads) {
return 0;
}
const index = downloads.findIndex((range) => moment.utc(range.to).diff(range.from, 'd') === 90);
if (index === -1) {
throw new Error('Could not find entry in downloads');
}
const count90 = downloads[index].count;
const count30 = count90 / 3;
return count30;
}
/**
* Evaluates the downloads acceleration.
*
* @param {Object} collected - The collected information.
*
* @returns {Number} The downloads acceleration (from -Infinity to Infinity).
*/
function evaluateDownloadsAcceleration(collected) {
const downloads = collected.npm && collected.npm.downloads;
if (!downloads) {
return 0;
}
const range30 = find(downloads, (range) => moment.utc(range.to).diff(range.from, 'd') === 30);
const range90 = find(downloads, (range) => moment.utc(range.to).diff(range.from, 'd') === 90);
const range180 = find(downloads, (range) => moment.utc(range.to).diff(range.from, 'd') === 180);
const range365 = find(downloads, (range) => moment.utc(range.to).diff(range.from, 'd') === 365);
if (!range30 || !range90 || !range180 || !range365) {
throw new Error('Could not find entry in downloads');
}
const mean30 = range30.count / 30;
const mean90 = range90.count / 90;
const mean180 = range180.count / 180;
const mean365 = range365.count / 365;
return ((mean30 - mean90) * 0.25) +
((mean90 - mean180) * 0.25) +
((mean180 - mean365) * 0.5);
}
/**
* Evaluates the community interest on the package, using its stars, forks, subscribers and contributors count.
*
* @param {Object} collected - The collected information.
*
* @returns {Number} The community interest (from 0 to Infinity).
*/
function evaluateCommunityInterest(collected) {
const starsCount = (collected.github ? collected.github.starsCount : 0) + (collected.npm ? collected.npm.starsCount : 0);
const forksCount = collected.github ? collected.github.forksCount : 0;
const subscribersCount = collected.github ? collected.github.subscribersCount : 0;
const contributorsCount = collected.github ? (collected.github.contributors || []).length : 0;
return starsCount + forksCount + subscribersCount + contributorsCount;
}
// ----------------------------------------------------------------------------
/**
* Evaluates the package popularity.
*
* @param {Object} collected - The collected information.
*
* @returns {Object} The evaluation result.
*/
function popularity(collected) {
return {
communityInterest: evaluateCommunityInterest(collected),
downloadsCount: evaluateDownloadsCount(collected),
downloadsAcceleration: evaluateDownloadsAcceleration(collected),
dependentsCount: collected.npm ? collected.npm.dependentsCount || 0 : 0,
};
}
module.exports = popularity;
================================================
FILE: lib/analyze/evaluate/quality.js
================================================
/* eslint no-nested-ternary: 0 */
'use strict';
const url = require('url');
const semver = require('semver');
const get = require('lodash/get');
const normalizeValue = require('normalize-value');
const log = logger.child({ module: 'evaluate/quality' });
/**
* Evaluates the author's carefulness with the package.
* It evaluates the basics of a package, such as the README, license, stability, etc.
*
* @param {Object} collected - The collected information.
*
* @returns {Number} The carefulness evaluation (from 0 to 1).
*/
function evaluateCarefulness(collected) {
const licenseEvaluation = Number(!!collected.metadata.license);
const readmeEvaluation = normalizeValue(get(collected, 'source.files.readmeSize', 0), [
{ value: 0, norm: 0 },
{ value: 400, norm: 1 },
]);
const lintersEvaluation = Number(!!get(collected, 'source.linters', null));
const ignoreEvaluation = Number(get(collected, 'source.files.hasNpmIgnore') || collected.metadata.hasSelectiveFiles || false);
const changelogEvaluation = Number(get(collected, 'source.files.hasChangelog', false));
const isDeprecated = !!collected.metadata.deprecated;
const isStable = semver.gte(collected.metadata.version, '1.0.0', true); // `true` = loose semver
const finalWeightConditioning = isDeprecated ? 0 : (!isStable ? 0.5 : 1);
return (
(licenseEvaluation * 0.33) +
(readmeEvaluation * 0.38) +
(lintersEvaluation * 0.13) +
(ignoreEvaluation * 0.08) +
(changelogEvaluation * 0.08)
) * finalWeightConditioning;
}
/**
* Evaluates the package tests.
* Takes into the consideration the tests size, coverage % and build status.
*
* @param {Object} collected - The collected information.
*
* @returns {Number} The tests evaluation (from 0 to 1).
*/
function evaluateTests(collected) {
if (!collected.source) {
return 0;
}
const testsEvaluation = normalizeValue(collected.source.files.testsSize, [
{ value: 0, norm: 0 },
{ value: 400, norm: collected.metadata.hasTestScript ? 1 : 0.5 },
]);
const coverageEvaluation = collected.source.coverage || 0;
const statusEvaluation = ((collected.github && collected.github.statuses) || [])
.reduce((sum, status, index, arr) => {
switch (status.state) {
case 'success':
return sum + (1 / arr.length);
case 'pending':
return sum + (0.3 / arr.length);
case 'error':
case 'failure':
return sum;
default:
log.warn(`Unknown github status state: ${status}`);
return sum;
}
}, 0);
return (testsEvaluation * 0.6) +
(statusEvaluation * 0.25) +
(coverageEvaluation * 0.15);
}
/**
* Evaluates the package health.
* Takes into consideration vulnerabilities, outdated dependencies and unlocked dependencies (ones with * or >= 0.0.0).
*
* @param {Object} collected - The collected information.
*
* @returns {Number} The dependencies health evaluation (from 0 to 1).
*/
function evaluateHealth(collected) {
if (!collected.source) {
return 0;
}
const dependencies = collected.metadata.dependencies || {};
const dependenciesCount = Object.keys(dependencies).length;
if (!dependenciesCount) {
return 1;
}
// Calculate outdated count
const outdatedCount = collected.source.outdatedDependencies ?
Object.keys(collected.source.outdatedDependencies).length :
(collected.source.outdatedDependencies === false ? dependenciesCount : 0);
// Calculate vulnerabilities count
const vulnerabilitiesCount = collected.source.vulnerabilities ?
collected.source.vulnerabilities.length :
(collected.source.vulnerabilities === false ? dependenciesCount : 0);
// Calculate unlocked count - packages that have loose locking of versions, e.g.: '*' or >= 1.6.0
// Note that if the package has npm-shrinkwrap.json, then it actually has its versions locked down
const unlockedCount = collected.source.files.hasShrinkwrap ? 0 :
Object.values(dependencies).reduce((count, value) => {
const range = semver.validRange(value, true);
return range && !semver.gtr('1000000.0.0', range, true) ? count + 1 : count;
}, 0);
const outdatedEvaluation = normalizeValue(outdatedCount, [
{ value: 0, norm: 1 },
{ value: Math.max(2, dependenciesCount / 4), norm: 0 },
]);
const vulnerabilitiesEvaluation = normalizeValue(vulnerabilitiesCount, [
{ value: 0, norm: 1 },
{ value: Math.max(2, dependenciesCount / 4), norm: 0 },
]);
const finalWeightConditioning = !unlockedCount ? 1 : 1 / (unlockedCount + 1);
return (
(outdatedEvaluation * 0.5) +
(vulnerabilitiesEvaluation * 0.5)
) * finalWeightConditioning;
}
/**
* Evaluates the package branding.
* Takes into consideration if the package has badges, custom homepage, etc.
*
* @param {Object} collected - The collected information.
*
* @returns {Number} The branding evaluation (from 0 to 1).
*/
function evaluateBranding(collected) {
const parsedRepository = url.parse(get(collected.metadata, 'repository.url', ''));
const parsedHomepage = url.parse(get(collected.metadata, 'links.homepage', get(collected, 'github.homepage', '')));
const hasCustomHomepage = !!(parsedRepository.host && parsedHomepage.host &&
parsedRepository.host !== parsedHomepage.host);
const badgesCount = get(collected, 'source.badges.length', 0);
const homepageEvaluation = Number(hasCustomHomepage);
const badgesEvaluation = normalizeValue(badgesCount, [
{ value: 0, norm: 0 },
{ value: 4, norm: 1 },
]);
return (homepageEvaluation * 0.4) +
(badgesEvaluation * 0.6);
}
// ----------------------------------------------------------------------------
/**
* Evaluates the package quality.
*
* @param {Object} collected - The collected information.
*
* @returns {Object} The evaluation result.
*/
function quality(collected) {
return {
carefulness: evaluateCarefulness(collected),
tests: evaluateTests(collected),
health: evaluateHealth(collected),
branding: evaluateBranding(collected),
};
}
module.exports = quality;
================================================
FILE: lib/analyze/index.js
================================================
'use strict';
const promiseRetry = require('promise-retry');
const serializeError = require('serialize-error');
const omit = require('lodash/omit');
const collect = require('./collect');
const evaluate = require('./evaluate');
const download = require('./download');
const exec = require('./util/exec');
const packageJsonFromData = require('./util/packageJsonFromData');
const log = logger.child({ module: 'analyze' });
/**
* Gets a package analysis.
*
* @param {String} name - The package name.
* @param {Nano} npmsNano - The client nano instance for npms.
*
* @returns {Promise} The promise that fulfills when done.
*/
function get(name, npmsNano) {
return npmsNano.getAsync(`package!${name}`)
.catch({ error: 'not_found' }, () => {
throw Object.assign(new Error(`Analysis for package ${name} does not exist`), { code: 'ANALYSIS_NOT_FOUND' });
});
}
/**
* Removes a package analysis.
*
* @param {String} name - The package name.
* @param {Nano} npmsNano - The client nano instance for npms.
*
* @returns {Promise} The promise that fulfills when done.
*/
function remove(name, npmsNano) {
return promiseRetry((retry) => (
get(name, npmsNano)
.then((doc) => (
npmsNano.destroyAsync(doc._id, doc._rev)
.catch({ error: 'conflict' }, (err) => {
err = new Error(`Conflict while removing ${name} analysis`);
log.warn({ err }, err.message);
retry(err);
})
))
))
.catch({ code: 'ANALYSIS_NOT_FOUND' }, () => {})
.then(() => log.trace(`Removed analysis of ${name}`));
}
/**
* Saves a package analysis.
* Contains the collected info and the evaluation result.
*
* @param {Object} analysis - The analysis (can be the full doc to avoid having to fetch it).
* @param {Nano} npmsNano - The client nano instance for npms.
*
* @returns {Promise} The promise that fulfills when done.
*/
function save(analysis, npmsNano) {
const name = analysis.collected.metadata.name;
return promiseRetry((retry) =>
// Fetch the doc if necessary to obtain its rev
Promise.try(() => {
if (analysis._rev) {
return;
}
return get(name, npmsNano)
.then((doc) => { analysis._rev = doc._rev; })
.catch({ code: 'ANALYSIS_NOT_FOUND' }, () => {});
})
// Save it
.then(() => {
analysis._id = `package!${name}`;
retur
gitextract_mgcbttt5/
├── .editorconfig
├── .eslintignore
├── .eslintrc.json
├── .gitignore
├── .travis.yml
├── LICENSE
├── README.md
├── cli.js
├── cmd/
│ ├── consume.js
│ ├── observe.js
│ ├── scoring.js
│ ├── tasks/
│ │ ├── check-gh-tokens.js
│ │ ├── clean-extraneous.js
│ │ ├── enqueue-missing.js
│ │ ├── enqueue-outdated.js
│ │ ├── enqueue-view.js
│ │ ├── migrate.js
│ │ ├── process-package.js
│ │ ├── re-evaluate.js
│ │ └── re-metadata.js
│ ├── tasks.js
│ └── util/
│ ├── bootstrap.js
│ └── stats/
│ ├── index.js
│ ├── process.js
│ ├── progress.js
│ ├── queue.js
│ └── tokens.js
├── config/
│ ├── couchdb/
│ │ ├── npms-analyzer-npm.json
│ │ └── npms-analyzer-npms.json
│ ├── default.json5
│ └── elasticsearch/
│ └── npms.json5
├── docs/
│ ├── architecture.md
│ ├── deploys.md
│ ├── diagrams/
│ │ ├── analysis.xml
│ │ └── continuous-scoring.xml
│ └── setup.md
├── ecosystem.json5
├── lib/
│ ├── analyze/
│ │ ├── collect/
│ │ │ ├── bin/
│ │ │ │ └── david-json
│ │ │ ├── github.js
│ │ │ ├── index.js
│ │ │ ├── metadata.js
│ │ │ ├── npm.js
│ │ │ ├── source.js
│ │ │ └── util/
│ │ │ ├── fileContents.js
│ │ │ ├── fileSize.js
│ │ │ ├── pointsToRanges.js
│ │ │ └── promisePropsSettled.js
│ │ ├── download/
│ │ │ ├── git.js
│ │ │ ├── github.js
│ │ │ ├── index.js
│ │ │ ├── npm.js
│ │ │ └── util/
│ │ │ ├── assertFilesCount.js
│ │ │ ├── findPackageDir.js
│ │ │ ├── mergePackageJson.js
│ │ │ └── untar.js
│ │ ├── evaluate/
│ │ │ ├── index.js
│ │ │ ├── maintenance.js
│ │ │ ├── popularity.js
│ │ │ └── quality.js
│ │ ├── index.js
│ │ └── util/
│ │ ├── exec.js
│ │ ├── gotRetry.js
│ │ ├── hostedGitInfo.js
│ │ ├── normalizePackageJson.js
│ │ └── packageJsonFromData.js
│ ├── configure.js
│ ├── observers/
│ │ ├── realtime.js
│ │ └── stale.js
│ ├── queue.js
│ └── scoring/
│ ├── aggregate.js
│ ├── finalize.js
│ ├── prepare.js
│ ├── score.js
│ └── util/
│ └── paperNumerical.js
├── package.json
└── test/
├── .eslintrc.json
├── bin/
│ └── download-fixtures
├── fixtures/
│ └── analyze/
│ ├── collect/
│ │ ├── modules/
│ │ │ ├── 0/
│ │ │ │ ├── data.json
│ │ │ │ └── expected-source.json
│ │ │ ├── @bcoe%2fexpress-oauth-server/
│ │ │ │ ├── data.json
│ │ │ │ └── expected-metadata.json
│ │ │ ├── babel-jest/
│ │ │ │ ├── data.json
│ │ │ │ └── expected-source.json
│ │ │ ├── backoff/
│ │ │ │ ├── data.json
│ │ │ │ └── expected-source.json
│ │ │ ├── cross-spawn/
│ │ │ │ ├── data.json
│ │ │ │ ├── expected-github.json
│ │ │ │ ├── expected-metadata.json
│ │ │ │ ├── expected-npm.json
│ │ │ │ └── expected-source.json
│ │ │ ├── hapi/
│ │ │ │ ├── data.json
│ │ │ │ └── expected-source.json
│ │ │ ├── planify/
│ │ │ │ ├── data.json
│ │ │ │ └── expected-source.json
│ │ │ ├── react/
│ │ │ │ ├── data.json
│ │ │ │ └── expected-source.json
│ │ │ └── react-router/
│ │ │ ├── data.json
│ │ │ └── expected-source.json
│ │ └── recorded/
│ │ ├── github/
│ │ │ ├── 0bfbe2f1c03ff5ed9c3baa91d588e218
│ │ │ ├── 0bfbe2f1c03ff5ed9c3baa91d588e218.headers
│ │ │ ├── 2236c266c85b15946d7ca69cc2e1e091
│ │ │ ├── 2236c266c85b15946d7ca69cc2e1e091.headers
│ │ │ ├── 24d4b4797edc40614848f01802bbe2b3
│ │ │ ├── 24d4b4797edc40614848f01802bbe2b3.headers
│ │ │ ├── 2f87db1cf50593ec3f80835f624ec88a
│ │ │ ├── 2f87db1cf50593ec3f80835f624ec88a.headers
│ │ │ ├── 3a1a79735cab3e2c46da0f739eccd595
│ │ │ ├── 3a1a79735cab3e2c46da0f739eccd595.headers
│ │ │ ├── 3b5e34c45a594730608f6170cacf31fe
│ │ │ ├── 3b5e34c45a594730608f6170cacf31fe.headers
│ │ │ ├── 3d15804db16c597c23a14946a78b8e1b
│ │ │ ├── 3d15804db16c597c23a14946a78b8e1b.headers
│ │ │ ├── 4109ed740591855f9e48eb868f40db86
│ │ │ ├── 4109ed740591855f9e48eb868f40db86.headers
│ │ │ ├── 5bdd0f1e3c86f0114eb714a2ce79e905
│ │ │ ├── 5bdd0f1e3c86f0114eb714a2ce79e905.headers
│ │ │ ├── 6913bcb008bea8f6a0384da9bbae2293
│ │ │ ├── 6913bcb008bea8f6a0384da9bbae2293.headers
│ │ │ ├── 85c5e5ee4a806e7d405984c325f29007
│ │ │ ├── 85c5e5ee4a806e7d405984c325f29007.headers
│ │ │ ├── 92a4188d8af9e8c1ff665859b3cd86b8
│ │ │ ├── 92a4188d8af9e8c1ff665859b3cd86b8.headers
│ │ │ ├── 9ccf24e28c94543e4ae601d5aa9c8cba
│ │ │ ├── 9ccf24e28c94543e4ae601d5aa9c8cba.headers
│ │ │ ├── a3360f53aa71342ad67cded65f6ea1da
│ │ │ ├── a3360f53aa71342ad67cded65f6ea1da.headers
│ │ │ ├── a7d2cca72c7267fd27fe769cb4d2f611
│ │ │ ├── a7d2cca72c7267fd27fe769cb4d2f611.headers
│ │ │ ├── afcd15ede3deaa855315f5a1fbc3e61d
│ │ │ ├── afcd15ede3deaa855315f5a1fbc3e61d.headers
│ │ │ ├── b32671b71119c7fc156f3aa050d1cb12
│ │ │ ├── b32671b71119c7fc156f3aa050d1cb12.headers
│ │ │ ├── c23e4492fdaefde5fe60524fab308532
│ │ │ ├── c23e4492fdaefde5fe60524fab308532.headers
│ │ │ ├── c389930a56f1bad9257ed1490fc32c9b
│ │ │ ├── c389930a56f1bad9257ed1490fc32c9b.headers
│ │ │ ├── cc88ef857a3a1492913c066047c5c033
│ │ │ ├── cc88ef857a3a1492913c066047c5c033.headers
│ │ │ ├── d3b7e3ec7ad3c841c45ff000fd77b711
│ │ │ ├── d3b7e3ec7ad3c841c45ff000fd77b711.headers
│ │ │ ├── d495e09987382290004f52a8fa39243b
│ │ │ ├── d495e09987382290004f52a8fa39243b.headers
│ │ │ ├── dadb0a8973a79e019a2a0affeb248deb
│ │ │ ├── dadb0a8973a79e019a2a0affeb248deb.headers
│ │ │ ├── ebd826ea6dcd2abd0dcc961dd0fe4176
│ │ │ ├── ebd826ea6dcd2abd0dcc961dd0fe4176.headers
│ │ │ ├── ede6348d94fd4d537bcc1e42b050d8ed
│ │ │ ├── ede6348d94fd4d537bcc1e42b050d8ed.headers
│ │ │ ├── ff4470aede5dba39b4736419dda38443
│ │ │ └── ff4470aede5dba39b4736419dda38443.headers
│ │ ├── index/
│ │ │ ├── 474829c21c2e69d2c0d889af2a714584
│ │ │ ├── 474829c21c2e69d2c0d889af2a714584.headers
│ │ │ ├── ae18615611dfb3f32feaf1c607df7bac
│ │ │ ├── ae18615611dfb3f32feaf1c607df7bac.headers
│ │ │ ├── d40cbb49129f01a9d5130a95f54d4f79
│ │ │ └── d40cbb49129f01a9d5130a95f54d4f79.headers
│ │ ├── npm/
│ │ │ ├── 278e742ec691d9647761d9e06a93c852
│ │ │ ├── 278e742ec691d9647761d9e06a93c852.headers
│ │ │ ├── 6da574a19e30e15a2628bc2a7ae7d5a4
│ │ │ └── 6da574a19e30e15a2628bc2a7ae7d5a4.headers
│ │ └── source/
│ │ ├── 0046732f19ee23072e08f55d2a400eca
│ │ ├── 0046732f19ee23072e08f55d2a400eca.headers
│ │ ├── 02777f766910df6791475f44c0e2b57b
│ │ ├── 02777f766910df6791475f44c0e2b57b.headers
│ │ ├── 0429ac4bdf217161d9a2772fdb7861e2
│ │ ├── 0429ac4bdf217161d9a2772fdb7861e2.headers
│ │ ├── 0c774ab0c1fa72fb9dbba94b96693973
│ │ ├── 0c774ab0c1fa72fb9dbba94b96693973.headers
│ │ ├── 1152654e8bfc034a9d043925c55fbe48
│ │ ├── 1152654e8bfc034a9d043925c55fbe48.headers
│ │ ├── 180995f905c69e6355ccbeb197109fb9
│ │ ├── 180995f905c69e6355ccbeb197109fb9.headers
│ │ ├── 1b4935ddf796087a37e45c313edddd4f
│ │ ├── 1b4935ddf796087a37e45c313edddd4f.headers
│ │ ├── 310c26a3622e22be3798b810bb056cd2
│ │ ├── 310c26a3622e22be3798b810bb056cd2.headers
│ │ ├── 32532aa076ea4d37a94def3f370e23fc
│ │ ├── 32532aa076ea4d37a94def3f370e23fc.headers
│ │ ├── 36d2b9e3113bb8656477a0866759fca3
│ │ ├── 36d2b9e3113bb8656477a0866759fca3.headers
│ │ ├── 3f5a8e5d9434bbd4ec976c36835cdc49
│ │ ├── 3f5a8e5d9434bbd4ec976c36835cdc49.headers
│ │ ├── 40971c651c26e9ab81128eb0c10f37af
│ │ ├── 40971c651c26e9ab81128eb0c10f37af.headers
│ │ ├── 44c08d748b72275a181e7820e9c258e8
│ │ ├── 44c08d748b72275a181e7820e9c258e8.headers
│ │ ├── 4d7966a6e7249722abdff0a7555a2527
│ │ ├── 4d7966a6e7249722abdff0a7555a2527.headers
│ │ ├── 4ece6fa3645c526294eaf1b270113e6d
│ │ ├── 4ece6fa3645c526294eaf1b270113e6d.headers
│ │ ├── 50ec845be323513dc05d4ce6eeb56639
│ │ ├── 50ec845be323513dc05d4ce6eeb56639.headers
│ │ ├── 586e879d6364ca5313dd5f956d47dbd4
│ │ ├── 586e879d6364ca5313dd5f956d47dbd4.headers
│ │ ├── 5fcfa736ff5a936752226c33baab7ce5
│ │ ├── 5fcfa736ff5a936752226c33baab7ce5.headers
│ │ ├── 6566d9e3adefb1ed8fe60a17bbf13133
│ │ ├── 6566d9e3adefb1ed8fe60a17bbf13133.headers
│ │ ├── 6632454a445cfcd8152c30b4b8c64783
│ │ ├── 6632454a445cfcd8152c30b4b8c64783.headers
│ │ ├── 672a67ca0e6ca2125e9601f4e532dc2e
│ │ ├── 672a67ca0e6ca2125e9601f4e532dc2e.headers
│ │ ├── 6ef5fd0be1ac70c0cc78c63dc72f97da
│ │ ├── 6ef5fd0be1ac70c0cc78c63dc72f97da.headers
│ │ ├── 6f6f60463501ffe7964700fdb5262ea7
│ │ ├── 6f6f60463501ffe7964700fdb5262ea7.headers
│ │ ├── 77c0ab2484ae068cadf90515cd2bb6d4
│ │ ├── 77c0ab2484ae068cadf90515cd2bb6d4.headers
│ │ ├── 7fb20e04d9d456482d62b369a1c268c0
│ │ ├── 7fb20e04d9d456482d62b369a1c268c0.headers
│ │ ├── 833f1d757cdb7cfa152b54680d0d2d73
│ │ ├── 833f1d757cdb7cfa152b54680d0d2d73.headers
│ │ ├── 8881281d102f7688a2b0e5d7ffb48299
│ │ ├── 8881281d102f7688a2b0e5d7ffb48299.headers
│ │ ├── 8acf97225f349b2c99978025ba5b8e92
│ │ ├── 8acf97225f349b2c99978025ba5b8e92.headers
│ │ ├── 8dcbc0b25ce4f37fd5c0bc06d633eb52
│ │ ├── 8dcbc0b25ce4f37fd5c0bc06d633eb52.headers
│ │ ├── 92d4837e4094c3e096f398bc89aabb0f
│ │ ├── 92d4837e4094c3e096f398bc89aabb0f.headers
│ │ ├── 981d42650123f10d1600074d65aa4f43
│ │ ├── 981d42650123f10d1600074d65aa4f43.headers
│ │ ├── 9a7bfe12b0910e8bd69ed06184030276
│ │ ├── 9a7bfe12b0910e8bd69ed06184030276.headers
│ │ ├── a27623f8433973cd8a3c9ac32782dd9b
│ │ ├── a27623f8433973cd8a3c9ac32782dd9b.headers
│ │ ├── a9be16a47e9b4e3762bfbcfbec14effd
│ │ ├── a9be16a47e9b4e3762bfbcfbec14effd.headers
│ │ ├── acd2d3271b8cd130cd75e572ada409c1
│ │ ├── acd2d3271b8cd130cd75e572ada409c1.headers
│ │ ├── afd6397af26789ff7e024c758e094e02
│ │ ├── afd6397af26789ff7e024c758e094e02.headers
│ │ ├── b00ce31e18a32896ac83d6819e9816fe
│ │ ├── b00ce31e18a32896ac83d6819e9816fe.headers
│ │ ├── b6d2435e45a7f8f3b88152e577c55b84
│ │ ├── b6d2435e45a7f8f3b88152e577c55b84.headers
│ │ ├── bb391b38e8f2529e20e3e313a7875566
│ │ ├── bb391b38e8f2529e20e3e313a7875566.headers
│ │ ├── bbd9372c326ea4fb4acc82bd30e9491c
│ │ ├── bbd9372c326ea4fb4acc82bd30e9491c.headers
│ │ ├── c2a263604b39c741dcba960ab7bdf64e
│ │ ├── c2a263604b39c741dcba960ab7bdf64e.headers
│ │ ├── ca1b5b9f76e662b613be43d64e92c4b4
│ │ ├── ca1b5b9f76e662b613be43d64e92c4b4.headers
│ │ ├── d4bf4a43cf4e7b6c27e40c732c9d8bfa
│ │ ├── d4bf4a43cf4e7b6c27e40c732c9d8bfa.headers
│ │ ├── daaa6494600d82aa3e21e56452a8702a
│ │ ├── daaa6494600d82aa3e21e56452a8702a.headers
│ │ ├── e2d10b245b6bce60976eb41c755c5333
│ │ ├── e2d10b245b6bce60976eb41c755c5333.headers
│ │ ├── e66bf57e7754e3a75c0b3da3c7d3b894
│ │ ├── e66bf57e7754e3a75c0b3da3c7d3b894.headers
│ │ ├── f13ef1d336f7343f21a8f1e755ee4a1d
│ │ ├── f13ef1d336f7343f21a8f1e755ee4a1d.headers
│ │ ├── f4fbc0e6ea0806cdebfe05e95480858f
│ │ ├── f4fbc0e6ea0806cdebfe05e95480858f.headers
│ │ ├── fba7a93bdb3f483048d32ccc0a105e2b
│ │ ├── fba7a93bdb3f483048d32ccc0a105e2b.headers
│ │ ├── febf07de22a7d2d7ffe02742c6b81857
│ │ └── febf07de22a7d2d7ffe02742c6b81857.headers
│ └── download/
│ ├── mocked/
│ │ ├── broken-archive.tgz
│ │ └── non-gzip-archive.tgz
│ └── recorded/
│ ├── github/
│ │ ├── 0ca08d9404d3be6b0f4b710e7dce325c
│ │ ├── 0ca08d9404d3be6b0f4b710e7dce325c.headers
│ │ ├── 0d6bf2e4d590ee9d6ece01c851500563
│ │ ├── 0d6bf2e4d590ee9d6ece01c851500563.headers
│ │ ├── 2197e3675f7b1189860675d45228c712
│ │ ├── 2197e3675f7b1189860675d45228c712.headers
│ │ ├── 324da49a49b1bf9799ad0af735d42175
│ │ ├── 324da49a49b1bf9799ad0af735d42175.headers
│ │ ├── 347ad9c22b702976e2e6304bee584cd2
│ │ ├── 347ad9c22b702976e2e6304bee584cd2.headers
│ │ ├── 39c4db447b629049bd6c82625ecbb182
│ │ ├── 39c4db447b629049bd6c82625ecbb182.headers
│ │ ├── 598868e39b0c5f898b243dc6a7799590
│ │ ├── 598868e39b0c5f898b243dc6a7799590.headers
│ │ ├── 6b4c11cf7f1c30a3c0f45d6dee2c98c8
│ │ ├── 6b4c11cf7f1c30a3c0f45d6dee2c98c8.headers
│ │ ├── ce4fa3241f0364d1ea4e654f3cf13cb9
│ │ ├── ce4fa3241f0364d1ea4e654f3cf13cb9.headers
│ │ ├── d7c449b25de454b1b362bc5af32cc777
│ │ └── d7c449b25de454b1b362bc5af32cc777.headers
│ └── npm/
│ ├── 57f54040bdda5ac6ffd196cac24be2d8
│ ├── 57f54040bdda5ac6ffd196cac24be2d8.headers
│ ├── afed24ca0f8e9f12344ae6a851b46159
│ └── afed24ca0f8e9f12344ae6a851b46159.headers
├── mocha.opts
├── spec/
│ └── analyze/
│ ├── collect/
│ │ ├── github.js
│ │ ├── index.js
│ │ ├── metadata.js
│ │ ├── npm.js
│ │ ├── source.js
│ │ └── util/
│ │ ├── fileContents.js
│ │ ├── fileSize.js
│ │ ├── pointsToRanges.js
│ │ └── promisePropsSettled.js
│ ├── download/
│ │ ├── git.js
│ │ ├── github.js
│ │ ├── index.js
│ │ ├── npm.js
│ │ └── util/
│ │ ├── findPackageDir.js
│ │ ├── mergePackageJson.js
│ │ └── untar.js
│ ├── evaluate/
│ │ ├── index.js
│ │ ├── maintenance.js
│ │ ├── popularity.js
│ │ └── quality.js
│ └── util/
│ ├── exec.js
│ ├── gotRetry.js
│ ├── hostedGitInfo.js
│ ├── normalizePackageJson.js
│ └── packageJsonFromData.js
├── test.js
└── util/
└── sepia.js
SYMBOL INDEX (173 symbols across 50 files)
FILE: cmd/consume.js
function onMessage (line 26) | function onMessage(msg, npmNano, npmsNano, esClient) {
function onFailedAnalysis (line 68) | function onFailedAnalysis(name, err, npmsNano, esClient) {
FILE: cmd/observe.js
function onPackage (line 23) | function onPackage(name, priority, queue) {
FILE: cmd/scoring.js
function waitRemaining (line 22) | function waitRemaining(delay, esClient) {
function cycle (line 46) | function cycle(delay, npmsNano, esClient) {
FILE: cmd/tasks/clean-extraneous.js
function fetchNpmPackages (line 15) | function fetchNpmPackages(npmNano) {
function fetchNpmsPackages (line 33) | function fetchNpmsPackages(npmsNano) {
function fetchNpmsObservedPackages (line 53) | function fetchNpmsObservedPackages(npmsNano) {
function cleanExtraneousNpmsPackages (line 76) | function cleanExtraneousNpmsPackages(npmPackages, npmsPackages, npmsNano...
function cleanExtraneousNpmsObservedPackages (line 122) | function cleanExtraneousNpmsObservedPackages(npmPackages, npmsObservedPa...
FILE: cmd/tasks/enqueue-missing.js
function fetchNpmPackages (line 16) | function fetchNpmPackages(npmNano) {
function fetchNpmsPackages (line 34) | function fetchNpmsPackages(npmsNano) {
function enqueueMissingPackages (line 51) | function enqueueMissingPackages(npmPackages, npmsPackages, queue, dryRun) {
FILE: cmd/tasks/enqueue-outdated.js
function fetchNpmPackages (line 15) | function fetchNpmPackages(npmNano) {
function fetchNpmsPackages (line 32) | function fetchNpmsPackages(npmsNano) {
function enqueueOutdated (line 52) | function enqueueOutdated(npmPackages, npmsPackages, queue, dryRun) {
FILE: cmd/tasks/enqueue-view.js
function fetchView (line 17) | function fetchView(view, npmNano) {
function enqueueViewPackages (line 38) | function enqueueViewPackages(packages, queue, dryRun) {
FILE: cmd/tasks/migrate.js
function extractScope (line 10) | function extractScope(name) {
FILE: cmd/util/bootstrap.js
function bootstrap (line 22) | function bootstrap(deps, options) {
function bootstrapCouchdb (line 58) | function bootstrapCouchdb(config, options) {
function bootstrapElasticsearch (line 87) | function bootstrapElasticsearch(config, options) {
function bootstrapQueue (line 115) | function bootstrapQueue(config, options) {
FILE: cmd/util/stats/process.js
function statProcess (line 11) | function statProcess() {
FILE: cmd/util/stats/progress.js
function statProgress (line 15) | function statProgress(npmNano, npmsNano) {
FILE: cmd/util/stats/queue.js
function statQueue (line 12) | function statQueue(queue) {
FILE: cmd/util/stats/tokens.js
function statTokens (line 15) | function statTokens(tokens, group) {
FILE: lib/analyze/collect/github.js
function extractCommits (line 26) | function extractCommits(commitActivity) {
function githubRequest (line 47) | function githubRequest(resource, options) {
function fetchIssuesStats (line 123) | function fetchIssuesStats(repository, options) {
function github (line 170) | function github(packageJson, downloaded, options) {
FILE: lib/analyze/collect/index.js
function checkRepositoryOwnership (line 24) | function checkRepositoryOwnership(data, packageJson, downloaded, npmNano) {
function empty (line 65) | function empty(name) {
function collect (line 82) | function collect(data, packageJson, downloaded, npmNano, options) {
FILE: lib/analyze/collect/metadata.js
function extractReleases (line 26) | function extractReleases(data) {
function normalizeLicense (line 48) | function normalizeLicense(name, license) {
function extractLicense (line 87) | function extractLicense(packageJson) {
function extractLinks (line 118) | function extractLinks(packageJson) {
function extractAuthor (line 176) | function extractAuthor(packageJson, maintainers) {
function extractPublisher (line 200) | function extractPublisher(packageJson, maintainers) {
function extractScope (line 223) | function extractScope(packageJson) {
function extractMaintainers (line 242) | function extractMaintainers(data, packageJson) {
function empty (line 266) | function empty(name) {
function metadata (line 284) | function metadata(data, packageJson) {
FILE: lib/analyze/collect/npm.js
function fetchDownloads (line 21) | function fetchDownloads(name) {
function extractStarsCount (line 99) | function extractStarsCount(data) {
function npm (line 114) | function npm(data, packageJson) {
FILE: lib/analyze/collect/source.js
function inspectFiles (line 33) | function inspectFiles(data, downloaded) {
function getReadmeBadges (line 72) | function getReadmeBadges(data, downloaded) {
function getRepoLinters (line 93) | function getRepoLinters(downloaded) {
function fetchCodeCoverage (line 120) | function fetchCodeCoverage(packageJson, badges) {
function checkVulnerabilities (line 154) | function checkVulnerabilities(packageJson) {
function checkOutdatedDeps (line 202) | function checkOutdatedDeps(name, dir, options) {
function source (line 261) | function source(data, packageJson, downloaded, options) {
FILE: lib/analyze/collect/util/fileContents.js
function fileContents (line 14) | function fileContents(path) {
FILE: lib/analyze/collect/util/fileSize.js
function fileSize (line 15) | function fileSize(path) {
function fileSizeDir (line 48) | function fileSizeDir(dir) {
FILE: lib/analyze/collect/util/pointsToRanges.js
function pointsToRanges (line 15) | function pointsToRanges(points, buckets) {
function bucketsFromBreakpoints (line 35) | function bucketsFromBreakpoints(breakpoints) {
FILE: lib/analyze/collect/util/promisePropsSettled.js
function promisePropsSettled (line 13) | function promisePropsSettled(object) {
FILE: lib/analyze/download/git.js
function download (line 22) | function download(url, ref, tmpDir, options) {
function getCloneUrl (line 77) | function getCloneUrl(gitInfo) {
function git (line 103) | function git(packageJson, options) {
FILE: lib/analyze/download/github.js
function download (line 26) | function download(shorthand, ref, tmpDir, options) {
function github (line 136) | function github(packageJson, options) {
FILE: lib/analyze/download/index.js
function getRandomStr (line 21) | function getRandomStr() {
function createTmpDir (line 34) | function createTmpDir(name) {
function cleanTmpDir (line 49) | function cleanTmpDir() {
function download (line 68) | function download(packageJson, options) {
FILE: lib/analyze/download/npm.js
function download (line 22) | function download(target, url, tmpDir, options) {
function npm (line 80) | function npm(packageJson, options) {
FILE: lib/analyze/download/util/assertFilesCount.js
function assertFilesCount (line 16) | function assertFilesCount(dir, maxFiles) {
FILE: lib/analyze/download/util/findPackageDir.js
function lookForPackageDir (line 17) | function lookForPackageDir(packageJson, dir) {
function isSamePackage (line 53) | function isSamePackage(packageJson, dir) {
function findPackageDir (line 85) | function findPackageDir(packageJson, dir) {
FILE: lib/analyze/download/util/mergePackageJson.js
function mergePackageJson (line 26) | function mergePackageJson(packageJson, packageDir) {
FILE: lib/analyze/download/util/untar.js
function assertFilesCount (line 23) | function assertFilesCount(file, maxFiles) {
function decompress (line 59) | function decompress(file, destDir) {
function untar (line 88) | function untar(file, options) {
FILE: lib/analyze/evaluate/index.js
function evaluate (line 12) | function evaluate(collected) {
FILE: lib/analyze/evaluate/maintenance.js
function evaluateReleasesFrequency (line 19) | function evaluateReleasesFrequency(collected) {
function evaluateCommitsFrequency (line 60) | function evaluateCommitsFrequency(collected) {
function evaluateOpenIssues (line 98) | function evaluateOpenIssues(collected) {
function evaluateIssuesDistribution (line 133) | function evaluateIssuesDistribution(collected) {
function isPackageFinished (line 182) | function isPackageFinished(collected) {
function maintenance (line 206) | function maintenance(collected) {
FILE: lib/analyze/evaluate/popularity.js
function evaluateDownloadsCount (line 13) | function evaluateDownloadsCount(collected) {
function evaluateDownloadsAcceleration (line 39) | function evaluateDownloadsAcceleration(collected) {
function evaluateCommunityInterest (line 72) | function evaluateCommunityInterest(collected) {
function popularity (line 90) | function popularity(collected) {
FILE: lib/analyze/evaluate/quality.js
function evaluateCarefulness (line 20) | function evaluateCarefulness(collected) {
function evaluateTests (line 51) | function evaluateTests(collected) {
function evaluateHealth (line 91) | function evaluateHealth(collected) {
function evaluateBranding (line 147) | function evaluateBranding(collected) {
function quality (line 173) | function quality(collected) {
FILE: lib/analyze/index.js
function get (line 22) | function get(name, npmsNano) {
function remove (line 37) | function remove(name, npmsNano) {
function save (line 62) | function save(analysis, npmsNano) {
function saveFailed (line 104) | function saveFailed(name, err, npmsNano) {
function analyze (line 139) | function analyze(name, npmNano, npmsNano, options) {
FILE: lib/analyze/util/exec.js
function escape (line 14) | function escape(pieces, ...substitutions) {
function exec (line 32) | function exec(command, options) {
FILE: lib/analyze/util/hostedGitInfo.js
function hostedGitInfo (line 15) | function hostedGitInfo(repositoryUrl) {
FILE: lib/analyze/util/normalizePackageJson.js
function removePathFromRepositoryUrl (line 17) | function removePathFromRepositoryUrl(url) {
function normalizePackageJson (line 34) | function normalizePackageJson(packageJson) {
FILE: lib/analyze/util/packageJsonFromData.js
function packageJsonFromData (line 15) | function packageJsonFromData(name, data) {
FILE: lib/observers/realtime.js
class RealtimeObserver (line 9) | class RealtimeObserver {
method constructor (line 21) | constructor(npmNano, npmsNano, onPackage, options) {
method destroy (line 43) | destroy() {
method _ignoreIfStopped (line 57) | _ignoreIfStopped(fn) {
method _start (line 73) | _start() {
method _stop (line 98) | _stop() {
method _fetchLastSeq (line 122) | _fetchLastSeq() {
method _updateLastSeq (line 143) | _updateLastSeq(seq) {
method _followChanges (line 169) | _followChanges() {
method _addToBuffer (line 194) | _addToBuffer(change) {
method _flushBuffer (line 225) | _flushBuffer() {
method _filterModified (line 291) | _filterModified(names) {
function realtime (line 327) | function realtime(npmAddr, npmsAddr, onPackage, options) {
FILE: lib/observers/stale.js
class StaleObserver (line 9) | class StaleObserver {
method constructor (line 21) | constructor(npmsNano, onPackage, options) {
method destroy (line 45) | destroy() {
method _ignoreIfStopped (line 59) | _ignoreIfStopped(fn) {
method _start (line 74) | _start() {
method _stop (line 85) | _stop() {
method _check (line 99) | _check() {
method _checkType (line 120) | _checkType(type) {
method _filterNotNotified (line 166) | _filterNotNotified(names, type) {
function stale (line 200) | function stale(npmsAddr, onPackage, options) {
FILE: lib/queue.js
class Queue (line 24) | class Queue extends EventEmitter {
method constructor (line 32) | constructor(name, addr, options) {
method push (line 55) | push(data, priority) {
method consume (line 91) | consume(fn, options) {
method stat (line 126) | stat() {
method destroy (line 139) | destroy() {
method _connect (line 161) | _connect() {
method _reconnect (line 226) | _reconnect(err) {
method _disconnect (line 265) | _disconnect() {
method _registerConsumer (line 290) | _registerConsumer(consumer) {
method _handleConsumerSuccess (line 332) | _handleConsumerSuccess(message, queueMessage) {
method _handleConsumerError (line 356) | _handleConsumerError(err, message, queueMessage) {
method _sendToQueue (line 395) | _sendToQueue(message) {
method _isConnected (line 415) | _isConnected() {
method _assertConnected (line 422) | _assertConnected() {
function queue (line 427) | function queue(name, addr, options) {
FILE: lib/scoring/aggregate.js
function calculateAggregation (line 21) | function calculateAggregation(evaluations) {
function get (line 57) | function get(npmsNano) {
function remove (line 74) | function remove(npmsNano) {
function save (line 98) | function save(aggregation, npmsNano) {
function aggregate (line 137) | function aggregate(npmsNano) {
FILE: lib/scoring/finalize.js
function finalize (line 14) | function finalize(esInfo, esClient) {
FILE: lib/scoring/prepare.js
constant JSON5 (line 4) | const JSON5 = require('json5');
function prepare (line 19) | function prepare(esClient) {
FILE: lib/scoring/score.js
function scoreQuality (line 22) | function scoreQuality(quality, aggregation) {
function scorePopularity (line 46) | function scorePopularity(popularity, aggregation) {
function scoreMaintenance (line 70) | function scoreMaintenance(maintenance, aggregation) {
function calculateScore (line 99) | function calculateScore(value, aggregation, avgY) {
function buildScore (line 128) | function buildScore(analysis, aggregation) {
function getLivingIndices (line 169) | function getLivingIndices(esClient) {
function storeScore (line 188) | function storeScore(score, livingIndices, esClient) {
function get (line 216) | function get(name, esClient) {
function remove (line 235) | function remove(name, esClient) {
function save (line 260) | function save(score, esClient) {
function all (line 279) | function all(aggregation, npmsNano, esClient) {
function score (line 335) | function score(analysis, npmsNano, esClient) {
FILE: lib/scoring/util/paperNumerical.js
function clamp (line 76) | function clamp(value, min, max) {
function getDiscriminant (line 80) | function getDiscriminant(a, b, c) {
function getNormalizationFactor (line 107) | function getNormalizationFactor() {
function evaluate (line 352) | function evaluate(x0) {
FILE: test/spec/analyze/collect/source.js
function mockExternal (line 17) | function mockExternal(mocks, dir) {
FILE: test/spec/analyze/download/git.js
function mock (line 12) | function mock(mocks) {
FILE: test/util/sepia.js
function enable (line 50) | function enable() {
function disable (line 56) | function disable() {
Condensed preview — 314 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (2,430K chars).
[
{
"path": ".editorconfig",
"chars": 220,
"preview": "root = true\n\n[*]\nindent_style = space\nindent_size = 4\nend_of_line = lf\ncharset = utf-8\ntrim_trailing_whitespace = true\ni"
},
{
"path": ".eslintignore",
"chars": 28,
"preview": "/test/coverage\n/dev\n/cli.js\n"
},
{
"path": ".eslintrc.json",
"chars": 168,
"preview": "{\n \"root\": true,\n \"extends\": [\n \"eslint-config-moxy/es8\",\n \"eslint-config-moxy/addons/node\"\n ],\n "
},
{
"path": ".gitignore",
"chars": 101,
"preview": "node_modules\nnpm-debug.*\n\n/config/local.*\n/test/coverage\n/test/tmp\n/test/fixtures/**/downloaded\n/dev\n"
},
{
"path": ".travis.yml",
"chars": 194,
"preview": "language: node_js\nnode_js:\n - node\n - lts/*\nscript: \"npm run test-travis\"\nbefore_install:\n - sudo apt-get install -y "
},
{
"path": "LICENSE",
"chars": 1048,
"preview": "Copyright (c) 2016 npms\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software a"
},
{
"path": "README.md",
"chars": 2723,
"preview": "# npms-analyzer\n\n[![Build status][travis-image]][travis-url] [![Coverage status][codecov-image]][codecov-url] [![Depende"
},
{
"path": "cli.js",
"chars": 685,
"preview": "#!/bin/sh\n':' //; exec \"$(command -v node)\" --max-old-space-size=4192 \"$0\" \"$@\"\n\n'use strict';\n\n// require('heapdump');"
},
{
"path": "cmd/consume.js",
"chars": 4421,
"preview": "'use strict';\n\nconst assert = require('assert');\nconst config = require('config');\nconst analyze = require('../lib/analy"
},
{
"path": "cmd/observe.js",
"chars": 2537,
"preview": "'use strict';\n\nconst assert = require('assert');\nconst config = require('config');\nconst promiseRetry = require('promise"
},
{
"path": "cmd/scoring.js",
"chars": 4430,
"preview": "'use strict';\n\nconst assert = require('assert');\nconst humanizeDuration = require('humanize-duration');\nconst prepare = "
},
{
"path": "cmd/tasks/check-gh-tokens.js",
"chars": 1372,
"preview": "'use strict';\n\nconst config = require('config');\nconst got = require('got');\n\nconst githubTokens = config.get('githubTok"
},
{
"path": "cmd/tasks/clean-extraneous.js",
"chars": 6477,
"preview": "'use strict';\n\nconst stats = require('../util/stats');\nconst bootstrap = require('../util/bootstrap');\n\nconst log = logg"
},
{
"path": "cmd/tasks/enqueue-missing.js",
"chars": 3606,
"preview": "'use strict';\n\nconst difference = require('lodash/difference');\nconst bootstrap = require('../util/bootstrap');\nconst st"
},
{
"path": "cmd/tasks/enqueue-outdated.js",
"chars": 3970,
"preview": "'use strict';\n\nconst bootstrap = require('../util/bootstrap');\nconst stats = require('../util/stats');\n\nconst log = logg"
},
{
"path": "cmd/tasks/enqueue-view.js",
"chars": 3225,
"preview": "'use strict';\n\nconst assert = require('assert');\nconst bootstrap = require('../util/bootstrap');\nconst stats = require('"
},
{
"path": "cmd/tasks/migrate.js",
"chars": 1868,
"preview": "'use strict';\n\nconst couchdbIterator = require('couchdb-iterator');\nconst analyze = require('../../lib/analyze');\nconst "
},
{
"path": "cmd/tasks/process-package.js",
"chars": 2038,
"preview": "'use strict';\n\nconst config = require('config');\nconst analyze = require('../../lib/analyze');\nconst score = require('.."
},
{
"path": "cmd/tasks/re-evaluate.js",
"chars": 2120,
"preview": "'use strict';\n\nconst couchdbIterator = require('couchdb-iterator');\nconst evaluate = require('../../lib/analyze/evaluate"
},
{
"path": "cmd/tasks/re-metadata.js",
"chars": 3170,
"preview": "'use strict';\n\nconst couchdbIterator = require('couchdb-iterator');\nconst metadata = require('../../lib/analyze/collect/"
},
{
"path": "cmd/tasks.js",
"chars": 353,
"preview": "'use strict';\n\nexports.command = 'tasks';\nexports.describe = 'Execute a task';\n\nexports.builder = (yargs) =>\n yargs\n "
},
{
"path": "cmd/util/bootstrap.js",
"chars": 4304,
"preview": "'use strict';\n\nconst config = require('config');\nconst nano = require('nano');\nconst elasticsearch = require('elasticsea"
},
{
"path": "cmd/util/stats/index.js",
"chars": 96,
"preview": "'use strict';\n\nmodule.exports = require('require-directory')(module, './', { recurse: false });\n"
},
{
"path": "cmd/util/stats/process.js",
"chars": 774,
"preview": "'use strict';\n\nconst pino = require('pino');\nconst humanizeDuration = require('humanize-duration');\n\nconst log = logger."
},
{
"path": "cmd/util/stats/progress.js",
"chars": 1578,
"preview": "'use strict';\n\nconst pino = require('pino');\n\nconst log = logger.child({ module: 'stats/progress' });\n\n// TODO: Add stat"
},
{
"path": "cmd/util/stats/queue.js",
"chars": 898,
"preview": "'use strict';\n\nconst pino = require('pino');\n\nconst log = logger.child({ module: 'stats/queue' });\n\n/**\n * Continuously "
},
{
"path": "cmd/util/stats/tokens.js",
"chars": 1359,
"preview": "'use strict';\n\nconst pino = require('pino');\nconst tokenDealer = require('token-dealer');\nconst minBy = require('lodash/"
},
{
"path": "config/couchdb/npms-analyzer-npm.json",
"chars": 284,
"preview": "{\n \"_id\": \"_design/npms-analyzer\",\n \"language\": \"javascript\",\n \"views\": {\n \"packages-version\": {\n "
},
{
"path": "config/couchdb/npms-analyzer-npms.json",
"chars": 1003,
"preview": "{\n \"_id\": \"_design/npms-analyzer\",\n \"language\": \"javascript\",\n \"views\": {\n \"packages-evaluation\": {\n "
},
{
"path": "config/default.json5",
"chars": 1283,
"preview": "{\n // Databases & similar stuff\n couchdbNpm: {\n url: 'http://admin:admin@127.0.0.1:5984/npm',\n reque"
},
{
"path": "config/elasticsearch/npms.json5",
"chars": 17791,
"preview": "{\n // ------------------------------------------------------------------------------\n // Index settings\n // ---"
},
{
"path": "docs/architecture.md",
"chars": 8052,
"preview": "# Architecture\n\nThe `npms-analyzer` runs two continuous and distinct processes. One is the `analysis` process where each"
},
{
"path": "docs/deploys.md",
"chars": 1247,
"preview": "# Deploys\n\nWe use `pm2` to deploy `npms-analyzer`, install it by running `$ npm install -g pm2`. You may find the pm2 co"
},
{
"path": "docs/diagrams/analysis.xml",
"chars": 2951,
"preview": "<mxfile type=\"device\" userAgent=\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) "
},
{
"path": "docs/diagrams/continuous-scoring.xml",
"chars": 1935,
"preview": "<mxfile type=\"device\" userAgent=\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) "
},
{
"path": "docs/setup.md",
"chars": 2985,
"preview": "# Setup\n\nBelow you will find a list of items that you must do to get the project working on your local machine. The prod"
},
{
"path": "ecosystem.json5",
"chars": 1537,
"preview": "// This is the pm2 configuration file for npms-analyzer\n{\n apps: [\n {\n name: 'npms-analyzer-observe"
},
{
"path": "lib/analyze/collect/bin/david-json",
"chars": 1525,
"preview": "#!/usr/bin/env node\n\n'use strict';\n\nglobal.Promise = require('bluebird');\n\n// Some packages have a .npmrc to integrate w"
},
{
"path": "lib/analyze/collect/github.js",
"chars": 9322,
"preview": "'use strict';\n\nconst got = require('got');\nconst moment = require('moment');\nconst ghIssuesStats = require('gh-issues-st"
},
{
"path": "lib/analyze/collect/index.js",
"chars": 4278,
"preview": "'use strict';\n\nconst pickBy = require('lodash/pickBy');\nconst intersectionWith = require('lodash/intersectionWith');\ncon"
},
{
"path": "lib/analyze/collect/metadata.js",
"chars": 11787,
"preview": "'use strict';\n\nconst moment = require('moment');\nconst spdx = require('spdx');\nconst spdxCorrect = require('spdx-correct"
},
{
"path": "lib/analyze/collect/npm.js",
"chars": 3936,
"preview": "'use strict';\n\nconst got = require('got');\nconst moment = require('moment');\nconst size = require('lodash/size');\nconst "
},
{
"path": "lib/analyze/collect/source.js",
"chars": 11713,
"preview": "'use strict';\n\nconst path = require('path');\nconst detectRepoLinters = require('detect-repo-linters');\nconst detectRepoT"
},
{
"path": "lib/analyze/collect/util/fileContents.js",
"chars": 1147,
"preview": "'use strict';\n\nconst readFile = Promise.promisify(require('fs').readFile);\n\nconst log = logger.child({ module: 'util/fil"
},
{
"path": "lib/analyze/collect/util/fileSize.js",
"chars": 1903,
"preview": "'use strict';\n\nconst stat = Promise.promisify(require('fs').stat);\nconst globby = require('globby');\n\nconst log = logger"
},
{
"path": "lib/analyze/collect/util/pointsToRanges.js",
"chars": 1520,
"preview": "'use strict';\n\nconst moment = require('moment');\n\n/**\n * Aggregates an array of points into buckets of date ranges.\n *\n "
},
{
"path": "lib/analyze/collect/util/promisePropsSettled.js",
"chars": 778,
"preview": "'use strict';\n\nconst mapValues = require('lodash/mapValues');\n\n/**\n * Promise utility similar to bluebird's .props() but"
},
{
"path": "lib/analyze/download/git.js",
"chars": 6322,
"preview": "'use strict';\n\nconst urlLib = require('url');\nconst exec = require('../util/exec');\nconst hostedGitInfo = require('../ut"
},
{
"path": "lib/analyze/download/github.js",
"chars": 7415,
"preview": "'use strict';\n\nconst fs = require('fs');\nconst tokenDealer = require('token-dealer');\nconst got = require('got');\nconst "
},
{
"path": "lib/analyze/download/index.js",
"chars": 3263,
"preview": "'use strict';\n\nconst os = require('os');\nconst stat = Promise.promisify(require('fs').stat);\nconst downloaders = require"
},
{
"path": "lib/analyze/download/npm.js",
"chars": 3942,
"preview": "'use strict';\n\nconst fs = require('fs');\nconst got = require('got');\nconst untar = require('./util/untar');\nconst gotRet"
},
{
"path": "lib/analyze/download/util/assertFilesCount.js",
"chars": 1020,
"preview": "'use strict';\n\nconst exec = require('../../util/exec');\n\n/**\n * Asserts that the number of files in a directory is withi"
},
{
"path": "lib/analyze/download/util/findPackageDir.js",
"chars": 3224,
"preview": "'use strict';\n\nconst path = require('path');\nconst loadJsonFile = require('load-json-file');\nconst globby = require('glo"
},
{
"path": "lib/analyze/download/util/mergePackageJson.js",
"chars": 2048,
"preview": "'use strict';\n\nconst writeFile = Promise.promisify(require('fs').writeFile);\nconst loadJsonFile = require('load-json-fil"
},
{
"path": "lib/analyze/download/util/untar.js",
"chars": 4107,
"preview": "'use strict';\n\nconst path = require('path');\nconst which = require('which');\nconst unlink = Promise.promisify(require('f"
},
{
"path": "lib/analyze/evaluate/index.js",
"chars": 482,
"preview": "'use strict';\n\nconst evaluators = require('require-directory')(module, './', { recurse: false });\n\n/**\n * Runs all the e"
},
{
"path": "lib/analyze/evaluate/maintenance.js",
"chars": 7350,
"preview": "'use strict';\n\nconst moment = require('moment');\nconst find = require('lodash/find');\nconst get = require('lodash/get');"
},
{
"path": "lib/analyze/evaluate/popularity.js",
"chars": 3247,
"preview": "'use strict';\n\nconst moment = require('moment');\nconst find = require('lodash/find');\n\n/**\n * Evaluates the downloads co"
},
{
"path": "lib/analyze/evaluate/quality.js",
"chars": 6376,
"preview": "/* eslint no-nested-ternary: 0 */\n\n'use strict';\n\nconst url = require('url');\nconst semver = require('semver');\nconst ge"
},
{
"path": "lib/analyze/index.js",
"chars": 7077,
"preview": "'use strict';\n\nconst promiseRetry = require('promise-retry');\nconst serializeError = require('serialize-error');\nconst o"
},
{
"path": "lib/analyze/util/exec.js",
"chars": 1443,
"preview": "'use strict';\n\nconst cp = require('child_process');\nconst escapeshellarg = require('php-escape-shell').php_escapeshellar"
},
{
"path": "lib/analyze/util/gotRetry.js",
"chars": 561,
"preview": "/* eslint no-bitwise: 0 */\n\n'use strict';\n\nconst got = require('got');\nconst normalize = require('got/source/normalize-a"
},
{
"path": "lib/analyze/util/hostedGitInfo.js",
"chars": 647,
"preview": "'use strict';\n\nconst hostedGitInfoFromUrl = require('hosted-git-info').fromUrl;\n\nconst log = logger.child({ module: 'uti"
},
{
"path": "lib/analyze/util/normalizePackageJson.js",
"chars": 2595,
"preview": "'use strict';\n\nconst normalizePackageData = require('normalize-package-data');\n\nconst log = logger.child({ module: 'util"
},
{
"path": "lib/analyze/util/packageJsonFromData.js",
"chars": 1381,
"preview": "'use strict';\n\nconst normalizePackageJson = require('./normalizePackageJson');\n\nconst log = logger.child({ module: 'util"
},
{
"path": "lib/configure.js",
"chars": 1826,
"preview": "'use strict';\n\nconst Promise = require('bluebird');\nconst pino = require('pino');\nconst forIn = require('lodash/forIn');"
},
{
"path": "lib/observers/realtime.js",
"chars": 11948,
"preview": "'use strict';\n\nconst couchdbForce = require('couchdb-force');\nconst get = require('lodash/get');\nconst uniq = require('l"
},
{
"path": "lib/observers/stale.js",
"chars": 7426,
"preview": "'use strict';\n\nconst couchdbIterator = require('couchdb-iterator');\nconst couchdbForce = require('couchdb-force');\nconst"
},
{
"path": "lib/queue.js",
"chars": 15977,
"preview": "'use strict';\n\nconst EventEmitter = require('events').EventEmitter;\nconst assert = require('assert');\nconst amqp = requi"
},
{
"path": "lib/scoring/aggregate.js",
"chars": 5825,
"preview": "'use strict';\n\nconst couchdbIterator = require('couchdb-iterator');\nconst promiseRetry = require('promise-retry');\nconst"
},
{
"path": "lib/scoring/finalize.js",
"chars": 1488,
"preview": "'use strict';\n\nconst log = logger.child({ module: 'scoring/finalize' });\n\n/**\n * Finalizes the scoring cycle.\n * Updates"
},
{
"path": "lib/scoring/prepare.js",
"chars": 2886,
"preview": "'use strict';\n\nconst fs = require('fs');\nconst JSON5 = require('json5');\nconst difference = require('lodash/difference')"
},
{
"path": "lib/scoring/score.js",
"chars": 12739,
"preview": "'use strict';\n\nconst couchdbIterator = require('couchdb-iterator');\nconst weightedMean = require('weighted-mean');\nconst"
},
{
"path": "lib/scoring/util/paperNumerical.js",
"chars": 19340,
"preview": "/* eslint-disable */\n\n// This was copied from the https://raw.githubusercontent.com/paperjs/paper.js project\n// It's use"
},
{
"path": "package.json",
"chars": 3263,
"preview": "{\n \"name\": \"npms-analyzer\",\n \"version\": \"1.0.0\",\n \"description\": \"The analyzer behind npms.io\",\n \"main\": \"index.js\","
},
{
"path": "test/.eslintrc.json",
"chars": 45,
"preview": "{\n \"env\": {\n \"mocha\": true\n }\n}\n"
},
{
"path": "test/bin/download-fixtures",
"chars": 686,
"preview": "#!/bin/sh\n\n# Thanks Patrick Marques <patrickfmarques@gmail.com> for helping with bash\n\n# Travis runs on a old linux dist"
},
{
"path": "test/fixtures/analyze/collect/modules/0/data.json",
"chars": 1376,
"preview": "{\n \"name\": \"0\",\n \"dist-tags\": {\n \"latest\": \"0.0.0\"\n },\n \"versions\": {\n \"0.0.0\": {\n \"name\":"
},
{
"path": "test/fixtures/analyze/collect/modules/0/expected-source.json",
"chars": 61,
"preview": "{\n \"files\": {\n \"readmeSize\": 0,\n \"testsSize\": 0\n }\n}\n"
},
{
"path": "test/fixtures/analyze/collect/modules/@bcoe%2fexpress-oauth-server/data.json",
"chars": 4387,
"preview": "{\n \"_id\": \"@bcoe/express-oauth-server\",\n \"_rev\": \"2-1eafc256421b304557faf317e9b16b3c\",\n \"name\": \"@bcoe/express-oauth-"
},
{
"path": "test/fixtures/analyze/collect/modules/@bcoe%2fexpress-oauth-server/expected-metadata.json",
"chars": 3134,
"preview": "{\n \"name\": \"@bcoe/express-oauth-server\",\n \"scope\": \"bcoe\",\n \"version\": \"1.0.0\",\n \"description\": \"OAuth provider for "
},
{
"path": "test/fixtures/analyze/collect/modules/babel-jest/data.json",
"chars": 98182,
"preview": "{\n \"_id\": \"babel-jest\",\n \"_rev\": \"25-ad9120705c652660c3dd2ad91893a4b5\",\n \"name\": \"babel-jest\",\n \"description\": \"[Bab"
},
{
"path": "test/fixtures/analyze/collect/modules/babel-jest/expected-source.json",
"chars": 1634,
"preview": "{\n \"files\": {\n \"readmeSize\": 803,\n \"testsSize\": 783379,\n \"hasNpmIgnore\": true,\n \"hasChangelog\": true\n },\n "
},
{
"path": "test/fixtures/analyze/collect/modules/backoff/data.json",
"chars": 42294,
"preview": "{\n \"_id\": \"backoff\",\n \"_rev\": \"2-c1f0bcee39d50baac252504546d7d13d\",\n \"name\": \"backoff\",\n \"description\": \"Fibonac"
},
{
"path": "test/fixtures/analyze/collect/modules/backoff/expected-source.json",
"chars": 702,
"preview": "{\n \"files\": {\n \"readmeSize\": 11287,\n \"testsSize\": 27463,\n \"hasChangelog\": true\n },\n \"linters\": [\n \"jshint"
},
{
"path": "test/fixtures/analyze/collect/modules/cross-spawn/data.json",
"chars": 59838,
"preview": "{\n\t\"name\": \"cross-spawn\",\n\t\"description\": \"Cross platform child_process#spawn and child_process#spawnSync\",\n\t\"dist-tags\""
},
{
"path": "test/fixtures/analyze/collect/modules/cross-spawn/expected-github.json",
"chars": 1621,
"preview": "{\n \"starsCount\": 74,\n \"forksCount\": 11,\n \"subscribersCount\": 12,\n \"issues\": {\n \"count\": 31,\n \"openCount\": 2,\n "
},
{
"path": "test/fixtures/analyze/collect/modules/cross-spawn/expected-metadata.json",
"chars": 4784,
"preview": "{\n \"name\": \"cross-spawn\",\n \"scope\": \"unscoped\",\n \"version\": \"2.2.3\",\n \"description\": \"Cross platform child_process#s"
},
{
"path": "test/fixtures/analyze/collect/modules/cross-spawn/expected-npm.json",
"chars": 750,
"preview": "{\n \"downloads\": [\n {\n \"from\": \"2016-05-08T00:00:00.000Z\",\n \"to\": \"2016-05-09T00:00:00.000Z\",\n \"count\""
},
{
"path": "test/fixtures/analyze/collect/modules/cross-spawn/expected-source.json",
"chars": 2995,
"preview": "{\n \"files\": {\n \"readmeSize\": 2709,\n \"testsSize\": 20813,\n \"hasNpmIgnore\": true\n },\n \"linters\": [\n \"editorc"
},
{
"path": "test/fixtures/analyze/collect/modules/hapi/data.json",
"chars": 621068,
"preview": "{\n \"name\": \"hapi\",\n \"description\": \"HTTP Server framework\",\n \"dist-tags\": {\n \"latest\": \"13.4.0\",\n \"lts"
},
{
"path": "test/fixtures/analyze/collect/modules/hapi/expected-source.json",
"chars": 497,
"preview": "{\n \"files\": {\n \"readmeSize\": 1213,\n \"testsSize\": 959339,\n \"hasShrinkwrap\": true\n },\n \"badges\": [\n {\n "
},
{
"path": "test/fixtures/analyze/collect/modules/planify/data.json",
"chars": 30531,
"preview": "{\n \"name\": \"planify\",\n \"description\": \"Plan a series of steps and display the output in a beautiful way\",\n \"dist-t"
},
{
"path": "test/fixtures/analyze/collect/modules/planify/expected-source.json",
"chars": 2815,
"preview": "{\n \"files\": {\n \"readmeSize\": 9014,\n \"testsSize\": 140994\n },\n \"linters\": [\n \"editorconfig\",\n \"eslint\"\n ],"
},
{
"path": "test/fixtures/analyze/collect/modules/react/data.json",
"chars": 221490,
"preview": "{\n \"_id\": \"react\",\n \"_rev\": \"145-228ca42672da2fe6f9f5b5fc3bccb4da\",\n \"name\": \"react\",\n \"description\": \"React is a Ja"
},
{
"path": "test/fixtures/analyze/collect/modules/react/expected-source.json",
"chars": 1584,
"preview": "{\n \"files\": {\n \"readmeSize\": 1198,\n \"testsSize\": 1491255,\n \"hasChangelog\": true\n },\n \"badges\": [\n {\n "
},
{
"path": "test/fixtures/analyze/collect/modules/react-router/data.json",
"chars": 354149,
"preview": "{\n \"_id\": \"react-router\",\n \"_rev\": \"71-41439c737028709c0e5084dcfac9d06c\",\n \"name\": \"react-router\",\n \"description\": \""
},
{
"path": "test/fixtures/analyze/collect/modules/react-router/expected-source.json",
"chars": 2729,
"preview": "{\n \"files\": {\n \"readmeSize\": 968,\n \"testsSize\": 31281,\n \"hasChangelog\": true\n },\n \"badges\": [\n {\n \"u"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/0bfbe2f1c03ff5ed9c3baa91d588e218.headers",
"chars": 1314,
"preview": "{\n \"statusCode\": 404,\n \"headers\": {\n \"server\": \"GitHub.com\",\n \"date\": \"Wed, 11 May 2016 15:37:46 GMT\",\n \"cont"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/2236c266c85b15946d7ca69cc2e1e091",
"chars": 182,
"preview": "{\"message\":\"Repository access blocked\",\"block\":{\"reason\":\"dmca\",\"created_at\":\"2012-10-03T18:57:04Z\",\"html_url\":\"https://"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/2236c266c85b15946d7ca69cc2e1e091.headers",
"chars": 1239,
"preview": "{\n \"statusCode\": 451,\n \"headers\": {\n \"server\": \"GitHub.com\",\n \"date\": \"Wed, 11 May 2016 16:21:00 GMT\",\n \"cont"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/24d4b4797edc40614848f01802bbe2b3.headers",
"chars": 1488,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"server\": \"GitHub.com\",\n \"date\": \"Wed, 11 May 2016 12:50:47 GMT\",\n \"cont"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/2f87db1cf50593ec3f80835f624ec88a",
"chars": 0,
"preview": ""
},
{
"path": "test/fixtures/analyze/collect/recorded/github/2f87db1cf50593ec3f80835f624ec88a.headers",
"chars": 1186,
"preview": "{\n \"statusCode\": 204,\n \"headers\": {\n \"server\": \"GitHub.com\",\n \"date\": \"Wed, 11 May 2016 15:21:30 GMT\",\n \"stat"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/3a1a79735cab3e2c46da0f739eccd595.headers",
"chars": 1477,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"server\": \"GitHub.com\",\n \"date\": \"Wed, 11 May 2016 12:50:47 GMT\",\n \"cont"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/3b5e34c45a594730608f6170cacf31fe",
"chars": 0,
"preview": ""
},
{
"path": "test/fixtures/analyze/collect/recorded/github/3b5e34c45a594730608f6170cacf31fe.headers",
"chars": 1189,
"preview": "{\n \"statusCode\": 204,\n \"headers\": {\n \"server\": \"GitHub.com\",\n \"date\": \"Wed, 11 May 2016 15:21:30 GMT\",\n \"stat"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/3d15804db16c597c23a14946a78b8e1b.headers",
"chars": 1294,
"preview": "{\n \"statusCode\": 404,\n \"headers\": {\n \"server\": \"GitHub.com\",\n \"date\": \"Wed, 11 May 2016 15:37:46 GMT\",\n \"cont"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/4109ed740591855f9e48eb868f40db86",
"chars": 2,
"preview": "[]"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/4109ed740591855f9e48eb868f40db86.headers",
"chars": 1421,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"server\": \"GitHub.com\",\n \"date\": \"Wed, 11 May 2016 15:21:30 GMT\",\n \"cont"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/5bdd0f1e3c86f0114eb714a2ce79e905",
"chars": 182,
"preview": "{\"message\":\"Repository access blocked\",\"block\":{\"reason\":\"dmca\",\"created_at\":\"2012-10-03T18:57:04Z\",\"html_url\":\"https://"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/5bdd0f1e3c86f0114eb714a2ce79e905.headers",
"chars": 1230,
"preview": "{\n \"statusCode\": 451,\n \"headers\": {\n \"server\": \"GitHub.com\",\n \"date\": \"Wed, 11 May 2016 16:20:59 GMT\",\n \"cont"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/6913bcb008bea8f6a0384da9bbae2293.headers",
"chars": 1482,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"server\": \"GitHub.com\",\n \"date\": \"Wed, 11 May 2016 15:21:31 GMT\",\n \"cont"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/85c5e5ee4a806e7d405984c325f29007.headers",
"chars": 1672,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"server\": \"GitHub.com\",\n \"date\": \"Wed, 11 May 2016 15:21:30 GMT\",\n \"cont"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/92a4188d8af9e8c1ff665859b3cd86b8.headers",
"chars": 1516,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"server\": \"GitHub.com\",\n \"date\": \"Wed, 11 May 2016 15:21:31 GMT\",\n \"cont"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/9ccf24e28c94543e4ae601d5aa9c8cba.headers",
"chars": 1494,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"server\": \"GitHub.com\",\n \"date\": \"Wed, 11 May 2016 15:21:30 GMT\",\n \"cont"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/a3360f53aa71342ad67cded65f6ea1da.headers",
"chars": 1486,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"server\": \"GitHub.com\",\n \"date\": \"Wed, 11 May 2016 15:21:30 GMT\",\n \"cont"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/a7d2cca72c7267fd27fe769cb4d2f611.headers",
"chars": 1512,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"server\": \"GitHub.com\",\n \"date\": \"Wed, 11 May 2016 12:50:47 GMT\",\n \"cont"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/afcd15ede3deaa855315f5a1fbc3e61d.headers",
"chars": 1494,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"server\": \"GitHub.com\",\n \"date\": \"Wed, 11 May 2016 15:21:30 GMT\",\n \"cont"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/b32671b71119c7fc156f3aa050d1cb12",
"chars": 182,
"preview": "{\"message\":\"Repository access blocked\",\"block\":{\"reason\":\"dmca\",\"created_at\":\"2012-10-03T18:57:04Z\",\"html_url\":\"https://"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/b32671b71119c7fc156f3aa050d1cb12.headers",
"chars": 1208,
"preview": "{\n \"statusCode\": 451,\n \"headers\": {\n \"server\": \"GitHub.com\",\n \"date\": \"Wed, 11 May 2016 16:20:59 GMT\",\n \"cont"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/c23e4492fdaefde5fe60524fab308532.headers",
"chars": 1518,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"server\": \"GitHub.com\",\n \"date\": \"Wed, 11 May 2016 15:21:30 GMT\",\n \"cont"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/c389930a56f1bad9257ed1490fc32c9b.headers",
"chars": 1324,
"preview": "{\n \"statusCode\": 404,\n \"headers\": {\n \"server\": \"GitHub.com\",\n \"date\": \"Wed, 11 May 2016 15:37:46 GMT\",\n \"cont"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/cc88ef857a3a1492913c066047c5c033.headers",
"chars": 1480,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"server\": \"GitHub.com\",\n \"date\": \"Wed, 11 May 2016 12:50:47 GMT\",\n \"cont"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/d3b7e3ec7ad3c841c45ff000fd77b711.headers",
"chars": 1318,
"preview": "{\n \"statusCode\": 404,\n \"headers\": {\n \"server\": \"GitHub.com\",\n \"date\": \"Wed, 11 May 2016 15:37:46 GMT\",\n \"cont"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/d495e09987382290004f52a8fa39243b",
"chars": 2,
"preview": "[]"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/d495e09987382290004f52a8fa39243b.headers",
"chars": 1427,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"server\": \"GitHub.com\",\n \"date\": \"Wed, 11 May 2016 15:21:30 GMT\",\n \"cont"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/dadb0a8973a79e019a2a0affeb248deb.headers",
"chars": 1316,
"preview": "{\n \"statusCode\": 404,\n \"headers\": {\n \"server\": \"GitHub.com\",\n \"date\": \"Wed, 11 May 2016 15:37:46 GMT\",\n \"cont"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/ebd826ea6dcd2abd0dcc961dd0fe4176",
"chars": 182,
"preview": "{\"message\":\"Repository access blocked\",\"block\":{\"reason\":\"dmca\",\"created_at\":\"2012-10-03T18:57:04Z\",\"html_url\":\"https://"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/ebd826ea6dcd2abd0dcc961dd0fe4176.headers",
"chars": 1227,
"preview": "{\n \"statusCode\": 451,\n \"headers\": {\n \"server\": \"GitHub.com\",\n \"date\": \"Wed, 11 May 2016 16:20:59 GMT\",\n \"cont"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/ede6348d94fd4d537bcc1e42b050d8ed",
"chars": 182,
"preview": "{\"message\":\"Repository access blocked\",\"block\":{\"reason\":\"dmca\",\"created_at\":\"2012-10-03T18:57:04Z\",\"html_url\":\"https://"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/ede6348d94fd4d537bcc1e42b050d8ed.headers",
"chars": 1232,
"preview": "{\n \"statusCode\": 451,\n \"headers\": {\n \"server\": \"GitHub.com\",\n \"date\": \"Wed, 11 May 2016 16:20:59 GMT\",\n \"cont"
},
{
"path": "test/fixtures/analyze/collect/recorded/github/ff4470aede5dba39b4736419dda38443.headers",
"chars": 1483,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"server\": \"GitHub.com\",\n \"date\": \"Wed, 11 May 2016 15:21:30 GMT\",\n \"cont"
},
{
"path": "test/fixtures/analyze/collect/recorded/index/474829c21c2e69d2c0d889af2a714584",
"chars": 31773,
"preview": "{\"_id\":\"graphql-shorthand-parser\",\"_rev\":\"2-8a4abe71d406323ccb96762a491f6a53\",\"name\":\"graphql-shorthand-parser\",\"descrip"
},
{
"path": "test/fixtures/analyze/collect/recorded/index/474829c21c2e69d2c0d889af2a714584.headers",
"chars": 568,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"server\": \"CouchDB/1.5.0 (Erlang OTP/R16B03)\",\n \"etag\": \"\\\"2-8a4abe71d40632"
},
{
"path": "test/fixtures/analyze/collect/recorded/index/ae18615611dfb3f32feaf1c607df7bac",
"chars": 171593,
"preview": "{\"_id\":\"bower\",\"_rev\":\"243-ee588285b884979435a37fcca5d73613\",\"name\":\"bower\",\"description\":\"The browser package manager\","
},
{
"path": "test/fixtures/analyze/collect/recorded/index/ae18615611dfb3f32feaf1c607df7bac.headers",
"chars": 553,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"server\": \"CouchDB/1.5.0 (Erlang OTP/R16B03)\",\n \"etag\": \"\\\"243-ee588285b884"
},
{
"path": "test/fixtures/analyze/collect/recorded/index/d40cbb49129f01a9d5130a95f54d4f79",
"chars": 41,
"preview": "{\"error\":\"not_found\",\"reason\":\"missing\"}\n"
},
{
"path": "test/fixtures/analyze/collect/recorded/index/d40cbb49129f01a9d5130a95f54d4f79.headers",
"chars": 520,
"preview": "{\n \"statusCode\": 404,\n \"headers\": {\n \"server\": \"CouchDB/1.5.0 (Erlang OTP/R16B03)\",\n \"date\": \"Fri, 13 May 2016 1"
},
{
"path": "test/fixtures/analyze/collect/recorded/npm/278e742ec691d9647761d9e06a93c852.headers",
"chars": 1170,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"server\": \"nginx/1.4.6 (Ubuntu)\",\n \"content-type\": \"application/json; chars"
},
{
"path": "test/fixtures/analyze/collect/recorded/npm/6da574a19e30e15a2628bc2a7ae7d5a4",
"chars": 40,
"preview": "{\"rows\":[\r\n{\"key\":null,\"value\":149}\r\n]}\n"
},
{
"path": "test/fixtures/analyze/collect/recorded/npm/6da574a19e30e15a2628bc2a7ae7d5a4.headers",
"chars": 691,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"transfer-encoding\": \"chunked\",\n \"server\": \"CouchDB/1.5.0 (Erlang OTP/R16B0"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/0046732f19ee23072e08f55d2a400eca.headers",
"chars": 932,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Sun, 26 Mar 2017 18:22:40 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/02777f766910df6791475f44c0e2b57b.headers",
"chars": 928,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Sun, 26 Mar 2017 18:20:43 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/0429ac4bdf217161d9a2772fdb7861e2.headers",
"chars": 1150,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"transfer-encoding\": \"chunked\",\n \"access-control-allow-origin\": \"https://re"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/0c774ab0c1fa72fb9dbba94b96693973.headers",
"chars": 934,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"server\": \"nginx/1.10.1\",\n \"content-type\": \"application/octet-stream\",\n "
},
{
"path": "test/fixtures/analyze/collect/recorded/source/1152654e8bfc034a9d043925c55fbe48.headers",
"chars": 1009,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Sun, 05 May 2019 23:28:51 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/180995f905c69e6355ccbeb197109fb9.headers",
"chars": 1127,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"transfer-encoding\": \"chunked\",\n \"access-control-allow-origin\": \"https://re"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/1b4935ddf796087a37e45c313edddd4f.headers",
"chars": 1157,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"transfer-encoding\": \"chunked\",\n \"access-control-allow-origin\": \"https://re"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/310c26a3622e22be3798b810bb056cd2.headers",
"chars": 934,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Mon, 08 Oct 2018 17:22:42 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/32532aa076ea4d37a94def3f370e23fc",
"chars": 0,
"preview": ""
},
{
"path": "test/fixtures/analyze/collect/recorded/source/32532aa076ea4d37a94def3f370e23fc.headers",
"chars": 1538,
"preview": "{\n \"statusCode\": 302,\n \"headers\": {\n \"server\": \"GitHub.com\",\n \"date\": \"Sun, 26 Mar 2017 18:20:44 GMT\",\n \"cont"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/36d2b9e3113bb8656477a0866759fca3",
"chars": 0,
"preview": ""
},
{
"path": "test/fixtures/analyze/collect/recorded/source/36d2b9e3113bb8656477a0866759fca3.headers",
"chars": 1544,
"preview": "{\n \"statusCode\": 302,\n \"headers\": {\n \"server\": \"GitHub.com\",\n \"date\": \"Sun, 26 Mar 2017 18:22:37 GMT\",\n \"cont"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/3f5a8e5d9434bbd4ec976c36835cdc49",
"chars": 0,
"preview": ""
},
{
"path": "test/fixtures/analyze/collect/recorded/source/3f5a8e5d9434bbd4ec976c36835cdc49.headers",
"chars": 1546,
"preview": "{\n \"statusCode\": 302,\n \"headers\": {\n \"server\": \"GitHub.com\",\n \"date\": \"Sun, 26 Mar 2017 18:20:41 GMT\",\n \"cont"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/40971c651c26e9ab81128eb0c10f37af.headers",
"chars": 935,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Mon, 08 Oct 2018 17:22:42 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/44c08d748b72275a181e7820e9c258e8.headers",
"chars": 912,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Sun, 02 Apr 2017 20:41:18 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/4d7966a6e7249722abdff0a7555a2527.headers",
"chars": 933,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Sun, 26 Mar 2017 18:20:43 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/4ece6fa3645c526294eaf1b270113e6d.headers",
"chars": 916,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Sun, 26 Mar 2017 18:22:41 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/50ec845be323513dc05d4ce6eeb56639.headers",
"chars": 909,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Sun, 26 Mar 2017 18:22:41 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/586e879d6364ca5313dd5f956d47dbd4.headers",
"chars": 944,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Mon, 08 Oct 2018 17:20:16 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/5fcfa736ff5a936752226c33baab7ce5.headers",
"chars": 920,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Sun, 26 Mar 2017 18:20:43 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/6566d9e3adefb1ed8fe60a17bbf13133.headers",
"chars": 919,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Sun, 26 Mar 2017 18:22:40 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/6632454a445cfcd8152c30b4b8c64783.headers",
"chars": 994,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Sun, 05 May 2019 23:28:49 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/672a67ca0e6ca2125e9601f4e532dc2e.headers",
"chars": 1095,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"transfer-encoding\": \"chunked\",\n \"access-control-allow-origin\": \"https://re"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/6ef5fd0be1ac70c0cc78c63dc72f97da.headers",
"chars": 944,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Mon, 08 Oct 2018 17:22:47 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/6f6f60463501ffe7964700fdb5262ea7.headers",
"chars": 931,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Sun, 02 Apr 2017 20:36:59 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/77c0ab2484ae068cadf90515cd2bb6d4.headers",
"chars": 946,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Mon, 08 Oct 2018 17:22:45 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/7fb20e04d9d456482d62b369a1c268c0.headers",
"chars": 945,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Mon, 08 Oct 2018 17:20:16 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/833f1d757cdb7cfa152b54680d0d2d73.headers",
"chars": 938,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Mon, 08 Oct 2018 17:20:18 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/8881281d102f7688a2b0e5d7ffb48299",
"chars": 0,
"preview": ""
},
{
"path": "test/fixtures/analyze/collect/recorded/source/8881281d102f7688a2b0e5d7ffb48299.headers",
"chars": 1465,
"preview": "{\n \"statusCode\": 302,\n \"headers\": {\n \"server\": \"GitHub.com\",\n \"date\": \"Sun, 02 Apr 2017 20:36:56 GMT\",\n \"cont"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/8acf97225f349b2c99978025ba5b8e92.headers",
"chars": 916,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Sun, 26 Mar 2017 18:22:35 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/8dcbc0b25ce4f37fd5c0bc06d633eb52.headers",
"chars": 916,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Sun, 26 Mar 2017 18:20:45 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/92d4837e4094c3e096f398bc89aabb0f.headers",
"chars": 904,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Sun, 02 Apr 2017 20:41:18 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/981d42650123f10d1600074d65aa4f43.headers",
"chars": 939,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Mon, 08 Oct 2018 17:20:17 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/9a7bfe12b0910e8bd69ed06184030276.headers",
"chars": 1011,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Sun, 05 May 2019 23:28:50 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/a27623f8433973cd8a3c9ac32782dd9b.headers",
"chars": 945,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Mon, 08 Oct 2018 17:22:47 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/a9be16a47e9b4e3762bfbcfbec14effd.headers",
"chars": 941,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Mon, 08 Oct 2018 17:22:43 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/acd2d3271b8cd130cd75e572ada409c1.headers",
"chars": 940,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Mon, 08 Oct 2018 17:20:17 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/afd6397af26789ff7e024c758e094e02.headers",
"chars": 1012,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Sun, 05 May 2019 23:28:48 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/b00ce31e18a32896ac83d6819e9816fe",
"chars": 0,
"preview": ""
},
{
"path": "test/fixtures/analyze/collect/recorded/source/b00ce31e18a32896ac83d6819e9816fe.headers",
"chars": 1510,
"preview": "{\n \"statusCode\": 302,\n \"headers\": {\n \"server\": \"GitHub.com\",\n \"date\": \"Sun, 26 Mar 2017 18:22:31 GMT\",\n \"cont"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/b6d2435e45a7f8f3b88152e577c55b84.headers",
"chars": 925,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Sun, 02 Apr 2017 20:36:59 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/bb391b38e8f2529e20e3e313a7875566.headers",
"chars": 926,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Sun, 26 Mar 2017 18:22:40 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/bbd9372c326ea4fb4acc82bd30e9491c.headers",
"chars": 937,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Mon, 08 Oct 2018 17:20:18 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/c2a263604b39c741dcba960ab7bdf64e.headers",
"chars": 1124,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"transfer-encoding\": \"chunked\",\n \"access-control-allow-origin\": \"https://re"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/ca1b5b9f76e662b613be43d64e92c4b4.headers",
"chars": 902,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Sun, 26 Mar 2017 18:22:35 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/d4bf4a43cf4e7b6c27e40c732c9d8bfa.headers",
"chars": 918,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Sun, 02 Apr 2017 20:36:59 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/daaa6494600d82aa3e21e56452a8702a",
"chars": 0,
"preview": ""
},
{
"path": "test/fixtures/analyze/collect/recorded/source/daaa6494600d82aa3e21e56452a8702a.headers",
"chars": 1441,
"preview": "{\n \"statusCode\": 302,\n \"headers\": {\n \"server\": \"GitHub.com\",\n \"date\": \"Sun, 02 Apr 2017 20:41:12 GMT\",\n \"cont"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/e2d10b245b6bce60976eb41c755c5333.headers",
"chars": 996,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Sun, 05 May 2019 23:28:52 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/e66bf57e7754e3a75c0b3da3c7d3b894.headers",
"chars": 909,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Sun, 26 Mar 2017 18:22:35 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/f13ef1d336f7343f21a8f1e755ee4a1d.headers",
"chars": 917,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Sun, 02 Apr 2017 20:41:18 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/f4fbc0e6ea0806cdebfe05e95480858f.headers",
"chars": 1157,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"transfer-encoding\": \"chunked\",\n \"access-control-allow-origin\": \"https://re"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/fba7a93bdb3f483048d32ccc0a105e2b.headers",
"chars": 945,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Mon, 08 Oct 2018 17:22:45 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/collect/recorded/source/febf07de22a7d2d7ffe02742c6b81857.headers",
"chars": 940,
"preview": "{\n \"statusCode\": 200,\n \"headers\": {\n \"date\": \"Mon, 08 Oct 2018 17:22:43 GMT\",\n \"content-type\": \"application/json"
},
{
"path": "test/fixtures/analyze/download/mocked/broken-archive.tgz",
"chars": 15,
"preview": "broken tarball\n"
},
{
"path": "test/fixtures/analyze/download/mocked/non-gzip-archive.tgz",
"chars": 1270,
"preview": "<!doctype html>\n<html>\n<head>\n <title>Example Domain</title>\n\n <meta charset=\"utf-8\" />\n <meta http-equiv=\"Cont"
}
]
// ... and 114 more files (download for full content)
About this extraction
This page contains the full source code of the npms-io/npms-analyzer GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 314 files (2.1 MB), approximately 561.2k tokens, and a symbol index with 173 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.