[
{
"path": ".gitignore",
"content": ".DS_Store\nnode_modules"
},
{
"path": ".travis.yml",
"content": "language: node_js\n\nnode_js:\n - 6\n - 4\n\ncache:\n directories:\n - node_modules\n"
},
{
"path": "LICENSE",
"content": "Copyright 2019 Max Ogden\n\nRedistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:\n\n1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.\n\n2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.\n\n3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n"
},
{
"path": "bin.js",
"content": "#!/usr/bin/env node\n\nvar fs = require('fs')\nvar path = require('path')\nvar nugget = require('./')\nvar args = require('minimist')(process.argv.slice(2))\n\nvar urls = args._\nif (urls.length === 0) {\n console.log(fs.readFileSync(path.join(__dirname, 'usage.txt')).toString())\n process.exit(1)\n}\n\nvar opts = {\n target: args.o || args.O || args.out,\n dir: args.d || args.dir,\n resume: args.c || args.continue,\n force: args.f || args.force,\n sockets: args.s || args.sockets,\n quiet: args.q || args.quiet,\n frequency: args.frequency ? +args.frequency : null,\n proxy: args.proxy ? args.proxy : null,\n tmpfile: args.t || args.tmpfile,\n strictSSL: args['strict-ssl']\n}\n\nnugget(urls, opts, function (err) {\n if (err) {\n console.error('Error:', err)\n process.exit(1)\n }\n process.exit(0)\n})\n"
},
{
"path": "collaborators.md",
"content": "## Collaborators\n\nnugget is only possible due to the excellent work of the following collaborators:\n\n\n"
},
{
"path": "index.js",
"content": "var request = require('request')\nvar fs = require('fs')\nvar path = require('path')\nvar log = require('single-line-log').stdout\nvar progress = require('progress-stream')\nvar prettyBytes = require('pretty-bytes')\nvar throttle = require('throttleit')\nvar EventEmitter = require('events').EventEmitter\nvar debug = require('debug')('nugget')\n\nfunction noop () {}\n\nmodule.exports = function (urls, opts, cb) {\n if (!Array.isArray(urls)) urls = [urls]\n if (urls.length === 1) opts.singleTarget = true\n\n var defaultProps = {}\n\n if (opts.sockets) {\n var sockets = +opts.sockets\n defaultProps.pool = {maxSockets: sockets}\n }\n\n if (opts.proxy) {\n defaultProps.proxy = opts.proxy\n }\n\n if (opts.strictSSL !== null) {\n defaultProps.strictSSL = opts.strictSSL\n }\n\n if (Object.keys(defaultProps).length > 0) {\n request = request.defaults(defaultProps)\n }\n\n var downloads = []\n var errors = []\n var pending = 0\n var truncated = urls.length * 2 >= (process.stdout.rows - 15)\n\n urls.forEach(function (url) {\n debug('start dl', url)\n pending++\n var dl = startDownload(url, opts, function done (err) {\n debug('done dl', url, pending)\n if (err) {\n debug('error dl', url, err)\n errors.push(err)\n dl.error = err.message\n }\n if (truncated) {\n var i = downloads.indexOf(dl)\n downloads.splice(i, 1)\n downloads.push(dl)\n }\n if (--pending === 0) {\n render()\n cb(errors.length ? errors : undefined)\n }\n })\n\n downloads.push(dl)\n\n dl.on('start', function (progressStream) {\n throttledRender()\n })\n\n dl.on('progress', function (data) {\n debug('progress', url, data.percentage)\n\n dl.speed = data.speed\n if (dl.percentage === 100) render()\n else throttledRender()\n })\n })\n\n var _log = opts.quiet ? noop : log\n render()\n var throttledRender = throttle(render, opts.frequency || 250)\n\n if (opts.singleTarget) return downloads[0]\n else return downloads\n\n function render () {\n var height = process.stdout.rows\n var rendered = 0\n var output = ''\n var totalSpeed = 0\n downloads.forEach(function (dl) {\n if (2 * rendered >= height - 15) return\n rendered++\n if (dl.error) {\n output += 'Downloading ' + path.basename(dl.target) + '\\n'\n output += 'Error: ' + dl.error + '\\n'\n return\n }\n var pct = dl.percentage\n var speed = dl.speed || 0\n var total = dl.fileSize\n totalSpeed += speed\n var bar = Array(Math.floor(45 * pct / 100)).join('=') + '>'\n while (bar.length < 45) bar += ' '\n output += 'Downloading ' + path.basename(dl.target) + '\\n' +\n '[' + bar + '] ' + pct.toFixed(1) + '%'\n if (total) output += ' of ' + prettyBytes(total)\n output += ' (' + prettyBytes(speed) + '/s)\\n'\n })\n if (rendered < downloads.length) output += '\\n... and ' + (downloads.length - rendered) + ' more\\n'\n if (downloads.length > 1) output += '\\nCombined Speed: ' + prettyBytes(totalSpeed) + '/s\\n'\n _log(output)\n }\n\n function startDownload (url, opts, cb) {\n var targetName = path.basename(url).split('?')[0]\n if (opts.singleTarget && opts.target) targetName = opts.target\n var target = path.resolve(opts.dir || process.cwd(), targetName)\n var origTarget = target\n if (opts.tmpfile) {\n target = target + '.tmp'\n }\n if (opts.resume) {\n resume(url, opts, cb)\n } else {\n download(url, opts, cb)\n }\n\n var progressEmitter = new EventEmitter()\n progressEmitter.target = target\n progressEmitter.speed = 0\n progressEmitter.percentage = 0\n\n function onprogress (p) {\n var pct = p.percentage\n progressEmitter.progress = p\n progressEmitter.percentage = pct\n progressEmitter.emit('progress', p)\n }\n\n return progressEmitter\n\n function resume (url, opts, cb) {\n var onStat = function (err, stats) {\n if (err && err.code === 'ENOENT') {\n return download(url, opts, cb)\n }\n if (err) {\n return cb(err)\n }\n var offset = stats.size\n var req = request.get(url)\n\n req.on('error', cb)\n req.on('response', function (resp) {\n resp.destroy()\n\n var length = parseInt(resp.headers['content-length'], 10)\n\n // file is already downloaded.\n if (length === offset) {\n onprogress({percentage: 100})\n return cb()\n }\n\n if (!isNaN(length) && length > offset && /bytes/.test(resp.headers['accept-ranges'])) {\n opts.range = [offset, length]\n }\n\n download(url, opts, cb)\n })\n }\n if (opts.tmpfile) {\n fs.stat(origTarget, function (err, origStats) {\n if (err && err.code === 'ENOENT') {\n fs.stat(target, onStat)\n } else {\n // file is already downloaded\n onprogress({percentage: 100})\n cb()\n }\n })\n } else {\n fs.stat(target, onStat)\n }\n \n }\n\n function download (url, opts, cb) {\n var headers = opts.headers || {}\n if (opts.range) {\n headers.Range = 'bytes=' + opts.range[0] + '-' + opts.range[1]\n }\n var read = request(url, { headers: headers })\n\n read.on('error', cb)\n read.on('response', function (resp) {\n debug('response', url, resp.statusCode)\n if (resp.statusCode > 299 && !opts.force) return cb(new Error('GET ' + url + ' returned ' + resp.statusCode))\n var write = fs.createWriteStream(target, {flags: opts.resume ? 'a' : 'w'})\n write.on('error', cb)\n write.on('finish', async () => {\n if (opts.tmpfile) {\n fs.rename(target, origTarget, cb)\n } else {\n process.nextTick(cb)\n }\n })\n\n var fullLen\n var contentLen = Number(resp.headers['content-length'])\n var range = resp.headers['content-range']\n if (range) {\n fullLen = Number(range.split('/')[1])\n } else {\n fullLen = contentLen\n }\n\n progressEmitter.fileSize = fullLen\n if (range) {\n var downloaded = fullLen - contentLen\n }\n var progressStream = progress({ length: fullLen, transferred: downloaded }, onprogress)\n progressEmitter.emit('start', progressStream)\n\n resp\n .pipe(progressStream)\n .pipe(write)\n })\n }\n }\n}\n"
},
{
"path": "package.json",
"content": "{\n \"name\": \"nugget\",\n \"version\": \"2.2.0\",\n \"description\": \"minimalist wget clone written in node. HTTP GETs a file and saves it to the current working directory\",\n \"main\": \"index.js\",\n \"bin\": {\n \"nugget\": \"bin.js\"\n },\n \"scripts\": {\n \"test\": \"standard && tape test/*.js\"\n },\n \"author\": \"max ogden\",\n \"license\": \"BSD\",\n \"dependencies\": {\n \"debug\": \"^2.1.3\",\n \"minimist\": \"^1.1.0\",\n \"pretty-bytes\": \"^4.0.2\",\n \"progress-stream\": \"^1.1.0\",\n \"request\": \"^2.45.0\",\n \"single-line-log\": \"^1.1.2\",\n \"throttleit\": \"0.0.2\"\n },\n \"devDependencies\": {\n \"standard\": \"^6.0.5\",\n \"tape\": \"^3.0.1\",\n \"tape-spawn\": \"^1.4.2\"\n },\n \"directories\": {\n \"test\": \"test\"\n },\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git://github.com/maxogden/nugget.git\"\n },\n \"bugs\": {\n \"url\": \"https://github.com/maxogden/nugget/issues\"\n },\n \"homepage\": \"https://github.com/maxogden/nugget\"\n}\n"
},
{
"path": "readme.md",
"content": "# nugget\n\nMinimalist command line downloader written in node, inspired by wget. HTTP GETs a file and streams it into a file in the current working directory. Specializes at downloading many files in parallel.\n\n[](https://nodei.co/npm/nugget/)\n\n[](https://travis-ci.org/maxogden/nugget)\n\n## installation\n\n```\nnpm install nugget -g\n```\n\n## usage\n\n```\nUsage: nugget [options]\n -o, --output output filename\n -d, --dir output parent directory\n -c, --continue resume aborted download\n -f, --force ignore response codes > 299\n -s, --sockets concurrent socket limit (default infinity)\n -q, --quiet disable logging\n -t, --tmpfile write files to .tmp while downloading\n --proxy specify a proxy to use\n --no-strict-ssl disable strict SSL cehcking\n```\n\n### examples\n\n```\nnugget http://foo.com/bar.jpg\n# downloads bar.jpg and stores it in the current directory\n```\n\nor\n\n```\nnugget http://foo.com/bar.jpg -O baz.jpg\n# saves it as baz.jpg. you can also do lowercase -o\n```\n\nif you get a statusCode of 300 or greater nugget will stop. you can force it to stream the response into a file anyway by doing `nugget http://404link.com/file.html -f` or `--force` works too\n\nyou can also download multiple files, just pass multiple urls:\n\n\n\n## options\n\nThe following options are recognized by nugget:\n\n- `-o|-O|--out` - specify the filename to write to. this only works if you are downloading a single file\n- `-d|--dir` - save files in a directory other than the current one.\n- `-c|--continue` - resume downloads if a partially complete target file already exists. If the target file exists and is the same size as the remote file, nothing will be done.\n- `-f|--force` - force the server response to be saved to the target file, even if it's a non-successful status code.\n- `-s|--sockets` - default Infinity. specify the number of http sockets to use at once (this controls concurrency)\n- `-q|--quiet` - disable logging\n- `-t, --tmpfile` - write files to .tmp while downloading\n- `--proxy` - specify a proxy to use\n- `--no-strict-ssl` - disable strict ssl\n"
},
{
"path": "test/cli.js",
"content": "var fs = require('fs')\nvar path = require('path')\nvar spawn = require('tape-spawn')\nvar test = require('tape')\n\ntest('usage', function (t) {\n var child = spawn(t, path.join(__dirname, '..', 'bin.js'))\n child.stdout.match(fs.readFileSync(path.join(__dirname, '..', 'usage.txt')).toString() + '\\n')\n child.stderr.empty()\n child.end()\n})\n"
},
{
"path": "test/resume.html",
"content": "hello"
},
{
"path": "test/resume.js",
"content": "var fs = require('fs')\nvar http = require('http')\nvar nugget = require('../')\nvar path = require('path')\nvar test = require('tape')\n\nvar data = new Buffer('hello everybody I am the data')\n\nvar testServer = http.createServer(function (req, res) {\n if (!req.headers['range']) {\n res.setHeader('content-length', data.length)\n res.setHeader('accept-ranges', 'bytes')\n res.end(data)\n } else {\n var range = req.headers['range'].split('=').pop().split('-').map(function (s) {\n return parseInt(s, 10)\n })\n res.setHeader('content-length', range[1] - range[0])\n res.setHeader('content-range', range[0] + '-' + range[1] + '/' + data.length)\n res.end(data.slice(range[0], range[1]))\n }\n})\n\nvar target = path.join(__dirname, 'foobar.html')\nif (fs.existsSync(target)) fs.unlinkSync(target)\n\nfs.writeFileSync(target, data.slice(0, 10))\n\ntestServer.listen(0, function () {\n var port = this.address().port\n test('fetches rest of file', function (t) {\n nugget('http://localhost:' + port + '/foobar.html', {dir: __dirname, resume: true, quiet: true}, function (err) {\n if (err) t.ifErr(err)\n t.ok(fs.existsSync(target), 'downloaded file')\n t.equal(fs.statSync(target).size, data.length, 'file is complete')\n if (fs.existsSync(target)) fs.unlinkSync(target)\n t.end()\n testServer.close()\n })\n })\n})\n"
},
{
"path": "test/test.js",
"content": "var fs = require('fs')\nvar http = require('http')\nvar nugget = require('../')\nvar path = require('path')\nvar test = require('tape')\n\nvar testServer = http.createServer(function (req, res) {\n res.end('hello')\n})\n\nvar target = path.join(__dirname, 'resume.html')\nif (fs.existsSync(target)) fs.unlinkSync(target)\n\ntestServer.listen(0, function () {\n var port = this.address().port\n test('fetches file', function (t) {\n nugget('http://localhost:' + port + '/resume.html', {dir: __dirname, quiet: true}, function (err) {\n if (err) t.ifErr(err)\n t.ok(fs.existsSync(target), 'downloaded file')\n if (fs.existsSync(target)) fs.unlinkSync(target)\n t.end()\n })\n })\n\n test('has progress events', function (t) {\n var gotProgress = false\n var dl = nugget('http://localhost:' + port + '/resume.html', {dir: __dirname, quiet: true}, function (err) {\n t.notOk(err, 'no error')\n t.ok(gotProgress, 'got progress event')\n t.end()\n testServer.close()\n })\n dl.once('progress', function (data) {\n t.ok(data.hasOwnProperty('percentage'), 'has percentage')\n gotProgress = true\n })\n })\n})\n"
},
{
"path": "usage.txt",
"content": "Usage: nugget [options]\n -o, --output output filename\n -d, --dir output parent directory\n -c, --continue resume aborted download\n -f, --force ignore response codes > 299\n -s, --sockets concurrent socket limit (default infinity)\n -q, --quiet disable logging\n -t, --tmpfile write files to .tmp while downloading\n --proxy specify a proxy to use\n --no-strict-ssl disable strict SSL cehcking\n"
}
]