Repository: maxogden/nugget
Branch: master
Commit: 0211f6675401
Files: 13
Total size: 15.5 KB

Directory structure:
gitextract_ypf1xz91/

├── .gitignore
├── .travis.yml
├── LICENSE
├── bin.js
├── collaborators.md
├── index.js
├── package.json
├── readme.md
├── test/
│   ├── cli.js
│   ├── resume.html
│   ├── resume.js
│   └── test.js
└── usage.txt

================================================
FILE CONTENTS
================================================

================================================
FILE: .gitignore
================================================
.DS_Store
node_modules

================================================
FILE: .travis.yml
================================================
language: node_js

node_js:
  - 6
  - 4

cache:
  directories:
    - node_modules


================================================
FILE: LICENSE
================================================
Copyright 2019 Max Ogden

Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.

2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.

3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


================================================
FILE: bin.js
================================================
#!/usr/bin/env node

var fs = require('fs')
var path = require('path')
var nugget = require('./')
var args = require('minimist')(process.argv.slice(2))

var urls = args._
if (urls.length === 0) {
  console.log(fs.readFileSync(path.join(__dirname, 'usage.txt')).toString())
  process.exit(1)
}

var opts = {
  target: args.o || args.O || args.out,
  dir: args.d || args.dir,
  resume: args.c || args.continue,
  force: args.f || args.force,
  sockets: args.s || args.sockets,
  quiet: args.q || args.quiet,
  frequency: args.frequency ? +args.frequency : null,
  proxy: args.proxy ? args.proxy : null,
  tmpfile: args.t || args.tmpfile,
  strictSSL: args['strict-ssl']
}

nugget(urls, opts, function (err) {
  if (err) {
    console.error('Error:', err)
    process.exit(1)
  }
  process.exit(0)
})


================================================
FILE: collaborators.md
================================================
## Collaborators

nugget is only possible due to the excellent work of the following collaborators:

<table><tbody><tr><th align="left">maxogden</th><td><a href="https://github.com/maxogden">GitHub/maxogden</a></td></tr>
<tr><th align="left">grncdr</th><td><a href="https://github.com/grncdr">GitHub/grncdr</a></td></tr>
<tr><th align="left">mafintosh</th><td><a href="https://github.com/mafintosh">GitHub/mafintosh</a></td></tr>
<tr><th align="left">jlord</th><td><a href="https://github.com/jlord">GitHub/jlord</a></td></tr>
</tbody></table>


================================================
FILE: index.js
================================================
var request = require('request')
var fs = require('fs')
var path = require('path')
var log = require('single-line-log').stdout
var progress = require('progress-stream')
var prettyBytes = require('pretty-bytes')
var throttle = require('throttleit')
var EventEmitter = require('events').EventEmitter
var debug = require('debug')('nugget')

function noop () {}

module.exports = function (urls, opts, cb) {
  if (!Array.isArray(urls)) urls = [urls]
  if (urls.length === 1) opts.singleTarget = true

  var defaultProps = {}

  if (opts.sockets) {
    var sockets = +opts.sockets
    defaultProps.pool = {maxSockets: sockets}
  }

  if (opts.proxy) {
    defaultProps.proxy = opts.proxy
  }

  if (opts.strictSSL !== null) {
    defaultProps.strictSSL = opts.strictSSL
  }

  if (Object.keys(defaultProps).length > 0) {
    request = request.defaults(defaultProps)
  }

  var downloads = []
  var errors = []
  var pending = 0
  var truncated = urls.length * 2 >= (process.stdout.rows - 15)

  urls.forEach(function (url) {
    debug('start dl', url)
    pending++
    var dl = startDownload(url, opts, function done (err) {
      debug('done dl', url, pending)
      if (err) {
        debug('error dl', url, err)
        errors.push(err)
        dl.error = err.message
      }
      if (truncated) {
        var i = downloads.indexOf(dl)
        downloads.splice(i, 1)
        downloads.push(dl)
      }
      if (--pending === 0) {
        render()
        cb(errors.length ? errors : undefined)
      }
    })

    downloads.push(dl)

    dl.on('start', function (progressStream) {
      throttledRender()
    })

    dl.on('progress', function (data) {
      debug('progress', url, data.percentage)

      dl.speed = data.speed
      if (dl.percentage === 100) render()
      else throttledRender()
    })
  })

  var _log = opts.quiet ? noop : log
  render()
  var throttledRender = throttle(render, opts.frequency || 250)

  if (opts.singleTarget) return downloads[0]
  else return downloads

  function render () {
    var height = process.stdout.rows
    var rendered = 0
    var output = ''
    var totalSpeed = 0
    downloads.forEach(function (dl) {
      if (2 * rendered >= height - 15) return
      rendered++
      if (dl.error) {
        output += 'Downloading ' + path.basename(dl.target) + '\n'
        output += 'Error: ' + dl.error + '\n'
        return
      }
      var pct = dl.percentage
      var speed = dl.speed || 0
      var total = dl.fileSize
      totalSpeed += speed
      var bar = Array(Math.floor(45 * pct / 100)).join('=') + '>'
      while (bar.length < 45) bar += ' '
      output += 'Downloading ' + path.basename(dl.target) + '\n' +
      '[' + bar + '] ' + pct.toFixed(1) + '%'
      if (total) output += ' of ' + prettyBytes(total)
      output += ' (' + prettyBytes(speed) + '/s)\n'
    })
    if (rendered < downloads.length) output += '\n... and ' + (downloads.length - rendered) + ' more\n'
    if (downloads.length > 1) output += '\nCombined Speed: ' + prettyBytes(totalSpeed) + '/s\n'
    _log(output)
  }

  function startDownload (url, opts, cb) {
    var targetName = path.basename(url).split('?')[0]
    if (opts.singleTarget && opts.target) targetName = opts.target
    var target = path.resolve(opts.dir || process.cwd(), targetName)
    var origTarget = target
    if (opts.tmpfile) {
      target = target + '.tmp'
    }
    if (opts.resume) {
      resume(url, opts, cb)
    } else {
      download(url, opts, cb)
    }

    var progressEmitter = new EventEmitter()
    progressEmitter.target = target
    progressEmitter.speed = 0
    progressEmitter.percentage = 0

    function onprogress (p) {
      var pct = p.percentage
      progressEmitter.progress = p
      progressEmitter.percentage = pct
      progressEmitter.emit('progress', p)
    }

    return progressEmitter

    function resume (url, opts, cb) {
      var onStat = function (err, stats) {
        if (err && err.code === 'ENOENT') {
          return download(url, opts, cb)
        }
        if (err) {
          return cb(err)
        }
        var offset = stats.size
        var req = request.get(url)

        req.on('error', cb)
        req.on('response', function (resp) {
          resp.destroy()

          var length = parseInt(resp.headers['content-length'], 10)

          // file is already downloaded.
          if (length === offset) {
            onprogress({percentage: 100})
            return cb()
          }

          if (!isNaN(length) && length > offset && /bytes/.test(resp.headers['accept-ranges'])) {
            opts.range = [offset, length]
          }

          download(url, opts, cb)
        })
      }
      if (opts.tmpfile) {
        fs.stat(origTarget, function (err, origStats) {
          if (err && err.code === 'ENOENT') {
            fs.stat(target, onStat)
          } else {
            // file is already downloaded
            onprogress({percentage: 100})
            cb()
          }
        })
      } else {
        fs.stat(target, onStat)
      }
      
    }

    function download (url, opts, cb) {
      var headers = opts.headers || {}
      if (opts.range) {
        headers.Range = 'bytes=' + opts.range[0] + '-' + opts.range[1]
      }
      var read = request(url, { headers: headers })

      read.on('error', cb)
      read.on('response', function (resp) {
        debug('response', url, resp.statusCode)
        if (resp.statusCode > 299 && !opts.force) return cb(new Error('GET ' + url + ' returned ' + resp.statusCode))
        var write = fs.createWriteStream(target, {flags: opts.resume ? 'a' : 'w'})
        write.on('error', cb)
        write.on('finish', async () => {
          if (opts.tmpfile) {
            fs.rename(target, origTarget, cb)
          } else {
            process.nextTick(cb)
          }
        })

        var fullLen
        var contentLen = Number(resp.headers['content-length'])
        var range = resp.headers['content-range']
        if (range) {
          fullLen = Number(range.split('/')[1])
        } else {
          fullLen = contentLen
        }

        progressEmitter.fileSize = fullLen
        if (range) {
          var downloaded = fullLen - contentLen
        }
        var progressStream = progress({ length: fullLen, transferred: downloaded }, onprogress)
        progressEmitter.emit('start', progressStream)

        resp
          .pipe(progressStream)
          .pipe(write)
      })
    }
  }
}


================================================
FILE: package.json
================================================
{
  "name": "nugget",
  "version": "2.2.0",
  "description": "minimalist wget clone written in node. HTTP GETs a file and saves it to the current working directory",
  "main": "index.js",
  "bin": {
    "nugget": "bin.js"
  },
  "scripts": {
    "test": "standard && tape test/*.js"
  },
  "author": "max ogden",
  "license": "BSD",
  "dependencies": {
    "debug": "^2.1.3",
    "minimist": "^1.1.0",
    "pretty-bytes": "^4.0.2",
    "progress-stream": "^1.1.0",
    "request": "^2.45.0",
    "single-line-log": "^1.1.2",
    "throttleit": "0.0.2"
  },
  "devDependencies": {
    "standard": "^6.0.5",
    "tape": "^3.0.1",
    "tape-spawn": "^1.4.2"
  },
  "directories": {
    "test": "test"
  },
  "repository": {
    "type": "git",
    "url": "git://github.com/maxogden/nugget.git"
  },
  "bugs": {
    "url": "https://github.com/maxogden/nugget/issues"
  },
  "homepage": "https://github.com/maxogden/nugget"
}


================================================
FILE: readme.md
================================================
# nugget

Minimalist command line downloader written in node, inspired by wget. HTTP GETs a file and streams it into a file in the current working directory. Specializes at downloading many files in parallel.

[![NPM](https://nodei.co/npm/nugget.png?global=true)](https://nodei.co/npm/nugget/)
![dat](http://img.shields.io/badge/Development%20sponsored%20by-dat-green.svg?style=flat)
[![Travis](http://img.shields.io/travis/maxogden/nugget.svg?style=flat)](https://travis-ci.org/maxogden/nugget)

## installation

```
npm install nugget -g
```

## usage

```
Usage: nugget <urls> [options]
  -o, --output     output filename
  -d, --dir        output parent directory
  -c, --continue   resume aborted download
  -f, --force      ignore response codes > 299
  -s, --sockets    concurrent socket limit (default infinity)
  -q, --quiet      disable logging
  -t, --tmpfile    write files to <name>.tmp while downloading
  --proxy          specify a proxy to use
  --no-strict-ssl  disable strict SSL cehcking
```

### examples

```
nugget http://foo.com/bar.jpg
# downloads bar.jpg and stores it in the current directory
```

or

```
nugget http://foo.com/bar.jpg -O baz.jpg
# saves it as baz.jpg. you can also do lowercase -o
```

if you get a statusCode of 300 or greater nugget will stop. you can force it to stream the response into a file anyway by doing `nugget http://404link.com/file.html -f` or `--force` works too

you can also download multiple files, just pass multiple urls:

![download multiple](multiple.png)

## options

The following options are recognized by nugget:

- `-o|-O|--out` - specify the filename to write to. this only works if you are downloading a single file
- `-d|--dir` - save files in a directory other than the current one.
- `-c|--continue` - resume downloads if a partially complete target file already exists. If the target file exists and is the same size as the remote file, nothing will be done.
- `-f|--force` - force the server response to be saved to the target file, even if it's a non-successful status code.
- `-s|--sockets` - default Infinity. specify the number of http sockets to use at once (this controls concurrency)
- `-q|--quiet` - disable logging
- `-t, --tmpfile` - write files to <name>.tmp while downloading
- `--proxy` - specify a proxy to use
- `--no-strict-ssl` - disable strict ssl


================================================
FILE: test/cli.js
================================================
var fs = require('fs')
var path = require('path')
var spawn = require('tape-spawn')
var test = require('tape')

test('usage', function (t) {
  var child = spawn(t, path.join(__dirname, '..', 'bin.js'))
  child.stdout.match(fs.readFileSync(path.join(__dirname, '..', 'usage.txt')).toString() + '\n')
  child.stderr.empty()
  child.end()
})


================================================
FILE: test/resume.html
================================================
hello

================================================
FILE: test/resume.js
================================================
var fs = require('fs')
var http = require('http')
var nugget = require('../')
var path = require('path')
var test = require('tape')

var data = new Buffer('hello everybody I am the data')

var testServer = http.createServer(function (req, res) {
  if (!req.headers['range']) {
    res.setHeader('content-length', data.length)
    res.setHeader('accept-ranges', 'bytes')
    res.end(data)
  } else {
    var range = req.headers['range'].split('=').pop().split('-').map(function (s) {
      return parseInt(s, 10)
    })
    res.setHeader('content-length', range[1] - range[0])
    res.setHeader('content-range', range[0] + '-' + range[1] + '/' + data.length)
    res.end(data.slice(range[0], range[1]))
  }
})

var target = path.join(__dirname, 'foobar.html')
if (fs.existsSync(target)) fs.unlinkSync(target)

fs.writeFileSync(target, data.slice(0, 10))

testServer.listen(0, function () {
  var port = this.address().port
  test('fetches rest of file', function (t) {
    nugget('http://localhost:' + port + '/foobar.html', {dir: __dirname, resume: true, quiet: true}, function (err) {
      if (err) t.ifErr(err)
      t.ok(fs.existsSync(target), 'downloaded file')
      t.equal(fs.statSync(target).size, data.length, 'file is complete')
      if (fs.existsSync(target)) fs.unlinkSync(target)
      t.end()
      testServer.close()
    })
  })
})


================================================
FILE: test/test.js
================================================
var fs = require('fs')
var http = require('http')
var nugget = require('../')
var path = require('path')
var test = require('tape')

var testServer = http.createServer(function (req, res) {
  res.end('hello')
})

var target = path.join(__dirname, 'resume.html')
if (fs.existsSync(target)) fs.unlinkSync(target)

testServer.listen(0, function () {
  var port = this.address().port
  test('fetches file', function (t) {
    nugget('http://localhost:' + port + '/resume.html', {dir: __dirname, quiet: true}, function (err) {
      if (err) t.ifErr(err)
      t.ok(fs.existsSync(target), 'downloaded file')
      if (fs.existsSync(target)) fs.unlinkSync(target)
      t.end()
    })
  })

  test('has progress events', function (t) {
    var gotProgress = false
    var dl = nugget('http://localhost:' + port + '/resume.html', {dir: __dirname, quiet: true}, function (err) {
      t.notOk(err, 'no error')
      t.ok(gotProgress, 'got progress event')
      t.end()
      testServer.close()
    })
    dl.once('progress', function (data) {
      t.ok(data.hasOwnProperty('percentage'), 'has percentage')
      gotProgress = true
    })
  })
})


================================================
FILE: usage.txt
================================================
Usage: nugget <urls> [options]
  -o, --output     output filename
  -d, --dir        output parent directory
  -c, --continue   resume aborted download
  -f, --force      ignore response codes > 299
  -s, --sockets    concurrent socket limit (default infinity)
  -q, --quiet      disable logging
  -t, --tmpfile    write files to <name>.tmp while downloading
  --proxy          specify a proxy to use
  --no-strict-ssl  disable strict SSL cehcking