Full Code of maxogden/electron-microscope for AI

Repository: maxogden/electron-microscope
Branch: master
Commit: ce9245fb3d85
Files: 14
Total size: 16.4 KB

Directory structure:
gitextract_hh6ufl48/

├── .gitignore
├── collaborators.md
├── contributing.md
├── examples/
│   └── hoyt.js
├── index.js
├── package.json
├── readme.md
├── renderer.js
├── test/
│   ├── cats.html
│   ├── cool.html
│   ├── index.html
│   └── test.js
├── webview.js
└── window.html

================================================
FILE CONTENTS
================================================

================================================
FILE: .gitignore
================================================
node_modules
socket-client-bundle.js

================================================
FILE: collaborators.md
================================================
## Collaborators

electron-microscope is only possible due to the excellent work of the following collaborators:

<table><tbody><tr><th align="left">maxogden</th><td><a href="https://github.com/maxogden">GitHub/maxogden</a></td></tr>
<tr><th align="left">karissa</th><td><a href="https://github.com/karissa">GitHub/karissa</a></td></tr>
</tbody></table>
Error: Could not addkarissa on npm


================================================
FILE: contributing.md
================================================
## tests

Electron doesn't run on travis (to my knowledge), so please run `npm test` before making a pull request

================================================
FILE: examples/hoyt.js
================================================
var createMicroscope = require('../')
var electron = require('electron')

electron.app.commandLine.appendSwitch('disable-http-cache', true)

electron.app.on('ready', function () {
  createMicroscope(function (err, scope) {
    if (err) throw err
    // clears localstorage state
    scope.window.webContents.session.clearStorageData(function (err) {
      if (err) throw err
      scope.loadURL('http://hoytarboretum.gardenexplorer.org/taxalist.aspx', function (err) {
        if (err) throw err
        console.log('loaded home page')
        loop(scope)
      })
    })
  })
})

function loop (scope) {
  var data = scope.run(clickNextLetter)
  data.on('error', function (e) {
    console.error('Error:', e)
    scope.destroy()
  })
  scope.once('did-fail-load', function (error) {
    console.error('Failed to load', error)
    scope.destroy()
  })
  scope.once('did-finish-load', function () {
    var data = scope.run(getSpecies)
    data.on('data', function (d) {
      console.log('Species', d.toString() ? d.toString() : d)
    })
    data.on('finish', function () {
      console.log('go back')
      scope.window.webContents.executeJavaScript("document.querySelector('webview').goBack()")
      scope.once('did-fail-load', function (error) {
        console.error('Failed to go back', error)
        scope.destroy()
      })
      scope.once('did-finish-load', function () {
        loop(scope)
      })
    })
  })
}

// these two functions are executed on the page, .toString() is called on them!
function getSpecies (send, done) {
  var species = document.querySelectorAll('.taxalist a b')
  for (var i = 0; i < species.length; i++) send(species[i].innerText)
  done()
}

function clickNextLetter (send, done) {
  var links = document.querySelectorAll('.content input[type="button"]')
  var lastClicked = window.localStorage.getItem('last-clicked')
  if (typeof lastClicked === 'undefined') lastClicked = 0
  else lastClicked = +lastClicked
  var link = links[lastClicked]
  if (!link) return done(new Error('clicked all links'))
  window.localStorage.setItem('last-clicked', ++lastClicked)
  link.click()
  done()
}


================================================
FILE: index.js
================================================
var crypto = require('crypto')
var path = require('path')
var electron = require('electron')
var through = require('through2')
var events = require('events')
var inherits = require('inherits')
var debug = require('debug')('electron-microscope')
var BrowserWindow = electron.BrowserWindow

module.exports = Microscope

function Microscope (opts, ready) {
  if (!(this instanceof Microscope)) return new Microscope(opts, ready)
  events.EventEmitter.call(this)
  var self = this
  if (typeof opts === 'function') {
    ready = opts
    opts = {}
  }
  this.opts = opts || {}
  this.window = new BrowserWindow({
    width: 800,
    height: 600,
    show: true
  })
  this.window.loadURL(path.join('file://', __dirname, 'window.html'))
  this.window.webContents.once('did-finish-load', function () {
    debug('did-finish-load window.html')
    ready(null, self)
  })
  this.window.webContents.once('did-fail-load', function (err) {
    debug('did-fail-load window.html', err)
    ready(err)
  })
  electron.ipcMain.on('webview-event', function (event, channel, data) {
    debug('webview-event', channel, data)
    self.emit(channel, data)
  })
}

inherits(Microscope, events.EventEmitter)

Microscope.prototype.loadURL = function (url, cb) {
  debug('start loadURL', url)
  this.window.send('load-url', url)
  if (cb) {
    electron.ipcMain.once('webview-did-finish-load', function (event, error) {
      debug('finish loadURL', url, error || '')
      cb(error)
    })
  }
}

Microscope.prototype.run = function (code) {
  if (typeof code === 'function') code = code.toString()
  var outStream = through()
  var id = crypto.randomBytes(16).toString('hex')
  this.window.send('run', id, code)
  electron.ipcMain.on(id + '-send-data', function (event, data) {
    outStream.push(data)
  })
  electron.ipcMain.once(id + '-done-running', function (event, err) {
    if (err) outStream.destroy(err)
    else outStream.end()
  })
  return outStream
}

Microscope.prototype.destroy = function () {
  this.window.close()
}


================================================
FILE: package.json
================================================
{
  "name": "electron-microscope",
  "description": "Use electron to inspect websites and extract data. useful for automation, testing, web scraping, etc",
  "version": "2.0.0",
  "main": "index.js",
  "scripts": {
    "test": "standard && electron test/test.js"
  },
  "repository": {
    "type": "git",
    "url": "git+https://github.com/maxogden/electron-microscope.git"
  },
  "author": "",
  "license": "BSD-2-Clause",
  "bugs": {
    "url": "https://github.com/maxogden/electron-microscope/issues"
  },
  "homepage": "https://github.com/maxogden/electron-microscope#readme",
  "dependencies": {
    "debug": "^2.2.0",
    "domify": "^1.4.0",
    "inherits": "^2.0.1",
    "through2": "^2.0.1"
  },
  "standard": {
    "ignore": [
      "test/scrapers"
    ]
  },
  "devDependencies": {
    "concat-stream": "^1.5.1",
    "electron-prebuilt": "^0.36.7",
    "pump": "^1.0.1",
    "standard": "^6.0.4",
    "tape": "^4.4.0"
  }
}


================================================
FILE: readme.md
================================================
# electron-microscope

Use [electron](http://electron.atom.io/) to load websites and extract data. Intended for automation, testing, web scraping, etc.

Loads URLs inside an electron [webview tag](https://github.com/atom/electron/blob/master/docs/api/web-view-tag.md), allows you to execute code on them and stream data from the pages back to your main process.

Run this headlessly on Linux using `xvfb-run`.

Please note this is intended to be a fairly low level library that tries to not add much on top of what Electron is doing under the hood, so things that you might think are simple to do can turn out to be relatively complex due to the way web browser events end up working.

## usage

Use this in an electron app:

```js
var electron = require('electron')
var createMicroscope = require('electron-microscope')

electron.app.on('ready', function () {
  createMicroscope(function (err, scope) {
    if (err) throw err
    // use your new microscope
  })
}) 
```

Run it with electron:

```sh
$ npm install electron-prebuilt -g
$ electron my-code.js
```

## examples

See the `test/` and `examples/` folders

## API

### `require('electron-microscope')(options, ready)`

Requiring the module returns a constructor function that you use to create a new instance. Pass it an `options` object and a `ready` callback that will be called with `(error, scope)`. `scope` is your new instance all ready to go.

### scope.window

The electon [BrowserWindow](https://github.com/atom/electron/blob/master/docs/api/browser-window.md) instance, AKA the renderer, which contains the `<webview>` that pages are loaded in.

Currently because there are three node processes at play (main, renderer, webview), to access `webview` APIs you have to go through the `window`, e.g.:

```js
scope.window.webContents.executeJavaScript("document.querySelector('webview').goBack()")
```

### `scope.loadURL(url, cb)`

Load a `url`, and call `cb` with `(err)` when loading is done. If there was a problem loading the page `err` will be the error, otherwise it means it loaded successfully

### `var outputStream = scope.run(code)`

Run `code` on the currently loaded page. Run this after calling `loadURL`. Code must be a string, if it is a `function` then `.toString()` will be called on it. `scope.run` returns a readable stream that emits data generated by your code.

Uses the  [webview.executeJavascript](https://github.com/atom/electron/blob/master/docs/api/web-view-tag.md#webviewexecutejavascriptcode-usergesture) electron API, which doesn't provide an error handling mechamism. Electron microscope wraps your code in a `try/catch` and if an error occurs it will be emitted on the stream. However if you have a syntax error it will likely not catch it so it may appear nothing is happening.

You code must be a function that has this template:

```js
function (send, done) {
  // put your custom code here
  // call 'send(data)' to write data to the stream
  // call 'done()' to end the stream
  // calling send is optional, but you must eventually call done to end the stream
}
```

For example:

```js
var code = `function (send, done) {
  for (var i = 0; i < 5; i++) send(i)
  done()
}`

var output = scope.run(code)

output.on('data', function (data) {
  // will get called for every time send is called above
  // data will be the value passed to send
  // in this case 5 times: 1, 2, 3, 4, 5
})  

output.on('error', function (error) {
  // will get called if your code throws an exception
  // error will be an object with .message and .stack from the thrown error object
})
```

### scope.on('will-navigate', cb)

Emitted the page wants to start navigation. It can happen when the window.location object is changed or a link is clicked in the page.

Calls `cb` with `(url)`, forwarded from [this event](https://github.com/atom/electron/blob/master/docs/api/web-view-tag.md#event-will-navigate).

### scope.on('did-finish-load', cb)

This event is like `did-finish-load`, but fired when the load failed or was cancelled.

Calls `cb` with no arguments, forwarded from [this event](https://github.com/atom/electron/blob/master/docs/api/web-view-tag.md#event-did-finish-load).

### scope.on('did-fail-load', cb)

This event is like `did-finish-load`, but fired when the load failed or was cancelled.

Calls `cb` with `(error)`, forwarded from [this event](https://github.com/atom/electron/blob/master/docs/api/web-view-tag.md#event-did-fail-load).

### scope.on('did-start-loading', cb)

Corresponds to the points in time when the spinner of the tab starts spinning.

Calls `cb` with no arguments, forwarded from [this event](https://github.com/atom/electron/blob/master/docs/api/web-view-tag.md#event-did-start-loading).

### scope.on('did-stop-loading', cb)

Corresponds to the points in time when the spinner of the tab stops spinning.

Calls `cb` with no arguments, forwarded from [this event](https://github.com/atom/electron/blob/master/docs/api/web-view-tag.md#event-did-stop-loading).

### scope.destroy()

Call when you don't want to use the scope anymore. Causes the `browser-window` elecron-microscope uses internally to close, which may cause your electron app to exit if you do not have any other active windows.


================================================
FILE: renderer.js
================================================
var electron = require('electron')
var domify = require('domify')

module.exports = function () {
  electron.ipcRenderer.on('load-url', function (event, url) {
    var webview = domify('<webview src="' + url + '" preload="./webview.js"></webview>')
    document.body.innerHTML = ''
    document.body.appendChild(webview)
    webview.addEventListener('will-navigate', function (newUrl) {
      electron.ipcRenderer.send('webview-event', 'will-navigate', newUrl)
    })
    webview.addEventListener('did-finish-load', function () {
      electron.ipcRenderer.send('webview-event', 'did-finish-load')
      electron.ipcRenderer.send('webview-did-finish-load')
    })
    webview.addEventListener('did-fail-load', function (error) {
      electron.ipcRenderer.send('webview-event', 'did-fail-load', error)
      electron.ipcRenderer.send('webview-did-finish-load', error)
    })
    webview.addEventListener('did-start-loading', function () {
      electron.ipcRenderer.send('webview-event', 'did-start-loading')
    })
    webview.addEventListener('did-stop-loading', function () {
      electron.ipcRenderer.send('webview-event', 'did-stop-loading')
    })
  })

  electron.ipcRenderer.on('run', function (event, id, code) {
    var webview = document.querySelector('webview')
    webview.addEventListener('ipc-message', onIPC)

    function onIPC (event) {
      electron.ipcRenderer.send.apply(null, [id + '-' + event.channel].concat(event.args))
      if (event.channel === 'done-running') {
        webview.removeEventListener('ipc-message', onIPC)
      }
    }

    webview.executeJavaScript(ipcWrap(code))
  })
}

function ipcWrap (code) {
  return `;(function () {
  try {
    (${code})(ELECTRON_MICROSCOPE_SEND, ELECTRON_MICROSCOPE_DONE)
  } catch (err) {
    ELECTRON_MICROSCOPE_DONE(err)
  }
})();
`
}


================================================
FILE: test/cats.html
================================================
<!doctype html>
<html lang="en">
<head>
  <meta charset="utf-8">
  <title>electron-microscope cats page</title>
</head>
<body>
  cats page
  <p><a class="cool-button" href="cool.html">go to cool page</a></p>
</body>
</html>

================================================
FILE: test/cool.html
================================================
<!doctype html>
<html lang="en">
<head>
  <meta charset="utf-8">
  <title>electron-microscope cool test page</title>
</head>
<body>
  <p><div class="foo">cool</div></p>
  <p><a class="index-button" href="index.html">go to home page</a></p>
</body>
</html>

================================================
FILE: test/index.html
================================================
<!doctype html>
<html lang="en">
<head>
  <meta charset="utf-8">
  <title>electron-microscope test page</title>
</head>
<body>
  <p><div class="foo">bar</div></p>
</body>
</html>

================================================
FILE: test/test.js
================================================
var test = require('tape')
var concat = require('concat-stream')
var pump = require('pump')
var createMicroscope = require('../')
var electron = require('electron')
var execspawn = require('npm-execspawn')

electron.app.commandLine.appendSwitch('disable-http-cache', true)

var server, scope

test('wait for electron', function (t) {
  electron.app.on('window-all-closed', function () {
    server.kill()
    server.on('close', function () {
      electron.app.quit()
    })
  })
  electron.app.on('ready', function () {
    t.ok(true, 'electron ready')
    t.end()
  })
})

test('start test server', function (t) {
  server = execspawn('http-server ./ -p 54321', {cwd: __dirname})
  server.stdout.once('data', function (ch) {
    if (ch.toString().indexOf('Starting up') > -1) t.ok(true, 'server started')
    else t.ok(false, ch)
    t.end()
  })
})

test('retrieve the innerText of a div', function (t) {
  createMicroscope(function (err, newScope) {
    scope = newScope
    if (err) t.ifError(err)
    scope.loadURL('http://localhost:54321', function (err) {
      if (err) t.ifError(err)
      var scraper = `function (send, done) {
        send(document.querySelector('.foo').innerText)
        done()
      }`
      var output = scope.run(scraper)
      output.pipe(concat(function (out) {
        t.equal(out.toString(), 'bar', 'output matched')
        t.end()
      }))
    })
  })
})

test('invalid code causes stream error', function (t) {
  scope.loadURL('http://localhost:54321/cool.html', function (err) {
    if (err) t.ifError(err)
    var code = 'function () { donkeys() }'
    var output = scope.run(code)
    var concatter = concat(function (out) {
      t.ok(false, 'should not get here')
    })
    pump(output, concatter, function (err) {
      t.equal(err.message, 'donkeys is not defined', 'got error message')
      t.ok(!!err.stack, 'error has .stack')
      t.end()
    })
  })
})

test('load a new page', function (t) {
  t.plan(4)
  scope.loadURL('http://localhost:54321/cats.html', function (err) {
    if (err) t.ifError(err)
    var scraper = `function (send, done) {
      document.querySelector('a.cool-button').click()
      done()
    }`
    var output = scope.run(scraper)
    output.pipe(concat(function (out) {
      t.equal(out.toString(), '', 'no output')
    }))
    scope.on('will-navigate', function (newUrl) {
      t.equal(newUrl.url, 'http://localhost:54321/cool.html', 'navigating to cool.html')
    })
    scope.on('did-finish-load', function () {
      t.ok(true, 'stopped loading')
      var coolScraper = `function (send, done) {
        send(document.querySelector('.foo').innerText)
        done()
      }`
      var coolOutput = scope.run(coolScraper)
      coolOutput.pipe(concat(function (out) {
        t.equal(out.toString(), 'cool', 'got cool')
        scope.destroy()
      }))
    })
  })
})


================================================
FILE: webview.js
================================================
window.ELECTRON_MICROSCOPE_IPC = require('ipc')

window.ELECTRON_MICROSCOPE_SEND = function send (obj) {
  window.ELECTRON_MICROSCOPE_IPC.sendToHost('send-data', obj)
}

window.ELECTRON_MICROSCOPE_DONE = function done (error) {
  if (error && error.stack && error.message) {
    error = {message: error.message, stack: error.stack}
  }
  window.ELECTRON_MICROSCOPE_IPC.sendToHost('done-running', error)
}


================================================
FILE: window.html
================================================
<!DOCTYPE html>
<html>
  <head>
    <style type="text/css">
      html, body, webview { margin: 0; padding: 0; width: 100%; height: 100%; }
    </style>
    <script type="text/javascript">
      require('./renderer.js')()
    </script>
  </head>
  <body>
  </body>
</html>