[ { "path": ".gitignore", "content": "node_modules\nsocket-client-bundle.js" }, { "path": "collaborators.md", "content": "## Collaborators\n\nelectron-microscope is only possible due to the excellent work of the following collaborators:\n\n\n\n
maxogdenGitHub/maxogden
karissaGitHub/karissa
\nError: Could not addkarissa on npm\n" }, { "path": "contributing.md", "content": "## tests\n\nElectron doesn't run on travis (to my knowledge), so please run `npm test` before making a pull request" }, { "path": "examples/hoyt.js", "content": "var createMicroscope = require('../')\nvar electron = require('electron')\n\nelectron.app.commandLine.appendSwitch('disable-http-cache', true)\n\nelectron.app.on('ready', function () {\n createMicroscope(function (err, scope) {\n if (err) throw err\n // clears localstorage state\n scope.window.webContents.session.clearStorageData(function (err) {\n if (err) throw err\n scope.loadURL('http://hoytarboretum.gardenexplorer.org/taxalist.aspx', function (err) {\n if (err) throw err\n console.log('loaded home page')\n loop(scope)\n })\n })\n })\n})\n\nfunction loop (scope) {\n var data = scope.run(clickNextLetter)\n data.on('error', function (e) {\n console.error('Error:', e)\n scope.destroy()\n })\n scope.once('did-fail-load', function (error) {\n console.error('Failed to load', error)\n scope.destroy()\n })\n scope.once('did-finish-load', function () {\n var data = scope.run(getSpecies)\n data.on('data', function (d) {\n console.log('Species', d.toString() ? d.toString() : d)\n })\n data.on('finish', function () {\n console.log('go back')\n scope.window.webContents.executeJavaScript(\"document.querySelector('webview').goBack()\")\n scope.once('did-fail-load', function (error) {\n console.error('Failed to go back', error)\n scope.destroy()\n })\n scope.once('did-finish-load', function () {\n loop(scope)\n })\n })\n })\n}\n\n// these two functions are executed on the page, .toString() is called on them!\nfunction getSpecies (send, done) {\n var species = document.querySelectorAll('.taxalist a b')\n for (var i = 0; i < species.length; i++) send(species[i].innerText)\n done()\n}\n\nfunction clickNextLetter (send, done) {\n var links = document.querySelectorAll('.content input[type=\"button\"]')\n var lastClicked = window.localStorage.getItem('last-clicked')\n if (typeof lastClicked === 'undefined') lastClicked = 0\n else lastClicked = +lastClicked\n var link = links[lastClicked]\n if (!link) return done(new Error('clicked all links'))\n window.localStorage.setItem('last-clicked', ++lastClicked)\n link.click()\n done()\n}\n" }, { "path": "index.js", "content": "var crypto = require('crypto')\nvar path = require('path')\nvar electron = require('electron')\nvar through = require('through2')\nvar events = require('events')\nvar inherits = require('inherits')\nvar debug = require('debug')('electron-microscope')\nvar BrowserWindow = electron.BrowserWindow\n\nmodule.exports = Microscope\n\nfunction Microscope (opts, ready) {\n if (!(this instanceof Microscope)) return new Microscope(opts, ready)\n events.EventEmitter.call(this)\n var self = this\n if (typeof opts === 'function') {\n ready = opts\n opts = {}\n }\n this.opts = opts || {}\n this.window = new BrowserWindow({\n width: 800,\n height: 600,\n show: true\n })\n this.window.loadURL(path.join('file://', __dirname, 'window.html'))\n this.window.webContents.once('did-finish-load', function () {\n debug('did-finish-load window.html')\n ready(null, self)\n })\n this.window.webContents.once('did-fail-load', function (err) {\n debug('did-fail-load window.html', err)\n ready(err)\n })\n electron.ipcMain.on('webview-event', function (event, channel, data) {\n debug('webview-event', channel, data)\n self.emit(channel, data)\n })\n}\n\ninherits(Microscope, events.EventEmitter)\n\nMicroscope.prototype.loadURL = function (url, cb) {\n debug('start loadURL', url)\n this.window.send('load-url', url)\n if (cb) {\n electron.ipcMain.once('webview-did-finish-load', function (event, error) {\n debug('finish loadURL', url, error || '')\n cb(error)\n })\n }\n}\n\nMicroscope.prototype.run = function (code) {\n if (typeof code === 'function') code = code.toString()\n var outStream = through()\n var id = crypto.randomBytes(16).toString('hex')\n this.window.send('run', id, code)\n electron.ipcMain.on(id + '-send-data', function (event, data) {\n outStream.push(data)\n })\n electron.ipcMain.once(id + '-done-running', function (event, err) {\n if (err) outStream.destroy(err)\n else outStream.end()\n })\n return outStream\n}\n\nMicroscope.prototype.destroy = function () {\n this.window.close()\n}\n" }, { "path": "package.json", "content": "{\n \"name\": \"electron-microscope\",\n \"description\": \"Use electron to inspect websites and extract data. useful for automation, testing, web scraping, etc\",\n \"version\": \"2.0.0\",\n \"main\": \"index.js\",\n \"scripts\": {\n \"test\": \"standard && electron test/test.js\"\n },\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/maxogden/electron-microscope.git\"\n },\n \"author\": \"\",\n \"license\": \"BSD-2-Clause\",\n \"bugs\": {\n \"url\": \"https://github.com/maxogden/electron-microscope/issues\"\n },\n \"homepage\": \"https://github.com/maxogden/electron-microscope#readme\",\n \"dependencies\": {\n \"debug\": \"^2.2.0\",\n \"domify\": \"^1.4.0\",\n \"inherits\": \"^2.0.1\",\n \"through2\": \"^2.0.1\"\n },\n \"standard\": {\n \"ignore\": [\n \"test/scrapers\"\n ]\n },\n \"devDependencies\": {\n \"concat-stream\": \"^1.5.1\",\n \"electron-prebuilt\": \"^0.36.7\",\n \"pump\": \"^1.0.1\",\n \"standard\": \"^6.0.4\",\n \"tape\": \"^4.4.0\"\n }\n}\n" }, { "path": "readme.md", "content": "# electron-microscope\n\nUse [electron](http://electron.atom.io/) to load websites and extract data. Intended for automation, testing, web scraping, etc.\n\nLoads URLs inside an electron [webview tag](https://github.com/atom/electron/blob/master/docs/api/web-view-tag.md), allows you to execute code on them and stream data from the pages back to your main process.\n\nRun this headlessly on Linux using `xvfb-run`.\n\nPlease note this is intended to be a fairly low level library that tries to not add much on top of what Electron is doing under the hood, so things that you might think are simple to do can turn out to be relatively complex due to the way web browser events end up working.\n\n## usage\n\nUse this in an electron app:\n\n```js\nvar electron = require('electron')\nvar createMicroscope = require('electron-microscope')\n\nelectron.app.on('ready', function () {\n createMicroscope(function (err, scope) {\n if (err) throw err\n // use your new microscope\n })\n}) \n```\n\nRun it with electron:\n\n```sh\n$ npm install electron-prebuilt -g\n$ electron my-code.js\n```\n\n## examples\n\nSee the `test/` and `examples/` folders\n\n## API\n\n### `require('electron-microscope')(options, ready)`\n\nRequiring the module returns a constructor function that you use to create a new instance. Pass it an `options` object and a `ready` callback that will be called with `(error, scope)`. `scope` is your new instance all ready to go.\n\n### scope.window\n\nThe electon [BrowserWindow](https://github.com/atom/electron/blob/master/docs/api/browser-window.md) instance, AKA the renderer, which contains the `` that pages are loaded in.\n\nCurrently because there are three node processes at play (main, renderer, webview), to access `webview` APIs you have to go through the `window`, e.g.:\n\n```js\nscope.window.webContents.executeJavaScript(\"document.querySelector('webview').goBack()\")\n```\n\n### `scope.loadURL(url, cb)`\n\nLoad a `url`, and call `cb` with `(err)` when loading is done. If there was a problem loading the page `err` will be the error, otherwise it means it loaded successfully\n\n### `var outputStream = scope.run(code)`\n\nRun `code` on the currently loaded page. Run this after calling `loadURL`. Code must be a string, if it is a `function` then `.toString()` will be called on it. `scope.run` returns a readable stream that emits data generated by your code.\n\nUses the [webview.executeJavascript](https://github.com/atom/electron/blob/master/docs/api/web-view-tag.md#webviewexecutejavascriptcode-usergesture) electron API, which doesn't provide an error handling mechamism. Electron microscope wraps your code in a `try/catch` and if an error occurs it will be emitted on the stream. However if you have a syntax error it will likely not catch it so it may appear nothing is happening.\n\nYou code must be a function that has this template:\n\n```js\nfunction (send, done) {\n // put your custom code here\n // call 'send(data)' to write data to the stream\n // call 'done()' to end the stream\n // calling send is optional, but you must eventually call done to end the stream\n}\n```\n\nFor example:\n\n```js\nvar code = `function (send, done) {\n for (var i = 0; i < 5; i++) send(i)\n done()\n}`\n\nvar output = scope.run(code)\n\noutput.on('data', function (data) {\n // will get called for every time send is called above\n // data will be the value passed to send\n // in this case 5 times: 1, 2, 3, 4, 5\n}) \n\noutput.on('error', function (error) {\n // will get called if your code throws an exception\n // error will be an object with .message and .stack from the thrown error object\n})\n```\n\n### scope.on('will-navigate', cb)\n\nEmitted the page wants to start navigation. It can happen when the window.location object is changed or a link is clicked in the page.\n\nCalls `cb` with `(url)`, forwarded from [this event](https://github.com/atom/electron/blob/master/docs/api/web-view-tag.md#event-will-navigate).\n\n### scope.on('did-finish-load', cb)\n\nThis event is like `did-finish-load`, but fired when the load failed or was cancelled.\n\nCalls `cb` with no arguments, forwarded from [this event](https://github.com/atom/electron/blob/master/docs/api/web-view-tag.md#event-did-finish-load).\n\n### scope.on('did-fail-load', cb)\n\nThis event is like `did-finish-load`, but fired when the load failed or was cancelled.\n\nCalls `cb` with `(error)`, forwarded from [this event](https://github.com/atom/electron/blob/master/docs/api/web-view-tag.md#event-did-fail-load).\n\n### scope.on('did-start-loading', cb)\n\nCorresponds to the points in time when the spinner of the tab starts spinning.\n\nCalls `cb` with no arguments, forwarded from [this event](https://github.com/atom/electron/blob/master/docs/api/web-view-tag.md#event-did-start-loading).\n\n### scope.on('did-stop-loading', cb)\n\nCorresponds to the points in time when the spinner of the tab stops spinning.\n\nCalls `cb` with no arguments, forwarded from [this event](https://github.com/atom/electron/blob/master/docs/api/web-view-tag.md#event-did-stop-loading).\n\n### scope.destroy()\n\nCall when you don't want to use the scope anymore. Causes the `browser-window` elecron-microscope uses internally to close, which may cause your electron app to exit if you do not have any other active windows.\n" }, { "path": "renderer.js", "content": "var electron = require('electron')\nvar domify = require('domify')\n\nmodule.exports = function () {\n electron.ipcRenderer.on('load-url', function (event, url) {\n var webview = domify('')\n document.body.innerHTML = ''\n document.body.appendChild(webview)\n webview.addEventListener('will-navigate', function (newUrl) {\n electron.ipcRenderer.send('webview-event', 'will-navigate', newUrl)\n })\n webview.addEventListener('did-finish-load', function () {\n electron.ipcRenderer.send('webview-event', 'did-finish-load')\n electron.ipcRenderer.send('webview-did-finish-load')\n })\n webview.addEventListener('did-fail-load', function (error) {\n electron.ipcRenderer.send('webview-event', 'did-fail-load', error)\n electron.ipcRenderer.send('webview-did-finish-load', error)\n })\n webview.addEventListener('did-start-loading', function () {\n electron.ipcRenderer.send('webview-event', 'did-start-loading')\n })\n webview.addEventListener('did-stop-loading', function () {\n electron.ipcRenderer.send('webview-event', 'did-stop-loading')\n })\n })\n\n electron.ipcRenderer.on('run', function (event, id, code) {\n var webview = document.querySelector('webview')\n webview.addEventListener('ipc-message', onIPC)\n\n function onIPC (event) {\n electron.ipcRenderer.send.apply(null, [id + '-' + event.channel].concat(event.args))\n if (event.channel === 'done-running') {\n webview.removeEventListener('ipc-message', onIPC)\n }\n }\n\n webview.executeJavaScript(ipcWrap(code))\n })\n}\n\nfunction ipcWrap (code) {\n return `;(function () {\n try {\n (${code})(ELECTRON_MICROSCOPE_SEND, ELECTRON_MICROSCOPE_DONE)\n } catch (err) {\n ELECTRON_MICROSCOPE_DONE(err)\n }\n})();\n`\n}\n" }, { "path": "test/cats.html", "content": "\n\n\n \n electron-microscope cats page\n\n\n cats page\n

go to cool page

\n\n" }, { "path": "test/cool.html", "content": "\n\n\n \n electron-microscope cool test page\n\n\n

cool

\n

go to home page

\n\n" }, { "path": "test/index.html", "content": "\n\n\n \n electron-microscope test page\n\n\n

bar

\n\n" }, { "path": "test/test.js", "content": "var test = require('tape')\nvar concat = require('concat-stream')\nvar pump = require('pump')\nvar createMicroscope = require('../')\nvar electron = require('electron')\nvar execspawn = require('npm-execspawn')\n\nelectron.app.commandLine.appendSwitch('disable-http-cache', true)\n\nvar server, scope\n\ntest('wait for electron', function (t) {\n electron.app.on('window-all-closed', function () {\n server.kill()\n server.on('close', function () {\n electron.app.quit()\n })\n })\n electron.app.on('ready', function () {\n t.ok(true, 'electron ready')\n t.end()\n })\n})\n\ntest('start test server', function (t) {\n server = execspawn('http-server ./ -p 54321', {cwd: __dirname})\n server.stdout.once('data', function (ch) {\n if (ch.toString().indexOf('Starting up') > -1) t.ok(true, 'server started')\n else t.ok(false, ch)\n t.end()\n })\n})\n\ntest('retrieve the innerText of a div', function (t) {\n createMicroscope(function (err, newScope) {\n scope = newScope\n if (err) t.ifError(err)\n scope.loadURL('http://localhost:54321', function (err) {\n if (err) t.ifError(err)\n var scraper = `function (send, done) {\n send(document.querySelector('.foo').innerText)\n done()\n }`\n var output = scope.run(scraper)\n output.pipe(concat(function (out) {\n t.equal(out.toString(), 'bar', 'output matched')\n t.end()\n }))\n })\n })\n})\n\ntest('invalid code causes stream error', function (t) {\n scope.loadURL('http://localhost:54321/cool.html', function (err) {\n if (err) t.ifError(err)\n var code = 'function () { donkeys() }'\n var output = scope.run(code)\n var concatter = concat(function (out) {\n t.ok(false, 'should not get here')\n })\n pump(output, concatter, function (err) {\n t.equal(err.message, 'donkeys is not defined', 'got error message')\n t.ok(!!err.stack, 'error has .stack')\n t.end()\n })\n })\n})\n\ntest('load a new page', function (t) {\n t.plan(4)\n scope.loadURL('http://localhost:54321/cats.html', function (err) {\n if (err) t.ifError(err)\n var scraper = `function (send, done) {\n document.querySelector('a.cool-button').click()\n done()\n }`\n var output = scope.run(scraper)\n output.pipe(concat(function (out) {\n t.equal(out.toString(), '', 'no output')\n }))\n scope.on('will-navigate', function (newUrl) {\n t.equal(newUrl.url, 'http://localhost:54321/cool.html', 'navigating to cool.html')\n })\n scope.on('did-finish-load', function () {\n t.ok(true, 'stopped loading')\n var coolScraper = `function (send, done) {\n send(document.querySelector('.foo').innerText)\n done()\n }`\n var coolOutput = scope.run(coolScraper)\n coolOutput.pipe(concat(function (out) {\n t.equal(out.toString(), 'cool', 'got cool')\n scope.destroy()\n }))\n })\n })\n})\n" }, { "path": "webview.js", "content": "window.ELECTRON_MICROSCOPE_IPC = require('ipc')\n\nwindow.ELECTRON_MICROSCOPE_SEND = function send (obj) {\n window.ELECTRON_MICROSCOPE_IPC.sendToHost('send-data', obj)\n}\n\nwindow.ELECTRON_MICROSCOPE_DONE = function done (error) {\n if (error && error.stack && error.message) {\n error = {message: error.message, stack: error.stack}\n }\n window.ELECTRON_MICROSCOPE_IPC.sendToHost('done-running', error)\n}\n" }, { "path": "window.html", "content": "\n\n \n \n \n \n \n \n\n" } ]