Full Code of artpi/roam-research-private-api for AI

Repository: artpi/roam-research-private-api
Branch: master
Commit: 0fe3579dc83a
Files: 14
Total size: 42.2 KB

Directory structure:
gitextract_gez2ylil/

├── .editorconfig
├── .gitignore
├── .prettierrc
├── EvernoteSync.js
├── README.md
├── RoamPrivateApi.js
├── Sync.js
├── examples/
│   ├── cmd.js
│   ├── download.js
│   ├── import-data.js
│   ├── quick_capture.js
│   └── sync_evernote.js
├── package.json
└── secrets-template.json

================================================
FILE CONTENTS
================================================

================================================
FILE: .editorconfig
================================================
# http://editorconfig.org
root = true

[*]
# yes, we use tabs
indent_style = tab

# To control tab size in github diffs
# See https://github.com/isaacs/github/issues/170#issuecomment-150489692
indent_size = 4

end_of_line = lf
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true

[*.md]
trim_trailing_whitespace = false

[*.yml]
indent_size = 2
indent_style = space


================================================
FILE: .gitignore
================================================
tmp
secrets.json
.DS_Store
node_modules


================================================
FILE: .prettierrc
================================================
useTabs: true
tabWidth: 2
printWidth: 100
singleQuote: true
bracketSpacing: true
parenSpacing: true
jsxBracketSameLine: false
semi: true


================================================
FILE: EvernoteSync.js
================================================
const Evernote = require( 'evernote' );
const ENML = require( 'enml-js' );
const RoamSyncAdapter = require( './Sync' );
const moment = require( 'moment' );

class EvernoteSyncAdapter extends RoamSyncAdapter {
	EvernoteClient = null;
	NoteStore = null;
	notebookGuid = '';
	defaultNotebook = '';
	timeOut = 500;
	mapping;
	backlinks = {};
	notesBeingImported = [];

	wrapItem( string, title ) {
		return `<li>${ string }</li>`;
	}
	wrapText( string, title ) {
		const backlinks = [];
		string = this.htmlEntities( string );
		string = string.replace( '{{[[TODO]]}}', '[[TODO]]' ); // <en-todo/> will not achieve the same goal.
		string = string.replace( '{{{[[DONE]]}}}}', '<en-todo checked="true"/>' );
		string = string.replace( /\!\[([^\]]*?)\]\(([^\)]+)\)/g, '<img src="$2"/>' );
		string = string.replace( /\[([^\]]+)\]\((http|evernote)([^\)]+)\)/g, '<a href="$2$3">$1</a>' );
		string = string.replace( /(^|[^"?/])((evernote|http|https|mailto):[a-zA-Z0-9\/.\?\&=;\-_]+)/g, '$1<a href="$2">$2</a>' );
		string = string.replace( /\*\*([^*]+)\*\*/g, '<b>$1</b>' );
		string = string.replace( /__([^_]+)__/g, '<i>$1</i>' );
		string = string.replace( /#?\[\[([^\]]+)\]\]/g, ( match, contents ) => {
			const targetPage = this.titleMapping.get( contents );
			if (
				targetPage &&
				targetPage.uid &&
				this.mapping.get( targetPage.uid )
			) {
				const guid = this.mapping.get( targetPage.uid ).guid;
				const url = this.getNoteUrl( guid );
				backlinks.push( contents );
				return `<a href="${ url }">${ contents }</a>`;
			}
			return match;
		} );
		this.addBacklink( backlinks, title, string );
		return string;
	}

	wrapChildren( childrenString ) {
		childrenString = childrenString.join( '' );
		return `<ul>${ childrenString }</ul>`;
	}
	htmlEntities( str ) {
		return String( str )
			.replace( /&/g, '&amp;' )
			.replace( /</g, '&lt;' )
			.replace( />/g, '&gt;' )
			.replace( /"/g, '&quot;' );
	}
	htmlEntitiesDecode( str ) {
		return String( str )
			.replace( '&amp;', '&' )
			.replace( '&lt;', '<' )
			.replace( '&gt;', '>' )
			.replace( '&quot;', '"' );
	}
	wrapNote( noteBody ) {
		noteBody = noteBody.replace( '&Amp;', '&amp;' ); // Readwise artifact
		var nBody = '<?xml version="1.0" encoding="UTF-8"?>';
		nBody += '<!DOCTYPE en-note SYSTEM "http://xml.evernote.com/pub/enml2.dtd">';
		nBody += '<en-note>' + noteBody;
		nBody += '</en-note>';
		return nBody;
	}

	makeNote( noteTitle, noteBody, url, uid ) {
		// Create note object
		var nBody = this.wrapNote( noteBody );
		let foundNote;

		if ( this.mapping.get( uid ) ) {
			foundNote = Promise.resolve( { 
				totalNotes: 1,
				notes: [ this.mapping.get( uid ) ]
			} );
		} else {
			foundNote = this.findPreviousNote( url );
		}

		const foundNotes = ( notes ) => {
			if ( ! notes.totalNotes ) {
				return Promise.resolve( new Evernote.Types.Note() );
			}
			if ( notes.notes[0].contentLength === nBody.length ) {
				//console.log( '[[' + noteTitle + ']]: content length has not changed, skipping' );
				return Promise.resolve( false );
			}
			// These request we want to rate limit.
			return new Promise( ( resolve, reject ) => setTimeout( () => {
				this.NoteStore.getNote( notes.notes[0].guid, true, false, false, false )
					.catch( ( err ) => {
						console.warn( '[[' + noteTitle + ']] :Took too long to pull note ' );
						if ( err.code === 'ETIMEDOUT' ) {
							this.timeOut = this.timeOut * 2;
							console.log( 'Exponential Backoff increased: ', this.timeOut );
						}
						if ( this.timeOut > 60000 * 5 ) {
							console.error( 'Timeot reached 5 minutes. Exiting' );
							require('process').exit();
						}
						resolve( foundNotes( notes ) );
					} )
					.then( result => resolve( result ) );
			}, this.timeOut ) );

		}

		foundNote.then( foundNotes )
		.then( ( ourNote ) => {
			if ( ! ourNote ) {
				return Promise.resolve( false );
			}
			// Build body of note
			if ( ourNote.content && ourNote.content === nBody ) {
				console.log( '[[' + noteTitle + ']]: has not changed, skipping' );
				return Promise.resolve( ourNote );
			}
			ourNote.content = nBody;

			const attributes = new Evernote.Types.NoteAttributes();
			attributes.contentClass = 'piszek.roam';
			attributes.sourceURL = url;
			attributes.sourceApplication = 'piszek.roam';
			ourNote.attributes = attributes;
			ourNote.title = this.htmlEntities( noteTitle.trim() );

			if ( ourNote.guid ) {
				console.log( '[[' + noteTitle + ']]: updating' );
				ourNote.updated = Date.now();
				return this.NoteStore.updateNote( ourNote ).catch( err => {
					console.log( 'Update note problem', err, nBody );
					return Promise.resolve( false );
				} );
			} else {
				// parentNotebook is optional; if omitted, default notebook is used
				if ( this.notebookGuid ) {
					ourNote.notebookGuid = this.notebookGuid;
				}
				console.log( '[[' + noteTitle + ']] Creating new note ' );
				return this.NoteStore.createNote( ourNote ).then( ( note ) => {
					this.mapping.set( uid, {
						guid: note.guid,
						title: note.title,
						contentLength: note.contentLength
					} );
					return Promise.resolve( note );
				} ).catch( err => {
					console.warn( 'Error creating note:', err );
					Promise.resolve( false );
				});
			}
		} );
	}

	findNotebook() {
		return new Promise( ( resolve, reject ) => {
			this.NoteStore.listNotebooks().then( ( notebooks ) => {
				const filtered = notebooks.filter( ( nb ) => nb.name === 'Roam' );
				const def = notebooks.filter( ( nb ) => nb.defaultNotebook );
				this.defaultNotebook = def[ 0 ].guid
				if ( filtered ) {
					this.notebookGuid = filtered[ 0 ].guid;
					resolve( this.notebookGuid );
				} else {
					console.warn( 'You have to have a notebook named "Roam"' );
					reject( 'You have to have a notebook named "Roam"' );
				}
			} );
		} );
	}
	getNotesToImport() {
		const filter = new Evernote.NoteStore.NoteFilter();
		const spec = new Evernote.NoteStore.NotesMetadataResultSpec();
		spec.includeTitle = false;
		filter.words = '-tag:RoamImported';
		filter.notebookGuid = this.defaultNotebook;
		const batchCount = 100;
		const loadMoreNotes = ( result ) => {
			if ( result.notes ) {
				this.notesBeingImported = this.notesBeingImported.concat(
					result.notes.map( ( note ) =>
						this.NoteStore.getNote( note.guid, true, false, false, false )
					)
				);
			}
			if ( result.startIndex < result.totalNotes ) {
				return this.NoteStore.findNotesMetadata(
					filter,
					result.startIndex + result.notes.length,
					batchCount,
					spec
				).then( loadMoreNotes );
			} else {
				return Promise.resolve( this.mapping );
			}
		};
		return this.NoteStore.findNotesMetadata( filter, 0, batchCount, spec )
			.then( loadMoreNotes )
			.then( () =>
				Promise.all( this.notesBeingImported ).then( ( notes ) => {
					this.notesBeingImported = notes;
					return Promise.resolve( notes );
				} )
			);
	}
	adjustTitle( title, force ) {
		if ( force || title === 'Bez tytułu' || title === 'Untitled Note' ) {
			return moment( new Date() ).format( 'MMMM Do, YYYY' );
		} else {
			return title;
		}
	}
	getRoamPayload() {
		return this.notesBeingImported.map( ( note ) => {
			const md = ENML.PlainTextOfENML( note.content );
			return {
				title: this.adjustTitle( note.title, true ),
				children: [ { string: 'Imported from Evernote: [' + note.title + '](' + this.getNoteUrl( note.guid ) + ')', children: [ { string: md } ] } ],
			};
		} );
	}
	cleanupImportNotes() {
		return Promise.all(
			this.notesBeingImported.map( ( note ) => {
				note.tagGuids = [];
				note.tagNames = [ 'RoamImported' ];
				return this.NoteStore.updateNote( note );
			} )
		);
	}

	findPreviousNote( url ) {
		const filter = new Evernote.NoteStore.NoteFilter();
		const spec = new Evernote.NoteStore.NotesMetadataResultSpec();
		spec.includeContentLength = true;
		spec.includeTitle = true;
		filter.words = `sourceUrl:"${url}" contentClass:piszek.roam`;
		return this.NoteStore.findNotesMetadata( filter, 0, 1, spec );
	}

	loadPreviousNotes() {
		let duplicates = 0;
		const filter = new Evernote.NoteStore.NoteFilter();
		const spec = new Evernote.NoteStore.NotesMetadataResultSpec();
		spec.includeTitle = true;
		spec.includeContentLength = true;
		spec.includeDeleted = false;
		spec.includeAttributes = true;
		filter.words = 'contentClass:piszek.roam';
		const batchCount = 100;

		const loadMoreNotes = ( result ) => {
			if ( result.notes ) {
				result.notes.forEach( ( note ) => {
					if ( ! note.attributes.sourceURL ) {
						console.log( note.title , 'no src url' );
						return;
					}
					const match = note.attributes.sourceURL.match( /https:\/\/roamresearch\.com\/\#\/app\/[a-z]+\/page\/([a-zA-Z0-9_-]+)/);
					if ( ! match ) {
						console.log( note.title ,'no match', note.attributes.sourceURL );
						return;
					}

					const uid = match[1];
						
					if ( this.mapping.get( uid ) && this.mapping.get( uid ).guid !== note.guid ) {
						console.log(
							'[[' + this.mapping.get( uid ).title + ']]',
							'Note is a duplicate ',
							this.getNoteUrl( this.mapping.get( uid ).guid ),
							this.getNoteUrl( note.guid )
						);
						// this.NoteStore.deleteNote( note.guid );
					} else {
						this.mapping.set( uid, {
							guid: note.guid,
							title: note.title,
							contentLength: note.contentLength
						} );
					}
				} );
			}
			if ( result.startIndex < result.totalNotes ) {
				return this.NoteStore.findNotesMetadata(
					filter,
					result.startIndex + result.notes.length,
					batchCount,
					spec
				).then( loadMoreNotes );
			} else {
				// console.log( batchCount );
				return Promise.resolve( this.mapping );
			}
		};
		return this.NoteStore.findNotesMetadata( filter, 0, batchCount, spec ).then( loadMoreNotes );
	}

	addBacklink( titles, target, text ) {
		titles.forEach( ( title ) => {
			if ( ! this.backlinks[ title ] ) {
				this.backlinks[ title ] = [];
			}
			this.backlinks[ title ].push( {
				target: target,
				text: text,
			} );
		} );
	}
	getNoteUrl( guid ) {
		return `evernote:///view/${ this.user.id }/${ this.user.shardId }/${ guid }/${ guid }/`;
	}
	init( prevData = {} ) {
		this.mapping = new Map( prevData );
		this.EvernoteClient = new Evernote.Client( this.credentials );
		this.NoteStore = this.EvernoteClient.getNoteStore();
		const loadNotesPromise = this.loadPreviousNotes();
		loadNotesPromise.then( () => console.log( 'Loaded previous notes: ', this.mapping.size ) );

		return Promise.all( [
			new Promise( ( resolve, reject ) => {
				this.EvernoteClient.getUserStore()
					.getUser()
					.then( ( user ) => {
						this.user = user;
						resolve();
					} );
			} ),
			this.findNotebook().catch( ( err ) => console.log( 'Cannot find notebook Roam:', err ) ),
			loadNotesPromise,
		] );
	}

	sync( pages ) {
		// This can potentially introduce a race condition, but it's unlikely. Famous last words.
		var p = Promise.resolve();
		pages.forEach( ( page ) => {
			p = p
				.then( () => this.syncPage( page ) )
				.catch( ( err ) => console.warn( 'Problem with syncing page ' + page.title, err, page.content ) );
		} );
		return p.then( () => Promise.resolve( this.mapping ) );
	}

	syncPage( page ) {
		let url;
		if ( page.uid ) {
			url = `https://roamresearch.com/#/app/${this.graphName}/page/` + page.uid;
		} else {
			console.warn( "Page must have UIDs and this one does not. Using title as a backup." );
			url = page.title;
		}
		let newContent = page.content;
		if ( this.backlinks[ page.title ] ) {
			const list = this.backlinks[ page.title ]
				.map( ( target ) => {
					let reference = '[[' + target.target + ']]';
					const targetPage = this.titleMapping.get( target.target );
					if (
						targetPage &&
						targetPage.uid &&
						this.mapping.get( targetPage.uid )
					) {
						reference =
							'<a href="' +
							this.getNoteUrl( this.mapping.get( targetPage.uid ).guid ) +
							'">' +
							target.target +
							'</a>';
					}
					return '<li>' + reference + ': ' + target.text + '</li>';
				} )
				.join( '' );

			const backlinks = '<h3>Linked References</h3><ul>' + list + '</ul>';
			newContent = page.content + backlinks;
		}

		return this.makeNote( page.title, newContent, url, page.uid );
	}
}

module.exports = EvernoteSyncAdapter;


================================================
FILE: README.md
================================================
## Roam Private API

This project exposes command line tool (`roam-api`) and a `node` library to connect Roam Research to your other software. You can use it in bash scripts, Github actions, or as a dependency of your project.
## How does it work?

It looks like Roam is not providing a REST API any time soon. If you want to bridge Roam with your other software, you can do so from within Roam (with JavaScript), but that has limited number of use cases.
Without a REST API, this project launches an invisible browser and performs automated actions, just as you would manually. No need to install chrome, this library comes with one. It uses your login and password, so **this won't work if you are using Google login**.
It wraps around import/export functionality and actions exposed via `roamAlphaApi`.
## Command line tool `roam-api`

This package exposes a `roam-api` tool in your system. You can use it to automate Roam and bridge other systems with your Roam graph.

### Installation:
This entire library is build on node, so you need `node v12` and `npm v6` in your system. You can install the package the following way:
```
npm i -g roam-research-private-api
```

Now you can use a variety of commands. All command take the following arguments, which you can also set as environmental variables:
- `-g`, `--graph` or env variable `ROAM_API_GRAPH` - this is your graph name
- `-e`, `--email` or env variable `ROAM_API_EMAIL` - email to log into your Roam
- `-p`, `--password` or env variable `ROAM_API_PASSWORD` - password to your Roam.

#### `roam-api export` will export your Roam graph to a directory of your choice. 

This example will export the graph to your desktop. It will appear as "db.json".
```
roam-api export ~/Desktop
```

It can also push the new version of the graph to an URL of your choosing. That way, you can upload the graph to some other system or use it with Zapier and similar tools.
```
roam-api export ~/Desktop http://secret.url?token=secret_token.
```

#### `roam-api search` will search your Roam graph for a phrase:

```
roam-api search "potatoes"
```

Result will be JSON array of objects `{ blockUid, pageTitle, string }`

#### `roam-api-query` will let you do a full Datalog query.

This will find all block uids in your database which have the content "Import".
```
roam-api query '[:find ?uid :where [?b :block/string "Import"] [?b :block/uid ?uid]]'
```

Check out [this fantastic article](https://www.zsolt.blog/2021/01/Roam-Data-Structure-Query.html) to know more about the Roam data structure.

#### `roam-api create` create a block under specified uid. If no uid is provided, it will be inserted into your daily page:

```
roam-api create "This will be prepended to my daily page"
```

## Library to use in your project.

As mentioned, this is also a library that you can use within your project. Here are examples on how to do so:

- [All the functionality in the roam-api tool](https://github.com/artpi/roam-research-private-api/blob/master/examples/cmd.js)
- [Sync your Roam graph to Evernote](https://github.com/artpi/roam-research-private-api/blob/master/examples/sync_evernote.js) - you can also use command-line utility `roam-evernote-sync`.
- [Import arbitrary data into any note](https://github.com/artpi/roam-research-private-api/blob/master/examples/import-data.js)
- [Send note to Roam using the Quick Capture feature](https://github.com/artpi/roam-research-private-api/blob/master/examples/quick_capture.js)

###

Pull requests welcome and I take no responsibility in case this messes up your Roam Graph :).


================================================
FILE: RoamPrivateApi.js
================================================
const puppeteer = require( 'puppeteer' );
const fs = require( 'fs' );
const path = require('path');
const os = require( 'os' );
const unzip = require( 'node-unzip-2' );
const { isString } = require( 'util' );
const moment = require( 'moment' );

/**
 * This class represents wraps Puppeteer and exposes a few methods useful in manipulating Roam Research.
 */
class RoamPrivateApi {
	options;
	browser;
	page;
	db;
	login;
	pass;

	constructor( db, login, pass, options = { headless: true, folder: null, nodownload: false } ) {
		// If you dont pass folder option, we will use the system tmp directory.
		if ( ! options.folder ) {
			options.folder = os.tmpdir();
		}
		options.folder = fs.realpathSync( options.folder );
		this.db = db;
		this.login = login;
		this.pass = pass;
		this.options = options;
	}

	/**
	 * Run a query on the new Roam Alpha API object.
	 * More about the query syntax: https://www.zsolt.blog/2021/01/Roam-Data-Structure-Query.html
	 * @param {string} query - datalog query.
	 */
	async runQuery( query ) {
		return await this.page.evaluate( ( query ) => {
			if ( ! window.roamAlphaAPI ) {
				return Promise.reject( 'No Roam API detected' );
			}
			const result = window.roamAlphaAPI.q( query );
			console.log( result );
			return Promise.resolve( result );
		}, query );
	}

	/**
	 * Create a block as a child of block.
	 * @param {string} text 
	 * @param {uid} uid - parent UID where block has to be inserted.
	 */
	async createBlock( text, uid ) {
		const result = await this.page.evaluate( ( text, uid ) => {
			if ( ! window.roamAlphaAPI ) {
				return Promise.reject( 'No Roam API detected' );
			}
			const result = window.roamAlphaAPI.createBlock(
				{"location": 
					{"parent-uid": uid, 
					 "order": 0}, 
				 "block": 
					{"string": text}})
			console.log( result );
			return Promise.resolve( result );
		}, text, uid );
		// Let's give time to sync.
		await this.page.waitForTimeout( 1000 );
		return result;
	}

	/**
	 * Delete blocks matching the query. Hass some protections, but
	 * THIS IS VERY UNSAFE. DO NOT USE THIS IF YOU ARE NOT 100% SURE WHAT YOU ARE DOING
	 * @param {string} query - datalog query to find blocks to delete. Has to return block uid.
	 * @param {int} limit - limit deleting to this many blocks. Default is 1.
	 */
	async deleteBlocksMatchingQuery( query, limit ) {
		if ( ! limit ) {
			limit = 1;
		}
		return await this.page.evaluate( ( query, limit ) => {
			if ( ! window.roamAlphaAPI ) {
				return Promise.reject( 'No Roam API detected' );
			}
			const result = window.roamAlphaAPI.q( query );
			console.log( result );
			if ( result.length > 100 ) {
				return Promise.reject( 'Too many results. Is your query ok?' );

			}
			const limited = result.slice( 0, limit );
			limited.forEach( ( block ) => {
				const id = block[0];
				console.log( 'DELETING', id );
				window.roamAlphaAPI.deleteBlock( { block: { uid: id } } );
			} );
			return Promise.resolve( limited );
		}, query, limit );
	}

	/**
	 * Returns a query to find blocks with exact text on the page with title.
	 * Useful with conjuction with deleteBlocksMatchingQuery,
	 * @param {string} text - Exact text in the block.
	 * @param {*} pageTitle - page title to find the blocks in.
	 */
	getQueryToFindBlocksOnPage( text, pageTitle ) {
		text = text.replace( '"', '\"' );
		pageTitle = pageTitle.replace( '"', '\"' );

		return `[:find ?uid
			:where [?b :block/string "${text}"]
				   [?b :block/uid  ?uid]
				   [?b :block/page ?p]
				   [?p :node/title "${pageTitle}"]]`;
	}

	/**
	 * Returns datalog query to find all blocks containing the text.
	 * Returns results in format [[ blockUid, text, pageTitle ]].
	 * @param {string} text - text to search.
	 */
	getQueryToFindBlocks( text ) {
		text = text.replace( '"', '\"' );
		return `[:find ?uid ?string ?title :where
			[?b :block/string ?string]
			[(clojure.string/includes? ?string "${text}")]
			[?b :block/uid  ?uid]
			[?b :block/page ?p]
			[?p :node/title ?title]
		]`;
	}

	/**
	 * When importing in Roam, import leaves an "Import" block.
	 * This removes that from your daily page.
	 * THIS IS UNSAFE since it deletes blocks.
	 */
	async removeImportBlockFromDailyNote() {
		await this.deleteBlocksMatchingQuery(
			this.getQueryToFindBlocksOnPage(
				'Import',
				this.dailyNoteTitle()
			),
			1
		);
		//Lets give time to sync
		await this.page.waitForTimeout( 1000 );
		return;
	}

	/**
	 * Return page title for the current daily note.
	 */
	dailyNoteTitle() {
		return moment( new Date() ).format( 'MMMM Do, YYYY' );
	}
	/**
	 * Return page uid for the current daily note.
	 */
	dailyNoteUid() {
		return moment( new Date() ).format( 'MM-DD-YYYY' );
	}

	/**
	 * Export your Roam database and return the JSON data.
	 * @param {boolean} autoremove - should the zip file be removed after extracting?
	 */
	async getExportData( autoremove ) {
		// Mostly for testing purposes when we want to use a preexisting download.
		if ( ! this.options.nodownload ) {
			await this.logIn();
			await this.downloadExport( this.options.folder );
		}
		const latestExport = this.getLatestFile( this.options.folder );
		const content = await this.getContentsOfRepo( this.options.folder, latestExport );
		if ( autoremove ) {
			fs.unlinkSync( latestExport );
		}
		await this.close();
		return content;
	}
	/**
	 * Logs in to Roam interface.
	 */
	async logIn() {
		if ( this.browser ) {
			return this.browser;
		}
		this.browser = await puppeteer.launch( this.options );
		try {
			this.page = await this.browser.newPage();
			this.page.setDefaultTimeout( 60000 );
			await this.page.goto( 'https://roamresearch.com/#/app/' + this.db );
			await this.page.waitForNavigation();
			await this.page.waitForSelector( 'input[name=email]' );
		} catch ( e ) {
			console.error( 'Cannot load the login screen!' );
			throw e;
		}
		// Login
		await this.page.type( 'input[name=email]', this.login );
		await this.page.type( 'input[name=password]', this.pass );
		await this.page.click( '.bp3-button' );
		await this.page.waitForSelector( '.bp3-icon-more' );
		return;
	}

	/**
	 * Import blocks to your Roam graph
	 * @see examples/import.js.
	 * @param {array} items 
	 */
	async import( items = [] ) {
		const fileName = path.resolve( this.options.folder, 'roam-research-private-api-sync.json' );
		fs.writeFileSync( fileName, JSON.stringify( items ) );
		await this.logIn();
		await this.page.waitForSelector( '.bp3-icon-more' );
		await this.clickMenuItem( 'Import Files' );
		// await this.page.click( '.bp3-icon-more' );
		// // This should contain "Export All"
		// await this.page.waitFor( 2000 );
		// await this.page.click( '.bp3-menu :nth-child(5) a' );
		await this.page.waitForSelector( 'input[type=file]' );
		await this.page.waitForTimeout( 1000 );
		// get the ElementHandle of the selector above
		const inputUploadHandle = await this.page.$( 'input[type=file]' );

		// Sets the value of the file input to fileToUpload
		inputUploadHandle.uploadFile( fileName );
		await this.page.waitForSelector( '.bp3-dialog .bp3-intent-primary' );
		await this.page.click( '.bp3-dialog .bp3-intent-primary' );
		await this.page.waitForTimeout( 3000 );
		await this.removeImportBlockFromDailyNote();
		return;
	}

	/**
	 * Inserts text to your quickcapture.
	 * @param {string} text 
	 */
	async quickCapture( text = [] ) {
		await this.logIn();
		const page = await this.browser.newPage();
		await page.emulate( puppeteer.devices[ 'iPhone X' ] );
		// set user agent (override the default headless User Agent)
		await page.goto( 'https://roamresearch.com/#/app/' + this.db );

		await page.waitForSelector( '#block-input-quick-capture-window-qcapture' );
		if ( isString( text ) ) {
			text = [ text ];
		}

		text.forEach( async function ( t ) {
			await page.type( '#block-input-quick-capture-window-qcapture', t );
			await page.click( 'button.bp3-intent-primary' );
		} );
		await page.waitForTimeout( 500 );
		// page.close();
		await this.close();
		return;
	}

	/**
	 * Click item in the side-menu. This is mostly internal.
	 * @param {string} title 
	 */
	async clickMenuItem( title ) {
		await this.page.click( '.bp3-icon-more' );
		// This should contain "Export All"
		await this.page.waitForTimeout( 1000 );
		await this.page.evaluate( ( title ) => {
			const items = [ ...document.querySelectorAll( '.bp3-menu li a' ) ];
			items.forEach( ( item ) => {
				console.log( item.innerText, title );
				if ( item.innerText === title ) {
					item.click();
					return;
				}
			} );
		}, title );
	}

	/**
	 * Download Roam export to a selected folder.
	 * @param {string} folder 
	 */
	async downloadExport( folder ) {
		await this.page._client.send( 'Page.setDownloadBehavior', {
			behavior: 'allow',
			downloadPath: folder,
		} );
		// Try to download
		// await this.page.goto( 'https://roamresearch.com/#/app/' + this.db );
		// await this.page.waitForNavigation();
		await this.page.waitForSelector( '.bp3-icon-more' );
		await this.clickMenuItem( 'Export All' );
		// await this.page.click( '.bp3-icon-more' );
		// // This should contain "Export All"
		// await this.page.waitFor( 2000 );
		// await this.page.click( '.bp3-menu :nth-child(4) a' );
		//Change markdown to JSON:
		// This should contain markdown
		await this.page.waitForTimeout( 2000 );
		await this.page.click( '.bp3-dialog-container .bp3-popover-wrapper button' );
		// This should contain JSON
		await this.page.waitForTimeout( 2000 );
		await this.page.click( '.bp3-dialog-container .bp3-popover-wrapper .bp3-popover-dismiss' );
		// This should contain "Export All"
		await this.page.waitForTimeout( 2000 );
		await this.page.click( '.bp3-dialog-container .bp3-intent-primary' );

		await this.page.waitForTimeout( 60000 ); // This can take quite some time on slower systems
		// Network idle is a hack to wait until we donwloaded stuff. I don't think it works though.
		await this.page.goto( 'https://news.ycombinator.com/', { waitUntil: 'networkidle2' } );
		return;
	}

	/**
	 * Close the fake browser session.
	 */
	async close() {
		if ( this.browser ) {
			await this.page.waitForTimeout( 1000 );
			await this.browser.close();
			this.browser = null;
		}
		return;
	}

	/**
	 * Get the freshest file in the directory, for finding the newest export.
	 * @param {string} dir 
	 */
	getLatestFile( dir ) {
		const orderReccentFiles = ( dir ) =>
			fs
				.readdirSync( dir )
				.filter( ( f ) => fs.lstatSync( path.resolve( dir, f ) ) && fs.lstatSync( path.resolve( dir, f ) ).isFile() )
				.filter( ( f ) => f.indexOf( 'Roam-Export' ) !== -1 )
				.map( ( file ) => ( { file, mtime: fs.lstatSync( path.resolve( dir, file ) ).mtime } ) )
				.sort( ( a, b ) => b.mtime.getTime() - a.mtime.getTime() );

		const getMostRecentFile = ( dir ) => {
			const files = orderReccentFiles( dir );
			return files.length ? files[ 0 ] : undefined;
		};
		return path.resolve( dir, getMostRecentFile( dir ).file );
	}

	/**
	 * Unzip the export and get the content.
	 * @param {string} dir 
	 * @param {string} file 
	 */
	getContentsOfRepo( dir, file ) {
		return new Promise( ( resolve, reject ) => {
			const stream = fs.createReadStream( file ).pipe( unzip.Parse() );
			stream.on( 'entry', function ( entry ) {
				var fileName = entry.path;
				var type = entry.type; // 'Directory' or 'File'
				var size = entry.size;
				if ( fileName.indexOf( '.json' ) != -1 ) {
					entry.pipe( fs.createWriteStream( path.resolve( dir, 'db.json' ) ) );
				} else {
					entry.autodrain();
				}
			} );
			// Timeouts are here so that the system locks can be removed - takes time on some systems.
			stream.on( 'close', function () {
				setTimeout( function() {
					fs.readFile( path.resolve( dir, 'db.json' ), 'utf8', function ( err, data ) {
						if ( err ) {
							reject( err );
						} else {
							resolve( JSON.parse( data ) );	
						}
					} );
				}, 1000 );
			} );
		} );
	}
}

module.exports = RoamPrivateApi;


================================================
FILE: Sync.js
================================================
class RoamSyncAdapter {
	credentials;
	graphName = '';
	pages = [];
	titleMapping;

	constructor( data, graphName ) {
		this.titleMapping = new Map();
		this.credentials = data;
		this.graphName = graphName;
	}

	sync( pages ) {
		return new Promise( ( resolve, reject ) => {
			console.log( pages );
			resolve( this.titleMapping );
		} );
	}

	wrapItem( string, title ) {
		const intend = ''; // this has to grow
		return (
			intend +
			' - ' +
			string +
			`
		`
		);
	}

	wrapChildren( childrenString, title ) {
		return childrenString.join( '' );
	}

	wrapText( string, title ) {
		return string;
	}
	flattenRoamDB( roamData, level, title ) {
		let ret = '';
		if ( roamData.string ) {
			ret += this.wrapText( roamData.string, title );
		}
		if ( roamData.children ) {
			ret += this.wrapChildren(
				roamData.children.map( ( child ) => this.flattenRoamDB( child, level + 1, title ) )
			);
		}
		return this.wrapItem( ret, title );
	}

	processJSON( newData ) {
		this.pages = newData.map( ( page ) => {
			const newPage = {
				uid: page.uid,
				title: page.title,
				updateTime: page[ 'edit-time' ],
				content: '',
			};
			if ( page.string ) {
				newPage.content = page.string;
			}
			if ( page.children && page.children[ 0 ] ) {
				newPage.content += this.flattenRoamDB( page, 0, page.title );
			}
			this.titleMapping.set( page.title, newPage );
			return newPage;
		} );
		return this.sync( this.pages );
	}
}

module.exports = RoamSyncAdapter;


================================================
FILE: examples/cmd.js
================================================
#!/usr/bin/env node
const yargs = require( 'yargs' );
const fs = require( 'fs' );
const fetch = require( 'node-fetch' );

const argv = yargs
	.option( 'graph', {
		alias: 'g',
		description: 'Your graph name',
		type: 'string',
	} )
	.option( 'email', {
		alias: 'e',
		description: 'Your Roam Email',
		type: 'string',
	} )
	.option( 'password', {
		alias: 'p',
		description: 'Your Roam Password',
		type: 'string',
	} )
	.option( 'debug', {
		description: 'enable debug mode',
		type: 'boolean',
		default: false,
	} )
	.option( 'stdin', {
		alias: 'i',
		description: 'Read from STDIN',
		type: 'boolean',
		default: false,
	} )
	.option( 'removezip', {
		description: 'If downloading the Roam Graph, should the timestamp zip file be removed after downloading?',
		type: 'boolean',
		default: true,
	} )
	.command(
		'query [query]',
		'Query your Roam Graph using datalog syntax',
		() => {},
		( argv ) => {
			let input = '';
			if ( argv.stdin ) {
				input = fs.readFileSync( 0, 'utf-8' );
			} else {
				input = argv['query'];
			}

			if ( ! input || input.length < 3 ) {
				console.warn( 'You have to provide a query at least 3 chars long' );
				return;
			}
			console.log( "Logging in to your Roam and running query:" );
			console.log( input );
			const RoamPrivateApi = require( '../' );
			const api = new RoamPrivateApi( argv.graph, argv.email, argv.password, {
				headless: ! argv.debug,
			} );

            api.logIn()
            .then( () => api.runQuery( input ) )
            .then( result => {
                console.log( JSON.stringify( result, null, 4 ) );
                api.close();
            } );
		}
	)
	.command(
		'search <query>',
		'Query your Roam Graph blocks using simple text search.',
		() => {},
		( argv ) => {
			const RoamPrivateApi = require( '../' );
			const api = new RoamPrivateApi( argv.graph, argv.email, argv.password, {
				headless: ! argv.debug,
			} );

            api.logIn()
            .then( () => api.runQuery( api.getQueryToFindBlocks( argv['query'] ) ) )
            .then( result => {
				result = result.map( result => ( {
					blockUid: result[0],
					pageTitle: result[2],
					string: result[1]
				} ) );
                console.log( JSON.stringify( result, null, 4 ) );
                api.close();
            } );
		}
	)
	.command(
		'create [text] [parentuid]',
		'Append a block to a block with a selected uid. If no uid is provided, block will be appended to the daily page. You can also pass data from stdin.',
		() => {},
		( argv ) => {
			let input = '';
			if ( argv.stdin ) {
				input = fs.readFileSync( 0, 'utf-8' );
			} else {
				input = argv['text'];
			}

			if ( ! input || input.length < 3 ) {
				console.warn( 'You have to provide content at least 3 chars long' );
				return;
			}

			const RoamPrivateApi = require( '../' );
			const api = new RoamPrivateApi( argv.graph, argv.email, argv.password, {
				headless: ! argv.debug,
			} );

			if ( ! argv['parentuid'] ) {
				argv['parentuid'] = api.dailyNoteUid();
			}

			api.logIn()
            .then( () => api.createBlock( input, argv['parentuid'] ) )
            .then( result => api.close() );
		}
	)
	.command(
		'export <dir> [exporturl]',
		'Export your Roam database to a selected directory. If URL is provided, then the concent will be sent by POST request to the specified URL.',
		() => {},
		( argv ) => {
			const RoamPrivateApi = require( '../' );
			const api = new RoamPrivateApi( argv.graph, argv.email, argv.password, {
				headless: ! argv.debug,
				folder: argv['dir']
			} );
			let promises = api.getExportData( argv['removezip'] );
			promises.then( data => console.log( 'Downloaded' ) );
			if ( argv['exporturl'] ) {
				promises.then( data => fetch( argv['exporturl'], {
					method: 'post',
					body: JSON.stringify( {
						graphContent: data,
						graphName: api.db
					} ),
					headers: {'Content-Type': 'application/json'}
				} ) )
				.catch( err => console.log( err ) )
				.then( () => console.log( "Uploaded to export url." ) )
			}
		}
	)
	.help()
	.alias( 'help', 'h' )
	.env( 'ROAM_API' )
	.demandOption(
		[ 'graph', 'email', 'password' ],
		'You need to provide graph name, email and password'
	).argv;


================================================
FILE: examples/download.js
================================================
const RoamPrivateApi = require( '../' );
const secrets = require( '../secrets.json' );

const api = new RoamPrivateApi( secrets.graph, secrets.email, secrets.password, {
	headless: false,
	folder: './tmp/',
	nodownload: false,
} );
api.getExportData().then( data => console.log( 'success', data ) );


================================================
FILE: examples/import-data.js
================================================
const RoamPrivateApi = require( '../' );
const secrets = require( '../secrets.json' );
var fs = require( 'fs' );

const api = new RoamPrivateApi( secrets.graph, secrets.email, secrets.password, {
	headless: false,
	folder: './tmp/',
} );
api.import( [
	{ title: 'test', children: [ { string: 'Test child' }, { string: 'Another test child' } ] },
] );


================================================
FILE: examples/quick_capture.js
================================================
#!/usr/bin/env node
const yargs = require( 'yargs' );
const fs = require( 'fs' );

const argv = yargs
	.option( 'graph', {
		alias: 'g',
		description: 'Your graph name',
		type: 'string',
	} )
	.option( 'email', {
		alias: 'e',
		description: 'Your Roam Email',
		type: 'string',
	} )
	.option( 'password', {
		alias: 'p',
		description: 'Your Roam Password',
		type: 'string',
	} )
	.option( 'debug', {
		description: 'enable debug mode',
		type: 'boolean',
	} )
	.option( 'stdin', {
		alias: 'i',
		description: 'Read from STDIN',
		type: 'boolean',
	} )
	.command(
		'$0',
		'Save Quick capture',
		() => {},
		( argv ) => {
			let input = '';
			if ( argv.stdin ) {
				input = fs.readFileSync( 0, 'utf-8' );
			} else {
				input = argv[ '_' ].join( ' ' );
			}

			if ( ! input || input.length < 3 ) {
				console.warn( 'You have to provide a note at least 3 chars long' );
				return;
			}
			const RoamPrivateApi = require( '../' );
			const api = new RoamPrivateApi( argv.graph, argv.email, argv.password, {
				headless: ! argv.debug,
			} );

			api.quickCapture( [ input ] );
		}
	)
	.help()
	.alias( 'help', 'h' )
	.demandOption(
		[ 'graph', 'email', 'password' ],
		'You need to provide graph name, email and password'
	).argv;


================================================
FILE: examples/sync_evernote.js
================================================
#!/usr/bin/env node
const helpText=`
The command exposed here (roam-evernote-sync sync) will sync your Roam Graph to your Evernote database.
Depending on a few configuration options, it will:
1. Download payload from external URL to import INTO roam (useful for connecting with other services)
2. Take all notes in your default Evernote notebook and import them into your daily note. They will be marked with 'RoamImported' tag to prevent doing so multiple times
3. Export all notes from your Roam and import them to "Roam" notebook in your Evernote account
4. The backlinks will be kept intact, notes will be updated when possible
5. After all that is done, DB will be pushed to external URL ('exporturl') if provided to provide connection for https://deliber.at/roam/wp-roam-block or similar projects

- 'dir' is a directory where database will be downloaded.
- 'mappingcachefile' is a JSON file that provides a cache for Roam UID <-> Evernote GUID mapping. This is used to relieve Evernote API a bit
`;

const yargs = require( 'yargs' );
const fetch = require( 'node-fetch' );
var fs = require( 'fs' ).promises;

const argv = yargs
	.option( 'graph', {
		alias: 'g',
		description: 'Your graph name',
		type: 'string',
	} )
	.option( 'email', {
		alias: 'e',
		description: 'Your Roam Email',
		type: 'string',
	} )
	.option( 'password', {
		alias: 'p',
		description: 'Your Roam Password',
		type: 'string',
	} )
	.option( 'evernote_token', {
		alias: 't',
		description: 'Your Evernote Token',
		type: 'string',
	} )
	.option( 'debug', {
		description: 'enable debug mode',
		type: 'boolean',
		default: false,
	} )
	.option( 'nodownload', {
		description: 'Skip the download of the roam graph. Default no - do download.',
		type: 'boolean',
		default: false,
	} )
	.option( 'nosandbox', {
		description: 'Skip the Chrome Sandbox.',
		type: 'boolean',
		default: false,
	} )
	.option( 'executable', {
		description: 'Executable path to Chromium.',
		type: 'string',
		default: '',
	} )
	.option( 'verbose', {
		alias: 'v',
		description: 'You know, verbose.',
		type: 'boolean',
		default: false,
	} )
	.option( 'privateapiurl', {
		description: 'Additional endpoint that provides data to sync INTO Roam. Has nothing to do with Evernote, its just convenient.',
		type: 'string',
		default: '',
	} )
	.option( 'removezip', {
		description: 'If downloading the Roam Graph, should the timestamp zip file be removed after downloading?',
		type: 'boolean',
		default: true,
	} )
	.command(
		'sync <dir> <mappingcachefile> [exporturl]',
		helpText,
		() => {},
		( argv ) => {

			const RoamPrivateApi = require( '../' );
			const EvernoteSyncAdapter = require( '../EvernoteSync' );
			const options = {
				headless: ! argv.debug,
				nodownload: argv.nodownload,
				folder: argv['dir']
			};
			if ( argv[ 'executable' ] ) {
				options['executablePath'] = argv[ 'executable' ];
			}
			if ( argv[ 'nosandbox' ] ) {
				options['args'] = ['--no-sandbox', '--disable-setuid-sandbox'];
			}

			const e = new EvernoteSyncAdapter( { token: argv.evernoteToken, sandbox: false }, argv.graph );
			const api = new RoamPrivateApi( argv.graph, argv.email, argv.password, options );

			// This downloads the private additional content for my Roam graph, served by other means.
			const importIntoRoam = [];
			if ( argv.privateapiurl ) {
				const private_api = fetch( argv.privateapiurl ).then( response => response.json() );
				private_api.then( data => console.log( 'Private API payload', JSON.stringify( data, null, 2 ) ) );
				importIntoRoam.push( private_api );
			}

			let evernote_to_roam;
			if ( argv.mappingcachefile ) {
				// There is a mapping file.
				evernote_to_roam = fs.readFile( argv.mappingcachefile )
				.then( ( data ) => e.init( JSON.parse( data ) ) )
				.catch( ( err ) => e.init( null ) )
			} else {
				evernote_to_roam = e.init( null );
			}

			// This finds notes IN Evernote to import into Roam:
			evernote_to_roam = evernote_to_roam
				.then( () => e.getNotesToImport() )
				.then( payload => Promise.resolve( e.getRoamPayload( payload ) ) );
				importIntoRoam.push( evernote_to_roam );

			// Let's start the flow with Roam:
			const roamdata = Promise.all( importIntoRoam )
				.then( results => {
					const payload = results[0].concat( results[1] );
					console.log( 'Importing into Roam', JSON.stringify( payload, null, 2 ) );
					if( payload.length > 0 ) {
						return api.import( payload );
					} else {
						return Promise.resolve();
					}
				} )
				.then( () => e.cleanupImportNotes() )
				.then( () => api.getExportData( ! argv.nodownload && argv['removezip'] ) ); // Removing zip is only possible if we downloaded it.

				// We are saving the intermediate step of mapping just in case.
				if ( argv.mappingcachefile ) {
					roamdata.then( data => fs.writeFile( argv.mappingcachefile, JSON.stringify( [ ...e.mapping ], null, 2 ), 'utf8' ) );
				}

				// This will push Roam graph to the URL of your choice - can be WordPress
				if ( argv.exporturl ) {
					roamdata.then( data => fetch( argv.exporturl, {
						method: 'post',
						body: JSON.stringify( {
							graphContent: data,
							graphName: api.db
						} ),
						headers: {'Content-Type': 'application/json'}
					} ) )
					.then( response => response.text() )
					.then( ( data ) => console.log( 'Updated in your remote URL', data ) );
				}

				// This is the actual moment where we sync to Evernote:
				let finish = roamdata.then( ( data ) => e.processJSON( data ) );
				// We are saving the final step of mapping just in case.
				if ( argv.mappingcachefile ) {
					finish = finish.then( data => fs.writeFile( argv.mappingcachefile, JSON.stringify( [ ...e.mapping ], null, 2 ), 'utf8' ) );
				}
				finish.then( () => console.log( 'success' ) );
		}
	)
	.help()
	.alias( 'help', 'h' )
	.env( 'ROAM_API' )
	.demandOption(
		[ 'graph', 'email', 'password' ],
		'You need to provide graph name, email and password'
	).argv;


================================================
FILE: package.json
================================================
{
	"name": "roam-research-private-api",
	"version": "0.9.3",
	"description": "Library that loads your Roam Research graph as a browser and performs tasks as you.",
	"homepage": "https://deliber.at/roam/roam-api/",
	"keywords": [
		"roam",
		"roam-research",
		"evernote",
		"puppeteer"
	],
	"main": "RoamPrivateApi.js",
	"repository": {
		"type": "git",
		"url": "https://github.com/artpi/roam-research-private-api"
	},
	"scripts": {
		"test": "echo \"Error: no test specified\" && exit 1",
		"evernote_sync": "node examples/sync_evernote.js",
		"reformat-files": "./node_modules/.bin/prettier --ignore-path .eslintignore --write \"**/*.{js,jsx,json,ts,tsx}\""
	},
	"bin": {
		"roam-api": "examples/cmd.js",
		"roam-evernote-sync": "examples/sync_evernote.js"
	},
	"author": "Artur Piszek ( piszek.com )",
	"license": "MIT",
	"dependencies": {
		"enml-js": "^0.1.3",
		"evernote": "^2.0.5",
		"moment": "^2.27.0",
		"node-fetch": "^2.6.1",
		"node-unzip-2": "^0.2.8",
		"puppeteer": "^5.5.0",
		"yargs": "^15.4.1"
	},
	"devDependencies": {
		"prettier": "npm:wp-prettier@2.0.5"
	},
	"engines": {
		"node": ">=12.0.0",
		"npm": ">=6.0.0"
	}
}


================================================
FILE: secrets-template.json
================================================
{
	"email": "your-roam-account@email.com",
	"password": "roam-account-password",
	"graph": "your-roam-graph-name",
	"evernote_token": "Evernote developer token from https://www.evernote.com/api/DeveloperToken.action"
}