Repository: apla/node-clickhouse
Branch: master
Commit: 46dc6b70d42c
Files: 20
Total size: 78.7 KB
Directory structure:
gitextract_73vqxqjg/
├── .gitignore
├── .travis.yml
├── LICENSE
├── NOTES.md
├── README.md
├── package.json
├── src/
│ ├── clickhouse.js
│ ├── consts.js
│ ├── legacy-support.js
│ ├── parse-error.js
│ ├── process-db-value.js
│ └── streams.js
└── test/
├── 01-simulated.js
├── 02-real-server.js
├── 03-errors.js
├── 04-select.js
├── 05-insert.js
├── 06-encode-value.js
├── 07-compat.js
└── 90-torture.js
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitignore
================================================
# Logs
logs
*.log
npm-debug.log*
# Runtime data
pids
*.pid
*.seed
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage used by tools like istanbul
coverage
coverage.*
# nyc test coverage
.nyc_output
# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# node-waf configuration
.lock-wscript
# Compiled binary addons (http://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules
jspm_packages
# Optional npm cache directory
.npm
# Optional REPL history
.node_repl_history
# OS-specific files
.DS_Store
.Trash*
.fseventsd
.Spotlight*
================================================
FILE: .travis.yml
================================================
language: node_js
matrix:
include:
# - node_js: '0.10'
# env: _CXXAUTO=1
# - node_js: '0.12'
# env: _CXXAUTO=1
# - node_js: '4'
# env: CXX=g++-4.8
- node_js: '6'
# env: CXX=g++-4.8
- node_js: '8'
# env: CXX=g++-4.8
- node_js: '10'
dist: trusty
sudo: required
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
# - gcc-4.8
# - g++-4.8
before_script:
- sudo apt-key adv --keyserver keyserver.ubuntu.com --recv E0C56BD4
- echo "deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" | sudo tee -a /etc/apt/sources.list
- sudo apt-get update -qq
- sudo apt-get install clickhouse-server-common clickhouse-client -y
- sudo sed -i -- 's/listen_host>::/listen_host>0.0.0.0/g' /etc/clickhouse-server/config.xml
- sudo service clickhouse-server start
- sudo netstat -ltn
- (tail -n 100 /var/log/clickhouse-server/clickhouse-server.err.log || exit 0)
- (tail -n 100 /var/log/clickhouse-server/clickhouse-server.log || exit 0)
- (id metrika || exit 0)
- (ls -la /var/lib/clickhouse/data/default/ || exit 0)
- (ls -la /var/lib/clickhouse/metadata/default/ || exit 0)
- curl -v "http://127.0.0.1:8123/"
- npm run legacy-install
after_script:
- npm run report
================================================
FILE: LICENSE
================================================
The MIT License (MIT)
Copyright (c) 2016 ivan baktsheev
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
================================================
FILE: NOTES.md
================================================
# Notes
## Testing on node 10.x
Seems like memwatch-next is not supported anymore,
so replace it with @airbnb/node-memwatch. Airbnb's version
not supports node < 8 (https://github.com/andywer/leakage/pull/27)
================================================
FILE: README.md
================================================
Simple and powerful interface for [ClickHouse](https://clickhouse.yandex/) [](https://travis-ci.org/apla/node-clickhouse) [](https://codecov.io/gh/apla/node-clickhouse)
===
```sh
npm install @apla/clickhouse
```
Synopsis
---
```javascript
const ClickHouse = require('@apla/clickhouse')
const ch = new ClickHouse({ host, port, user, password })
const stream = ch.query("SELECT 1", (err, data) => {})
stream.pipe(process.stdout)
// promise interface, not recommended for selects
// (requires 'util.promisify' for node < 8, Promise shim for node < 4)
await ch.querying("CREATE DATABASE test")
```
Examples:
- [Selecting large dataset](README.md#selecting-large-dataset)
- [Inserting large dataset](README.md#inserting-large-dataset)
- [Inserting single row](README.md#insert-single-row-of-data)
API
---
### `new ClickHouse(options: Options)`
#### `Options`
| | required | default | description
| :--------------- | :------: | :------------ | :----------
| `host` | ✓ | | Host to connect.
| `user` | | | Authentication user.
| `password` | | | Authentication password.
| `path` (`pathname`) | | `/` | Pathname of ClickHouse server.
| `port` | | `8123` | Server port number.
| `protocol` | | `'http:'` | `'https:'` or `'http:'`.
| `dataObjects` | | `false` | By default (`false`), you'll receive array of values for each row. <br /> If you set `dataObjects: true`, every row will become an object with format: `{ fieldName: fieldValue, … }`. <br /> Alias to `format: 'JSON'`.
| `format` | | `JSONCompact` | Adds the `FORMAT` statement for query if it did not have one. <br /> Specifies format of [selected](https://clickhouse.yandex/docs/en/query_language/select/#format-clause) or [inserted](https://clickhouse.yandex/docs/en/query_language/insert_into/#insert) data. <br /> See ["Formats for input and output data"](https://clickhouse.yandex/docs/en/interfaces/formats/#formats) to find out possible values.
| `queryOptions` | | | Object, can contain any ClickHouse option from [Settings](https://clickhouse.yandex/docs/en/operations/settings/index.html), [Restrictions](https://clickhouse.yandex/docs/en/operations/settings/query_complexity/) and [Permissions](https://clickhouse.yandex/docs/en/operations/settings/permissions_for_queries/). <br /> See [example](README.md#settings-for-connection).
| `readonly` | | `false` | Tells driver to send query with HTTP GET method. Same as [`readonly=1` setting](https://clickhouse.yandex/docs/en/operations/settings/permissions_for_queries/#settings_readonly). [More details](https://clickhouse.yandex/docs/en/interfaces/http/).
| `timeout`, <br /> `headers`, <br /> `agent`, <br /> `localAddress`, <br /> `servername`, <br /> etc… | | | Any [http.request](https://nodejs.org/api/http.html#http_http_request_options_callback) or [https.request](https://nodejs.org/api/https.html#https_https_request_options_callback) options are also available.
<!--
This are dangerous for using by end user
| `syncParser` | | `false` | **Not recommended for large amounts of data!** <br /> Collects all data, then parse entire response. <br /> May be faster, but for large datasets all your dataset goes into memory (actually, entire response + entire dataset).
# Might be completely replaced with promise interface.
| `omitFormat` | | `false` | By default `FORMAT JSONCompact` statement will be added to the query if it did not have it. <br /> You can change disable this behaviour by providing this option.
# Looks like internal option
-->
##### Options example:
```javascript
const ch = new ClickHouse({
host: "clickhouse.msk",
dataObjects: true,
readonly: true,
queryOptions: {
profile: "web",
database: "test",
},
})
```
### `clickHouse.query(query, [options], [callback])`
Sends a query statement to a server.
##### `query: string`
SQL query statement.
##### `options: Options`
The same [`Options`](README.md#options), excluding connection options.
##### `callback: (error, result) => void`
Will be always called upon completion.
##### Returns: [`DuplexStream`](https://nodejs.org/api/stream.html#stream_duplex_and_transform_streams)
It supports [`.pipe`](https://nodejs.org/api/stream.html#stream_readable_pipe_destination_options) to process records. <br/>
You should have at least one error handler listening. Via query callback or via stream `error` event.
| Stream event | Description
| ------------ | -----------
| `'error'` | Query execution finished with error. <br /> If you have both query `callback` and stream `error` listener, you'll have error notification in both listeners.
| `'metadata'` | When a column information is parsed.
| `'data'` | When a row is available.
| `'end'` | When entire response is processed. <blockquote>Regardless of whether there is an `'end'` listener, the query `callback` are always called.</blockquote> <blockquote>You should always listen to `'data'` event together with `'end'` event. <br/>["The 'end' event will not be emitted unless the data is completely consumed."](https://nodejs.org/api/stream.html#stream_event_end) <br/> If you don't need to handle `'data'` event prefer to use only `callback` or [Promise interface](#promise-interface).</blockquote>
##### `stream.supplemental`
After response is processed, you can read a supplemental response data from it, such as row count.
Examples:
- [Selecting with stream](README.md#selecting-with-stream)
- [Inserting with stream](README.md#inserting-with-stream)
### `clickHouse.ping(callback)`
Sends an empty query.
Doesn't requires authorization.
##### `callback: (error, result) => void`
Will be called upon completion.
<br />
## Promise interface
Promise interface **is not recommended** for `INSERT` and `SELECT` queries.
* `INSERT` can't do bulk load data with promise interface.
* `SELECT` will collect entire query result in the memory. See the [Memory size](README.md#memory-size) section.
With promise interface query result are parsed synchronously.
This means that large query result in promise interface:
* Will synchronously block JS thread/event loop.
* May lead to memory leaks in your app due peak GC loads.
Use it only for queries where resulting data size is is known and extremely small.<br/>
The good cases to use it is `DESCRIBE TABLE` or `EXISTS TABLE`
### `clickHouse.querying(query, [options])`
Similar to `ch.query(query)` but collects entire response in memory and resolves with complete query result. <br />
See the [Memory size](README.md#memory-size) section.
##### `options: Options`
The same [`Options`](README.md#options), excluding connection options.
##### Returns: `Promise`
Will be resolved with entire query result.
Example of [promise interface](README.md#promise-interface).
### `clickHouse.pinging()`
Promise interface for [`.ping`](README.md#clickhousepingcallback).
##### Returns: `Promise`
<br />
How it works
-----
### Bulk data loading with `INSERT` statements
`INSERT` can be used for bulk data loading. There is a 2 formats easily implementable
with javascript: CSV and TabSeparated/TSV.
CSV is useful for loading from file, thus you can read and `.pipe` into clickhouse
file contents. <br />
To activate CSV parsing you should set `format` driver option or query `FORMAT` statement to `CSV`:
```javascript
var csvStream = fs.createReadStream('data.csv')
var clickhouseStream = ch.query(statement, { format: CSV })
csvStream.pipe(clickhouseStream)
```
TSV is useful for loading from file and bulk loading from external sources, such as other databases.
Only `\\`, `\t` and `\n` need to be escaped in strings; numbers, nulls,
bools and date objects need some minor processing. You can send prepared TSV data strings
(line ending will be appended automatically), buffers (always passed as is) or Arrays with fields.
Internally, every field will be converted to the format which ClickHouse can accept.
Then escaped and joined with delimiter for the particular format.
If you ever need to store rows (in arrays) and send preformatted data, you can do it.
ClickHouse also supports [JSONEachRow](https://clickhouse.yandex/docs/en/formats/jsoneachrow.html) format
which can be useful to insert javascript objects if you have such recordset.
```js
const stream = ch.query(statement, { format: 'JSONEachRow' })
stream.write(object) // Do write as many times as possible
stream.end() // And don't forget to finish insert query
```
### Memory size
You can read all the records into memory in single call like this:
```javascript
var ch = new ClickHouse({ host: host, port: port })
ch.querying("SELECT number FROM system.numbers LIMIT 10", (err, result) => {
// result will contain all the data you need
})
```
In this case whole JSON response from the server will be read into memory,
then parsed into memory hogging your CPU. Default parser will parse server response
line by line and emits events. This is slower, but much more memory and CPU efficient
for larger datasets.
<br />
## Examples
#### Selecting with stream:
```javascript
const readableStream = ch.query(
'SELECT * FROM system.contributors FORMAT JSONEachRow',
(err, result) => {},
)
const writableStream = fs.createWriteStream('./contributors.json')
readableStream.pipe(writableStream)
```
#### Inserting with stream:
```javascript
const readableStream = fs.createReadStream('./x.csv')
const writableStream = ch.query('INSERT INTO table FORMAT CSV', (err, result) => {})
readableStream.pipe(writableStream)
```
#### Insert single row of data:
```javascript
const ch = new ClickHouse(options)
const writableStream = ch.query(`INSERT INTO table FORMAT TSV`, (err) => {
if (err) {
console.error(err)
}
console.log('Insert complete!')
})
// data will be formatted for you
writableStream.write([1, 2.22, "erbgwerg", new Date()])
// prepare data yourself
writableStream.write("1\t2.22\terbgwerg\t2017-07-17 17:17:17")
writableStream.end()
```
#### Selecting large dataset:
```javascript
const ch = new ClickHouse(options)
// it is better to use stream interface to fetch select results
const stream = ch.query("SELECT * FROM system.numbers LIMIT 10000000")
stream.on('metadata', (columns) => { /* do something with column list */ })
let rows = []
stream.on('data', (row) => rows.push(row))
stream.on('error', (err) => { /* handler error */ })
stream.on('end', () => {
console.log(
rows.length,
stream.supplemental.rows,
stream.supplemental.rows_before_limit_at_least, // how many rows in result are set without windowing
)
})
```
#### Inserting large dataset:
```javascript
const ch = new ClickHouse(options)
// insert from file
const tsvStream = fs.createReadStream('data.tsv')
const clickhouseStream = ch.query('INSERT INTO table FORMAT TSV')
tsvStream.pipe(clickhouseStream)
```
#### Settings for connection:
```javascript
const ch = new ClickHouse({
host: 'clickhouse.msk',
queryOptions: {
database: "test",
profile: "web",
readonly: 2,
force_index_by_date: 1,
max_rows_to_read: 10 * 1e6,
},
})
```
#### Settings for query:
```javascript
const ch = new ClickHouse({ host: 'clickhouse.msk' })
const stream = ch.query('INSERT INTO table FORMAT TSV', {
queryOptions: {
database: "test",
insert_quorum: 2,
},
})
```
#### Promise interface:
```js
const ch = new ClickHouse(options)
// Check connection to server. Doesn't requires authorization.
await ch.pinging()
```
```js
const { data } = await ch.querying("SELECT 1")
// [ [ 1 ] ]
const { data } = await ch.querying("DESCRIBE TABLE system.numbers", { dataObjects: true })
// [ { name: 'number', type: 'UInt64', default_type: '', default_expression: '' } ]
```
================================================
FILE: package.json
================================================
{
"name": "@apla/clickhouse",
"version": "1.6.4",
"description": "ClickHouse database interface",
"main": "src/clickhouse.js",
"scripts": {
"legacy-install": "node ./src/legacy-support.js",
"launch-docker-image": "docker run --rm -d -p 8123:8123 --name clickhouse-server clickhouse/clickhouse-server",
"stop-docker-image": "docker stop clickhouse-server",
"test": "nyc mocha --recursive ./test -R spec",
"report": "nyc report --reporter=lcov > coverage.lcov && codecov",
"simpletest": "mocha --recursive ./test -R spec",
"torturetest": "TORTURE=1 mocha -gc --recursive ./test -R spec"
},
"repository": {
"type": "git",
"url": "git+https://github.com/apla/node-clickhouse.git"
},
"keywords": [
"clickhouse",
"database",
"db"
],
"author": "Ivan Baktsheev <dot.and.thing@gmail.com>",
"license": "MIT",
"bugs": {
"url": "https://github.com/apla/node-clickhouse/issues"
},
"homepage": "https://github.com/apla/node-clickhouse#readme",
"dependencies": {},
"devDependencies": {
"bluebird": "^3.5.0",
"codecov": "^2.2.0",
"mocha": "^2.5.3",
"nyc": "^10.2.0"
},
"engines": {
"node": ">=0.10"
}
}
================================================
FILE: src/clickhouse.js
================================================
var http = require ('http');
var https = require('https');
var url = require ('url');
var qs = require ('querystring');
var util = require ('util');
// var debug = require ('debug')('clickhouse');
require ('./legacy-support');
var RecordStream = require ('./streams').RecordStream;
var JSONStream = require ('./streams').JSONStream;
var parseError = require ('./parse-error');
var LIBRARY_SPECIFIC_OPTIONS = require ('./consts').LIBRARY_SPECIFIC_OPTIONS;
function httpResponseHandler (stream, reqParams, reqData, cb, response) {
var str;
var error;
if (response.statusCode === 200) {
str = Buffer.alloc ? Buffer.alloc (0) : new Buffer (0);
} else {
error = Buffer.alloc ? Buffer.alloc (0) : new Buffer (0);
}
function errorHandler (e) {
var err = parseError (e);
// user should define callback or add event listener for the error event
if (!cb || (cb && stream.listeners ('error').length))
stream.emit ('error', err);
return cb && cb (err);
}
// In case of error, we're just throw away data
response.on ('error', errorHandler);
// TODO: use streaming interface
// from https://github.com/jimhigson/oboe.js
// or https://www.npmjs.com/package/stream-json or
// or https://github.com/creationix/jsonparse
// or implement it youself
var jsonParser = new JSONStream (stream);
var symbolsTransferred = 0;
//another chunk of data has been received, so append it to `str`
response.on ('data', function (chunk) {
symbolsTransferred += chunk.length;
// JSON response
if (
response.headers['content-type']
&& response.headers['content-type'].indexOf ('application/json') === 0
&& !reqData.syncParser
&& chunk.lastIndexOf ("\n") !== -1
&& str
) {
// store in buffer anything after
var newLinePos = chunk.lastIndexOf ("\n");
var remains = chunk.slice (newLinePos + 1);
Buffer.concat([str, chunk.slice (0, newLinePos)])
.toString ('utf8')
.split ("\n")
.forEach (jsonParser);
jsonParser.rows.forEach (function (row) {
// write to readable stream
stream.push (row);
});
jsonParser.rows = [];
str = remains;
// plaintext response
} else if (str) {
str = Buffer.concat ([str, chunk]);
} else {
error = Buffer.concat ([error, chunk]);
}
});
//the whole response has been received, so we just print it out here
response.on('end', function () {
// debug (response.headers);
if (error) {
return errorHandler (error);
}
var data;
var contentType = response.headers['content-type'];
// Early return and stream end in case when content-type means empty body
if (response.statusCode === 200 && (
!contentType
|| contentType.indexOf ('text/plain') === 0
|| contentType.indexOf ('text/html') === 0 // WTF: xenial - no content-type, precise - text/html
)) {
// probably this is a ping response or any other successful response with *empty* body
stream.push (null);
cb && cb (null, str.toString ('utf8'));
return;
}
var supplemental = {};
// we already pushed all the data
if (jsonParser.columns.length) {
try {
supplemental = JSON.parse (jsonParser.supplementalString + str.toString ('utf8'));
} catch (e) {
// TODO
}
stream.supplemental = supplemental;
// end stream
stream.push (null);
cb && cb (null, Object.assign ({}, supplemental, {
meta: jsonParser.columns,
transferred: symbolsTransferred
}));
return;
}
// one shot data parsing, should be much faster for smaller datasets
try {
data = JSON.parse (str.toString ('utf8'));
data.transferred = symbolsTransferred;
if (data.meta) {
stream.emit ('metadata', data.meta);
}
if (data.data) {
// no highWatermark support
data.data.forEach (function (row) {
stream.push (row);
});
}
} catch (e) {
if (!reqData.format || !reqData.format.match (/^(JSON|JSONCompact)$/)) {
data = str.toString ('utf8');
} else {
return errorHandler (e);
}
} finally {
if (!stream.readableEnded) {
stream.push (null);
cb && cb (null, data);
}
}
});
}
function httpRequest (reqParams, reqData, cb) {
if (reqParams.query) {
reqParams.path = (reqParams.pathname || reqParams.path) + '?' + qs.stringify (reqParams.query);
}
var stream = new RecordStream ({
format: reqData.format
});
var requestInstance = reqParams.protocol === 'https:' ? https : http;
var req = requestInstance.request (reqParams, httpResponseHandler.bind (
this, stream, reqParams, reqData, cb
));
req.on ('error', function (e) {
// user should define callback or add event listener for the error event
if (!cb || (cb && stream.listeners ('error').length))
stream.emit ('error', e);
return cb && cb (e);
});
req.on('timeout', function (e) {
req.abort();
})
stream.req = req;
if (reqData.query)
req.write (reqData.query);
if (reqData.finalized) {
req.end();
}
return stream;
}
function ClickHouse (options) {
if (!options) {
console.error ('You must provide at least host name to query ClickHouse');
return null;
}
if (options.constructor === String) {
options = {host: options};
}
this.options = options;
}
ClickHouse.prototype.getReqParams = function () {
var urlObject = {};
// avoid to set defaults - node http module is not happy
for (var name of Object.keys(this.options)) {
if (!LIBRARY_SPECIFIC_OPTIONS.has(name)) {
urlObject[name] = this.options[name];
}
}
if (this.options.hasOwnProperty('user') || this.options.hasOwnProperty('password')) {
urlObject.auth = (this.options.user || 'default')
+ ':'
+ (this.options.password || '')
}
urlObject.method = 'POST';
urlObject.path = urlObject.path || '/';
urlObject.port = urlObject.port || 8123;
return urlObject;
}
ClickHouse.prototype.query = function (chQuery, options, cb) {
chQuery = chQuery.trim ();
if (cb === undefined && options && options.constructor === Function) {
cb = options;
options = undefined;
}
if (!options)
options = {
queryOptions: {}
};
options.omitFormat = options.omitFormat || this.options.omitFormat || false;
options.dataObjects = options.dataObjects || this.options.dataObjects || false;
options.format = options.format || this.options.format || null;
options.readonly = options.readonly || this.options.readonly || this.options.useQueryString || false;
// we're adding `queryOptions` passed for constructor if any
var queryObject = Object.assign ({}, this.options.queryOptions, options.queryOptions);
var formatRegexp = /FORMAT\s+(BlockTabSeparated|CSV|CSVWithNames|JSON|JSONCompact|JSONEachRow|Native|Null|Pretty|PrettyCompact|PrettyCompactMonoBlock|PrettyNoEscapes|PrettyCompactNoEscapes|PrettySpaceNoEscapes|PrettySpace|RowBinary|TabSeparated|TabSeparatedRaw|TabSeparatedWithNames|TabSeparatedWithNamesAndTypes|TSKV|Values|Vertical|XML)/i;
var formatMatch = chQuery.match (formatRegexp);
if (!options.omitFormat && formatMatch) {
options.format = formatMatch[1];
options.omitFormat = true;
}
var reqData = {
syncParser: options.syncParser || this.options.syncParser || false,
finalized: true, // allows to write records into connection stream
};
var reqParams = this.getReqParams ();
var formatEnding = '';
// format should be added for data queries
if (chQuery.match (/^(?:SELECT|WITH|SHOW|DESC|DESCRIBE|EXISTS\s+TABLE)/i)) {
if (!options.format)
options.format = options.dataObjects ? 'JSON' : 'JSONCompact';
} else if (chQuery.match (/^INSERT/i)) {
// There is some variants according to the documentation:
// 1. Values already available in the query: INSERT INTO t VALUES (1),(2),(3)
// 2. Values must me provided with POST data: INSERT INTO t VALUES
// 3. Same as previous but without VALUES keyword: INSERT INTO t FORMAT Values
// 4. Insert from SELECT: INSERT INTO t SELECT…
// we need to handle 2 and 3 and http stream must stay open in that cases
if (chQuery.match (/\s+VALUES\b/i)) {
if (chQuery.match (/\s+VALUES\s*$/i))
reqData.finalized = false;
options.format = 'Values';
options.omitFormat = true;
} else if (chQuery.match (/INSERT\s+INTO\s+\S+\s+(?:\([^\)]+\)\s+)?SELECT/mi)) {
reqData.finalized = true;
options.omitFormat = true;
} else {
reqData.finalized = false;
// Newline is recomended https://clickhouse.yandex/docs/en/query_language/insert_into/#insert
formatEnding = '\n';
if (!chQuery.match (/FORMAT/i)) {
// simplest format to use, only need to escape \t, \\ and \n
options.format = options.format || 'TabSeparated';
} else {
options.omitFormat = true;
}
}
} else {
options.omitFormat = true;
}
reqData.format = options.format;
// use query string to submit ClickHouse query — useful to mock CH server
if (options.readonly) {
queryObject.query = chQuery + ((options.omitFormat) ? '' : ' FORMAT ' + options.format + formatEnding);
reqParams.method = 'GET';
} else {
// Trimmed query still may require `formatEnding` when FORMAT clause specified in query
reqData.query = chQuery + (options.omitFormat ? '' : ' FORMAT ' + options.format) + formatEnding;
reqParams.method = 'POST';
}
reqParams.query = queryObject;
var stream = httpRequest (reqParams, reqData, cb);
return stream;
}
ClickHouse.prototype.querying = function (chQuery, options) {
return new Promise (function (resolve, reject) {
// Force override `syncParser` option when using promise api
const queryOptions = Object.assign ({}, options, {syncParser: true})
var stream = this.query (chQuery, queryOptions, function (err, data) {
if (err)
return reject (err);
resolve (data);
});
}.bind (this));
}
ClickHouse.prototype.ping = function (cb) {
var reqParams = this.getReqParams ();
reqParams.method = 'GET';
var stream = httpRequest (reqParams, {finalized: true}, cb);
return stream;
}
ClickHouse.prototype.pinging = function () {
return new Promise (function (resolve, reject) {
var reqParams = this.getReqParams ();
reqParams.method = 'GET';
httpRequest (reqParams, {finalized: true}, function (err, data) {
if (err)
return reject (err);
resolve (data);
});
}.bind (this));
}
module.exports = ClickHouse;
================================================
FILE: src/consts.js
================================================
exports.LIBRARY_SPECIFIC_OPTIONS = new Set([
// "Auth" shorthand
'user',
'password',
// Database settings go to query string
'queryOptions',
// Driver options
'dataObjects',
'format',
'syncParser',
'omitFormat',
'readonly',
'useQueryString'
]);
================================================
FILE: src/legacy-support.js
================================================
var nodeVer = process.version.substr (1).split ('.');
if (nodeVer[0] >= 6)
return;
var legacyModulesInstallCmd = 'npm install object-assign buffer-indexof-polyfill';
if (process.mainModule === module) {
var spawn = require ('child_process').spawn;
var child = spawn (legacyModulesInstallCmd);
child.stdout.pipe (process.stdout);
child.stderr.pipe (process.stderr);
child.on ('error', function (err) {
process.exit (1);
});
child.on ('exit', function (code) {
process.exit (0);
});
return;
}
try {
if (nodeVer[0] < 4) {
global.Promise = global.Promise || require ('bluebird');
Object.assign = Object.assign || require ('object-assign');
Array.isArray = function(arg) {
return Object.prototype.toString.call(arg) === '[object Array]';
};
}
if (nodeVer[0] < 6) {
require ('buffer-indexof-polyfill');
}
} catch (err) {
console.warn ("You're using outdated nodejs version.");
console.warn ("This module supports nodejs down to the version 0.10, but some legwork required.");
console.warn ("Either install version >= 6, or add dependencies to your own project with `" + legacyModulesInstallCmd + "`");
}
================================================
FILE: src/parse-error.js
================================================
function parseError (e) {
var fields = new Error (e.toString ('utf8'));
e.toString ('utf8')
.split (/\,\s+(?=e\.)/gm)
.map (function (f) {
f = f.trim ().split (/\n/gm).join ('');
var m;
if (m = f.match (/^(?:Error: )?Code: (\d+)$/)) {
fields.code = parseInt (m[1]);
} else if (m = f.match (/^e\.displayText\(\) = ([A-Za-z0-9\:]+:) ([^]+)/m)) {
// e.displayText() = DB::Exception: Syntax error: failed at position 0: SEL
fields.scope = m[1];
fields.message = m[2];
if (m = fields.message.match (/Syntax error: (?:failed at position (\d+)(?:\s*\(line\s*(\d+)\,\s+col\s*(\d+)\))?)/)) {
// console.log ('!!! syntax error: pos %s line %s col %s', m[1], m[2], m[3]);
fields.lineno = parseInt (m[2] || 1, 10);
fields.colno = parseInt (m[3] || m[1], 10);
}
} else if (m = f.match (/^e\.what\(\) = (.*)/)) {
fields.type = m[1];
} else {
console.warn ('Unknown error field:', f)
}
});
return fields;
}
module.exports = parseError;
================================================
FILE: src/process-db-value.js
================================================
/*
Formats:
CSV: https://clickhouse.yandex/docs/en/formats/csv.html
During parsing, values could be enclosed or not enclosed in quotes.
Supported both single and double quotes. In particular,
Strings could be represented without quotes - in that case,
they are parsed up to comma or newline (CR or LF).
Contrary to RFC, in case of parsing strings without quotes,
leading and trailing spaces and tabs are ignored. As line delimiter,
both Unix (LF), Windows (CR LF) or Mac OS Classic (LF CR) variants are supported.
TSV/TabSeparated: https://clickhouse.yandex/docs/en/formats/tabseparated.html
In TabSeparated format, data is written by row. Each row contains values separated by tabs.
Each value is follow by a tab, except the last value in the row,
which is followed by a line break. Strictly Unix line breaks are assumed everywhere.
The last row also must contain a line break at the end. Values are written in text format,
without enclosing quotation marks, and with special characters escaped.
Minimum set of symbols that you must escape in TabSeparated format is tab, newline (LF) and backslash.
Arrays are formatted as a list of comma-separated values in square brackets.
Number items in the array are formatted as normally, but dates, dates with times,
and strings are formatted in single quotes with the same escaping rules as above.
As an exception, parsing DateTime is also supported in Unix timestamp format,
if it consists of exactly 10 decimal digits. The result is not time zone-dependent.
The formats YYYY-MM-DD hh:mm:ss and NNNNNNNNNN are differentiated automatically.
Values: https://clickhouse.yandex/docs/en/formats/values.html
Prints every row in parentheses. Rows are separated by commas.
There is no comma after the last row. The values inside the parentheses are also comma-separated.
Numbers are output in decimal format without quotes. Arrays are output in square brackets.
Strings, dates, and dates with times are output in quotes.
Escaping rules and parsing are same as in the TabSeparated format.
During formatting, extra spaces aren’t inserted, but during parsing,
they are allowed and skipped (except for spaces inside array values, which are not allowed).
Minimum set of symbols that you must escape in Values format is single quote and backslash.
Nulls: \N?
https://github.com/yandex/ClickHouse/issues/252
https://github.com/yandex/ClickHouse/issues/700
https://github.com/Infinidat/infi.clickhouse_orm/pull/42
*/
var SEPARATORS = {
TSV: "\t",
CSV: ",",
Values: ","
}
var ALIASES = {
TabSeparated: "TSV"
}
var ESCAPE_STRING = {
TSV: function (v, quote) {return v.replace (/\\/g, '\\\\').replace(/\t/g, '\\t').replace(/\n/g, '\\n')},
CSV: function (v, quote) {return v.replace (/\"/g, '""')},
}
var ESCAPE_NULL = {
TSV: "\\N",
CSV: "\\N",
Values: "\\N",
// JSONEachRow: "\\N",
}
function encodeValue (quote, v, format) {
format = ALIASES[format] || format;
switch (typeof v) {
case 'string':
return ESCAPE_STRING[format] ? ESCAPE_STRING[format] (v, quote) : v;
case 'number':
if (isNaN (v))
return 'nan';
if (v === +Infinity)
return '+inf';
if (v === -Infinity)
return '-inf';
if (v === Infinity)
return 'inf';
return v;
case 'object':
// clickhouse allows to use unix timestamp in seconds
if (v instanceof Date)
return ("" + v.valueOf ()).substr (0, 10);
// you can add array items
if (v instanceof Array)
return v.map (function(item) {
return encodeValue(true, item, format);
})
// TODO: tuples support
if (!format) console.trace ();
if (v === null)
return format in ESCAPE_NULL ? ESCAPE_NULL[format] : v;
return format in ESCAPE_NULL ? ESCAPE_NULL[format] : v;
console.warn ('Cannot stringify [Object]:', v);
case 'boolean':
return v === true ? 1 : 0;
}
}
function encodeRow (row, format) {
format = ALIASES[format] || format;
var encodedRow;
if (Array.isArray (row)) {
encodedRow = row.map (function (field) {
return encodeValue (false, field, format);
}.bind (this)).join (SEPARATORS[format]) + "\n";
} else if (row.toString () === "[object Object]" && format === "JSONEachRow") {
encodedRow = JSON.stringify (Object.keys (row).reduce (function (encodedRowObject, k) {
encodedRowObject[k] = encodeValue (false, row[k], format);
return encodedRowObject;
}.bind (this), {})) + "\n";
}
return encodedRow;
}
module.exports = {
encodeValue: encodeValue,
encodeRow: encodeRow
}
================================================
FILE: src/streams.js
================================================
var util = require ('util');
var Duplex = require ('stream').Duplex;
var encodeRow = require ('./process-db-value').encodeRow;
/**
* Simplified JSON stream parser
* @param {object} emitter parser will emit metadata event when metadata is available
* @returns {function} string consumer
*/
function JSONStream (emitter) {
// states are:
// start: { encountered, look for keys
// meta: meta encountered with `[`, parse everything until `]`
// data: data encountered with `[`, just parse every string until `]`
// other keys: concatenate until the end, then prepend `{` and JSON.parse
var state = null;
var objBuffer;
function processLine (l) {
// console.log ("LINE>", l);
l = l.trim ();
if (!l.length)
return;
if (state === null) {
// first string should contains `{`
if (l === '{') {
state = 'topKeys';
}
} else if (state === 'topKeys') {
// console.log ('TOP>', l);
if (l === '"meta":') {
state = 'meta';
} else if (l === '"data":') {
state = 'data';
} else if (l === '"meta": [') {
state = 'meta-array';
} else if (l === '"data": [') {
state = 'data-array';
} else {
processLine.supplementalString += l;
}
} else if (state === 'meta') {
if (l === '[') {
state = 'meta-array';
}
} else if (state === 'data') {
if (l === '[') {
state = 'data-array';
}
} else if (state === 'meta-array') {
if (l.match (/^},?$/)) {
processLine.columns.push (JSON.parse (objBuffer + '}'));
objBuffer = undefined;
} else if (l === '{') {
objBuffer = l;
} else if (l.match (/^],?$/)) {
emitter.emit ('metadata', processLine.columns);
state = 'topKeys';
} else {
objBuffer += l;
}
} else if (state === 'data-array') {
if (l.match (/^[\]\}],?$/) && objBuffer) {
processLine.rows.push (JSON.parse (objBuffer + l[0]));
objBuffer = undefined;
} else if (l === '{' || l === '[') {
objBuffer = l;
} else if (l.match (/^],?$/)) {
state = 'topKeys';
} else if (objBuffer === undefined) {
processLine.rows.push (JSON.parse (l[l.length - 1] !== ',' ? l : l.substr (0, l.length - 1)));
} else {
objBuffer += l;
}
}
}
processLine.columns = [];
processLine.rows = [];
processLine.supplementalString = '{';
return processLine;
}
/**
* Duplex stream to work with database
* @param {object} [options] options
* @param {object} [options.format] how to format filelds/rows internally
*/
function RecordStream (options) {
// if (! (this instanceof RecordStream)) return new RecordStream(options);
options = options || {};
options.objectMode = true;
Duplex.call (this, options);
this.format = options.format;
this._writeBuffer = [];
this._canWrite = false;
Object.defineProperty (this, 'req', {
get: function () {return this._req},
set: function (req) {this._req = req; this._canWrite = true;}
})
}
util.inherits(RecordStream, Duplex);
RecordStream.prototype._read = function read () {
// nothing to do there, when data comes, push will be called
};
// http://ey3ball.github.io/posts/2014/07/17/node-streams-back-pressure/
// https://nodejs.org/en/docs/guides/backpressuring-in-streams/
// https://nodejs.org/docs/latest/api/stream.html#stream_implementing_a_writable_stream
// TODO: implement _writev
RecordStream.prototype._write = function _write (chunk, enc, cb) {
if (!Buffer.isBuffer (chunk) && typeof chunk !== 'string')
chunk = encodeRow (chunk, this.format);
// there is no way to determine line ending efficiently for Buffer
if (typeof chunk === 'string') {
if (chunk.substr (chunk.length - 1) !== "\n") {
chunk = chunk + "\n";
}
chunk = Buffer.from ? Buffer.from (chunk, enc) : new Buffer (chunk, enc);
}
if (!(chunk instanceof Buffer)) {
return this.emit ('error', new Error ('Incompatible format'));
}
// node stores further write requests into `_writableState.bufferedRequest` chain
// until cb is called.
this._canWrite = this.req.write (chunk, cb);
};
RecordStream.prototype._destroy = function _destroy (err, cb) {
process.nextTick (function () {
RecordStream.super_.prototype._destroy.call (this, err, function (destroyErr) {
this.req.destroy (err);
cb (destroyErr || err);
}.bind (this));
}.bind (this));
}
RecordStream.prototype.end = function end (chunk, enc, cb) {
RecordStream.super_.prototype.end.call (this, chunk, enc, function () {
this.req.end (cb);
}.bind (this));
};
module.exports = {
JSONStream: JSONStream,
RecordStream: RecordStream
};
================================================
FILE: test/01-simulated.js
================================================
var ClickHouse = require ("../src/clickhouse");
var http = require ('http');
var url = require ('url');
var qs = require ('querystring');
var assert = require ("assert");
var responses = {
"SELECT 1 FORMAT JSONCompact": {
"meta": [
{"name": "1", "type": "UInt8"}
],
"data": [
[1]
],
"rows": 1
},
"SHOW DATABASES FORMAT JSONCompact": {
"meta": [
{"name": "name", "type": "String"}
],
"data": [
["default"],
["system"]
],
"rows": 2
},
"SELECT number FROM system.numbers LIMIT 10 FORMAT JSONCompact": {
"meta": [
{"name":"number","type":"UInt64"}
],
"data": [
["0"],["1"],["2"],["3"],["4"],["5"],["6"],["7"],["8"],["9"]
],
"rows": 10,
"rows_before_limit_at_least": 10
},
};
describe ("simulated queries", function () {
var server,
host,
port;
before (function (done) {
server = http.createServer(function (req, res) {
var queryString = url.parse (req.url).query;
// test only supports db queries using queryString
if (!queryString) {
res.writeHead (200, {});
res.end ("Ok.\n");
return;
}
var queryObject = qs.parse (queryString);
// console.log (queryObject);
if (queryObject.query in responses) {
res.writeHead (200, {"Content-Type": "application/json; charset=UTF-8"});
// console.log (JSON.stringify (responses[queryObject.query], null, "\t"));
res.end (JSON.stringify (responses[queryObject.query], null, "\t"));
return;
}
res.writeHead (500, {"Content-Type": "text/plain; charset=UTF-8"});
res.end ("Simulated error");
});
server.on('clientError', function (err, socket) {
socket.end('HTTP/1.1 400 Bad Request\r\n\r\n');
});
server.listen (0, function (evt) {
host = server.address().address;
port = server.address().port;
host = host === '0.0.0.0' ? '127.0.0.1' : host;
host = host === '::' ? '127.0.0.1' : host;
done();
});
})
after (function (done) {
server.close(function () {
done();
});
});
// this.timeout (5000);
it ("pings", function (done) {
var ch = new ClickHouse ({host: host, port: port});
ch.ping (function (err, ok) {
assert (!err);
assert (ok === "Ok.\n", "ping response should be 'Ok.\\n'");
done ();
});
});
it ("selects using callback", function (done) {
var ch = new ClickHouse ({host: host, port: port, readonly: true});
ch.query ("SELECT 1", {syncParser: true}, function (err, result) {
assert (!err);
assert (result.meta, "result should be Object with `data` key to represent rows");
assert (result.data, "result should be Object with `meta` key to represent column info");
assert (result.meta.constructor === Array, "metadata is an array with column descriptions");
done ();
});
});
it ("selects numbers using callback", function (done) {
var ch = new ClickHouse ({host: host, port: port, readonly: true});
ch.query ("SELECT number FROM system.numbers LIMIT 10", {syncParser: true}, function (err, result) {
assert (!err);
assert (result.data, "result should be Object with `data` key to represent rows");
assert (result.meta, "result should be Object with `meta` key to represent column info");
assert (result.meta.constructor === Array, "metadata is an array with column descriptions");
assert (result.meta[0].name === "number");
assert (result.data.constructor === Array, "data is a row set");
assert (result.data[0].constructor === Array, "each row contains list of values (using FORMAT JSONCompact)");
assert (result.data[9][0] === "9"); // this should be corrected at database side
assert (result.rows === 10);
assert (result.rows_before_limit_at_least === 10);
done ();
});
});
it ("selects numbers using stream", function (done) {
var ch = new ClickHouse ({host: host, port: port, readonly: true});
var rows = [];
var stream = ch.query ("SELECT number FROM system.numbers LIMIT 10", function (err, result) {
assert (!err);
assert (result.meta, "result should be Object with `meta` key to represent column info");
assert (result.meta.constructor === Array, "metadata is an array with column descriptions");
assert (result.meta[0].name === "number");
assert (rows[0].constructor === Array, "each row contains list of values (using FORMAT JSONCompact)");
assert (rows[9][0] === "9"); // this should be corrected at database side
assert (result.rows === 10);
assert (result.rows_before_limit_at_least === 10);
done ();
});
stream.on ('data', function (row) {
rows.push (row);
})
});
});
================================================
FILE: test/02-real-server.js
================================================
var ClickHouse = require ("../src/clickhouse");
var http = require ('http');
var url = require ('url');
var qs = require ('querystring');
var assert = require ("assert");
describe ("real server", function () {
var server,
host = process.env.CLICKHOUSE_HOST || '127.0.0.1',
port = process.env.CLICKHOUSE_PORT || 8123,
dbCreated = false;
it ("pings", function (done) {
var ch = new ClickHouse ({host: host, port: port});
ch.ping (function (err, ok) {
assert.ifError (err);
assert.equal (ok, "Ok.\n", "ping response should be 'Ok.\\n'");
done ();
});
});
it ("pinging using promise interface", function () {
var ch = new ClickHouse ({host: host, port: port});
return ch.pinging ();
});
it ("pinging using promise interface with bad connection option", function () {
var ch = new ClickHouse ();
return ch.pinging ().then (function () {
return Promise.reject (new Error ("Driver should throw without host name"))
}, function (e) {
return Promise.resolve ();
});
});
it ("pings with options as host", function (done) {
var ch = new ClickHouse (host);
ch.ping (function (err, ok) {
assert.ifError (err);
assert.equal (ok, "Ok.\n", "ping response should be 'Ok.\\n'");
done ();
});
});
it ("nothing to ping", function () {
var ch = new ClickHouse ();
assert (ch);
});
it ("returns error", function (done) {
var ch = new ClickHouse ({host: host, port: port, readonly: true});
var stream = ch.query ("ABCDEFGHIJKLMN", {syncParser: true}, function (err, result) {
// assert (err);
// done ();
});
stream.on ('error', function (err) {
assert (err);
// console.log (err);
done();
});
});
it ("authorises by credentials", function () {
var ch = new ClickHouse ({host: host, port: port, user: 'default', password: ''});
return ch.querying ("SELECT 1");
});
it ("selects from system columns", function (done) {
var ch = new ClickHouse ({host: host, port: port});
ch.query ("SELECT * FROM system.columns", function (err, result) {
assert (!err);
done ();
});
});
it ("selects from system columns no more than 10 rows throws exception", function (done) {
var ch = new ClickHouse ({host: host, port: port, queryOptions: {max_rows_to_read: 10}});
ch.query ("SELECT * FROM system.columns", function (err, result) {
assert (err);
done ();
});
});
it ("creates a database", function (done) {
var ch = new ClickHouse ({host: host, port: port});
ch.query ("CREATE DATABASE node_clickhouse_test", function (err, result) {
assert (!err, err);
dbCreated = true;
// console.log (result);
done ();
});
});
it ("creates a table", function (done) {
var ch = new ClickHouse ({host: host, port: port});
ch.query ("CREATE TABLE node_clickhouse_test.t (a UInt8) ENGINE = Memory", function (err, result) {
assert (!err, err);
done ();
});
});
it ("drops a table", function (done) {
var ch = new ClickHouse ({host: host, port: port, queryOptions: {database: 'node_clickhouse_test'}});
ch.query ("DROP TABLE t", function (err, result) {
assert (!err);
done ();
});
});
it ("creates a table", function (done) {
var ch = new ClickHouse ({host: host, port: port, queryOptions: {database: 'node_clickhouse_test'}});
ch.query ("CREATE TABLE t (a UInt8) ENGINE = Memory", function (err, result) {
assert (!err);
done ();
});
});
it ("inserts some data", function (done) {
var ch = new ClickHouse ({host: host, port: port});
ch.query ("INSERT INTO t VALUES (1),(2),(3)", {queryOptions: {database: 'node_clickhouse_test'}}, function (err, result) {
assert (!err, err);
// let's wait a few seconds
setTimeout (function () {done ()}, 500);
});
});
it ("gets back data", function (done) {
var ch = new ClickHouse ({host: host, port: port});
var rows = [];
var stream = ch.query ("select a FROM t", {queryOptions: {database: 'node_clickhouse_test'}});
stream.on ('data', function (row) {
rows.push (row);
});
stream.on ('end', function () {
done();
});
});
after (function (done) {
if (!dbCreated)
return done ();
var ch = new ClickHouse ({host: host, port: port});
ch.query ("DROP DATABASE node_clickhouse_test", function (err, result) {
assert (!err);
// console.log (result);
done ();
});
});
});
================================================
FILE: test/03-errors.js
================================================
var ClickHouse = require ("../src/clickhouse");
var http = require ('http');
var url = require ('url');
var qs = require ('querystring');
var assert = require ("assert");
describe ("error parsing", function () {
var server,
host = process.env.CLICKHOUSE_HOST || '127.0.0.1',
port = process.env.CLICKHOUSE_PORT || 8123,
dbCreated = false;
it ("will not throw on http error", function (done) {
var ch = new ClickHouse ({host: host, port: 59999, readonly: true});
var stream = ch.query ("ABCDEFGHIJKLMN", {syncParser: true}, function (err, result) {
// assert (err);
// done ();
});
stream.on ('error', function (err) {
done();
});
});
it ("returns error for unknown sql", function (done) {
var ch = new ClickHouse ({host: host, port: port, readonly: true});
var stream = ch.query ("ABCDEFGHIJKLMN", {syncParser: true}, function (err, result) {
// assert (err);
// done ();
});
stream.on ('error', function (err) {
assert (err);
assert (err.message.match (/Syntax error/));
assert.equal (err.lineno, 1, "line number should eq 1");
assert.equal (err.colno, 1, "col number should eq 1");
// console.log (err);
done();
});
});
it ("returns error with line/col for sql with garbage", function (done) {
var ch = new ClickHouse ({host: host, port: port, readonly: true});
var stream = ch.query ("CREATE\n\t\tABCDEFGHIJKLMN", {syncParser: true}, function (err, result) {
// assert (err);
// done ();
});
stream.on ('error', function (err) {
assert (err);
assert (err.message.match (/Syntax error/));
assert.equal (err.lineno, 2, "line number should eq 2");
assert.equal (err.colno, 3, "col number should eq 3");
// console.log (err);
done();
});
});
it ("returns error for empty sql", function (done) {
var ch = new ClickHouse ({host: host, port: port, readonly: true});
function countCallbacks (err) {
countCallbacks.count = (countCallbacks.count || 0) + 1;
if (countCallbacks.count === 2) {
//
done ();
}
}
var stream = ch.query ("-- nothing here", {syncParser: true}, function (err, result) {
countCallbacks (err);
// assert (err);
// done ();
});
stream.on ('error', function (err) {
// console.log (err); // failed at end of query
assert (err);
assert (err.message.match (/Empty query/) || err.message.match (/Syntax error/), err);
// clickhouse doesn't return lineno and colno for some queries
// assert.ifError ('lineno' in err);
// assert.ifError ('colno' in err);
countCallbacks (err);
});
});
it ("returns error for unknown table", function (done) {
var ch = new ClickHouse ({host: host, port: port, readonly: true});
var stream = ch.query ("SELECT * FROM xxx", {syncParser: true}, function (err, result) {
assert (err);
});
stream.on ('error', function (err) {
assert (err);
assert.ifError (err.message.match (/Syntax error/));
// clickhouse doesn't return lineno and colno for some queries
// assert.ifError ('lineno' in err);
// assert.ifError ('colno' in err);
done();
});
});
// TODO turn it on when fixed https://github.com/ClickHouse/ClickHouse/issues/9914
// it ("returns error for writing in readonly mode", function (done) {
// var ch = new ClickHouse ({host: host, port: port});
// ch.querying ("CREATE TABLE xxx (a UInt8) ENGINE = Memory()", {readonly: true})
// .then(() => done('Should fail in readonly mode'))
// .catch(() => done())
// });
});
================================================
FILE: test/04-select.js
================================================
var ClickHouse = require ("../src/clickhouse");
var assert = require ("assert");
describe ("select data from database", function () {
var server,
host = process.env.CLICKHOUSE_HOST || '127.0.0.1',
port = process.env.CLICKHOUSE_PORT || 8123,
dbCreated = false;
it ("selects using callback", function (done) {
var ch = new ClickHouse ({host: host, port: port, readonly: true});
ch.query ("SELECT 1", {syncParser: true}, function (err, result) {
assert (!err);
assert (result.meta, "result should be Object with `data` key to represent rows");
assert (result.data, "result should be Object with `meta` key to represent column info");
done ();
});
});
it ("selects using callback and query submitted in the POST body", function (done) {
var ch = new ClickHouse ({host: host, port: port});
ch.query ("SELECT 1", {syncParser: true}, function (err, result) {
assert (!err);
assert (result.meta, "result should be Object with `data` key to represent rows");
assert (result.data, "result should be Object with `meta` key to represent column info");
done ();
});
});
it ("selects numbers using callback", function (done) {
var ch = new ClickHouse ({host: host, port: port, readonly: true});
ch.query ("SELECT number FROM system.numbers LIMIT 10", {syncParser: true}, function (err, result) {
assert (!err);
assert (result.meta, "result should be Object with `data` key to represent rows");
assert (result.data, "result should be Object with `meta` key to represent column info");
assert (result.meta.constructor === Array, "metadata is an array with column descriptions");
assert (result.meta[0].name === "number");
assert (result.data.constructor === Array, "data is a row set");
assert (result.data[0].constructor === Array, "each row contains list of values (using FORMAT JSONCompact)");
assert (result.data[9][0] === "9"); // this should be corrected at database side
assert (result.rows === 10);
assert (result.rows_before_limit_at_least === 10);
done ();
});
});
it ("selects numbers using promise should already have parsed data", function () {
var ch = new ClickHouse ({host: host, port: port, readonly: true});
return ch.querying ("SELECT number FROM system.numbers LIMIT 10").then (function (result) {
assert (result.meta, "result should be Object with `data` key to represent rows");
assert (result.data, "result should be Object with `meta` key to represent column info");
assert (result.meta.constructor === Array, "metadata is an array with column descriptions");
assert (result.meta[0].name === "number");
assert (result.data.constructor === Array, "data is a row set");
assert (result.data[0].constructor === Array, "each row contains list of values (using FORMAT JSONCompact)");
assert (result.data[9][0] === "9"); // this should be corrected at database side
assert (result.rows === 10);
assert (result.rows_before_limit_at_least === 10);
return Promise.resolve ();
});
});
it ("selects numbers as dataObjects using promise", function () {
var ch = new ClickHouse ({host: host, port: port, readonly: true, dataObjects: true});
return ch.querying ("SELECT number FROM system.numbers LIMIT 10").then (function (result) {
assert (result.meta, "result should be Object with `data` key to represent rows");
assert (result.data, "result should be Object with `meta` key to represent column info");
assert (result.meta.constructor === Array, "metadata is an array with column descriptions");
assert (result.meta[0].name === "number");
assert (result.data.constructor === Array, "data is a row set");
assert (result.data[0].constructor === Object, "each row contains key-valued rows (using FORMAT JSON)");
assert (result.data[9].number === "9");
assert (result.data.length === 10);
assert (result.rows === 10);
assert (result.rows_before_limit_at_least === 10);
return Promise.resolve ();
});
});
it ("selects numbers using callback and query submitted in the POST body", function (done) {
var ch = new ClickHouse ({host: host, port: port});
ch.query ("SELECT number FROM system.numbers LIMIT 10", {syncParser: true}, function (err, result) {
assert (!err);
assert (result.meta, "result should be Object with `meta` key to represent rows");
assert (result.data, "result should be Object with `data` key to represent column info");
assert (result.meta.constructor === Array, "metadata is an array with column descriptions");
assert (result.meta[0].name === "number");
assert (result.data.constructor === Array, "data is a row set");
assert (result.data[0].constructor === Array, "each row contains list of values (using FORMAT JSONCompact)");
assert (result.data[9][0] === "9"); // this should be corrected at database side
assert (result.rows === 10);
assert (result.rows_before_limit_at_least === 10);
done ();
});
});
it ("selects numbers asynchronously using events and query submitted in the POST body", function (done) {
var ch = new ClickHouse ({host: host, port: port});
var rows = [];
var stream = ch.query ("SELECT number FROM system.numbers LIMIT 10", function (err, result) {
assert (!err);
assert (result.meta, "result should be Object with `meta` key to represent rows");
assert (rows, "result should be Object with `data` key to represent column info");
assert (result.meta.constructor === Array, "metadata is an array with column descriptions");
assert (result.meta[0].name === "number");
assert (rows.length === 10, "total 10 rows");
assert (rows[0].constructor === Array, "each row contains list of values (using FORMAT JSONCompact)");
assert (rows[9][0] === "9"); // this should be corrected at database side
assert (result.rows === 10);
assert (result.rows_before_limit_at_least === 10);
done ();
});
stream.on ('data', function (row) {
rows.push (row);
})
});
it ("selects numbers asynchronously using stream and query submitted in the POST body", function (done) {
var ch = new ClickHouse ({host: host, port: port});
var metadata;
var rows = [];
var stream = ch.query ("SELECT number FROM system.numbers LIMIT 10");
stream.on ('metadata', function (_meta) {
metadata = _meta;
});
stream.on ('data', function (row) {
rows.push (row);
});
stream.on ('error', function (err) {
assert (err);
});
stream.on ('end', function () {
assert (metadata, "result should be Object with `meta` key to represent rows");
assert (rows, "result should be Object with `data` key to represent column info");
assert (metadata.constructor === Array, "metadata is an array with column descriptions");
assert (metadata[0].name === "number");
assert (rows.length === 10, "total 10 rows");
assert (rows[0].constructor === Array, "each row contains list of values (using FORMAT JSONCompact)");
assert (rows[9][0] === "9"); // this should be corrected at database side
assert (stream.supplemental.rows === 10);
assert (stream.supplemental.rows_before_limit_at_least === 10);
done ();
});
});
it ("selects number objects asynchronously using stream and query submitted in the POST body", function (done) {
var ch = new ClickHouse ({host: host, port: port});
var metadata;
var rows = [];
var stream = ch.query ("SELECT number FROM system.numbers LIMIT 10", {dataObjects: true});
stream.on ('metadata', function (_meta) {
metadata = _meta;
});
stream.on ('data', function (row) {
rows.push (row);
});
stream.on ('error', function (err) {
assert (err);
});
stream.on ('end', function () {
assert (metadata, "result should be Object with `meta` key to represent rows");
assert (rows, "result should be Object with `data` key to represent column info");
assert (metadata.constructor === Array, "metadata is an array with column descriptions");
assert (metadata[0].name === "number");
assert (rows.length === 10, "total 10 rows");
assert ('number' in rows[0], "each row contains fields (using FORMAT JSON)");
assert (rows[9].number === "9"); // this should be corrected at database side
assert (stream.supplemental.rows === 10);
assert (stream.supplemental.rows_before_limit_at_least === 10);
done ();
});
});
it ("select data in unsupported format", function (done) {
var ch = new ClickHouse ({host: host, port: port});
ch.query ("SELECT number FROM system.numbers LIMIT 10", {format: "CSV"}, function (err, result) {
assert (!err, err);
assert (result.match (/1\n2\n3\n4\n5\n6\n7\n8\n9/));
done ();
});
});
it("can cancel an ongoing select by calling destroy", function (done) {
var RecordStream = require("../src/streams").RecordStream;
if (typeof RecordStream.prototype.destroy !== "function") {
// If current Node.js version lacks Stream#destroy impl, skip the test
this.skip ();
return;
}
var ch = new ClickHouse ({host: host, port: port});
var rows = [];
var limit = 10000;
var stream = ch.query ("SELECT number FROM system.numbers LIMIT " + limit, function () {
assert (stream.destroyed);
assert (rows.length < limit);
done ();
});
stream.on ('data', function (row) {
rows.push (row);
});
stream.once ('data', function () {
stream.destroy ();
});
});
it("select query with WITH clause will produces result formatted ", function (done) {
var ch = new ClickHouse({ host: host, port: port, format: "JSON" });
ch.query("WITH 10 as pagesize SELECT 1", { syncParser: true }, function (err, result) {
assert(!err);
assert(result.meta, "result should be Object with `data` key to represent rows");
assert(result.data, "result should be Object with `meta` key to represent column info");
assert(Object.prototype.toString.call(result.data[0]) === '[object Object]', "data should be formatted JSON" )
done();
});
})
});
================================================
FILE: test/05-insert.js
================================================
var ClickHouse = require ("../src/clickhouse");
var assert = require ("assert");
var fs = require ("fs");
var crypto = require ("crypto");
var encodeValue = require ('../src/process-db-value').encodeValue;
var encodeRow = require ('../src/process-db-value').encodeRow;
function randomDate(start, end) {
return new Date(start.getTime() + Math.random() * (end.getTime() - start.getTime()));
}
function generateData (format, fileName, cb) {
var rs = fs.createWriteStream (fileName);
for (var i = 0; i < 10; i++) {
rs.write (
encodeRow ([
Math.ceil (Math.random () * 1000),
Math.random () * 1000,
crypto.randomBytes(20).toString('hex'),
randomDate(new Date(2012, 0, 1), new Date())
], format)
);
}
rs.end (function () {
cb ();
});
}
var testDate = new Date ();
var testDateISO = testDate.toISOString ().replace (/\..*/, '').replace ('T', ' ');
describe ("insert data", function () {
var server,
host = process.env.CLICKHOUSE_HOST || '127.0.0.1',
port = process.env.CLICKHOUSE_PORT || 8123,
dbCreated = false,
dbName = 'node_clickhouse_test_insert';
before (function () {
var ch = new ClickHouse ({host: host, port: port});
var okFn = function () {return Promise.resolve()};
return ch.querying ("DROP DATABASE " + dbName).then (
okFn, okFn
).then (function () {
return ch.querying ("CREATE DATABASE " + dbName);
}).then (function (result) {
dbCreated = true;
// console.log (result);
return Promise.resolve ();
});
});
it ("creates a table", function (done) {
var ch = new ClickHouse ({host: host, port: port, queryOptions: {database: dbName}});
ch.query ("CREATE TABLE t (a UInt8) ENGINE = Memory", function (err, result) {
assert (!err);
done ();
});
});
it ("inserts some prepared data using stream", function (done) {
var ch = new ClickHouse ({host: host, port: port});
var stream = ch.query ("INSERT INTO t", {queryOptions: {database: dbName}}, function (err, result) {
assert (!err, err);
ch.query ("SELECT * FROM t", {syncParser: true, queryOptions: {database: dbName}}, function (err, result) {
assert.equal (result.data[0][0], 8);
assert.equal (result.data[1][0], 73);
assert.equal (result.data[2][0], 42);
done ();
});
});
stream.write ('8');
stream.write ('73');
stream.write (Buffer.from ? Buffer.from ('42') : new Buffer ('42'));
stream.end ();
});
it ("inserts some data", function (done) {
var ch = new ClickHouse ({host: host, port: port});
ch.query ("INSERT INTO t VALUES (1),(2),(3)", {queryOptions: {database: dbName}}, function (err, result) {
assert (!err, err);
done ();
});
});
it ("inserts some data using promise", function () {
var ch = new ClickHouse ({host: host, port: port});
return ch.querying ("INSERT INTO t VALUES (1),(2),(3)", {queryOptions: {database: dbName}});
});
it ("creates a table 2", function (done) {
var ch = new ClickHouse ({host: host, port: port, queryOptions: {database: dbName}});
ch.query ("CREATE TABLE t2 (a UInt8, b Float32, x Nullable(String), z DateTime) ENGINE = Memory", function (err, result) {
assert (!err);
done ();
});
});
it ("inserts data from array using stream", function (done) {
var ch = new ClickHouse ({host: host, port: port});
var stream = ch.query ("INSERT INTO t2", {queryOptions: {database: dbName}}, function (err, result) {
assert (!err, err);
ch.query ("SELECT * FROM t2", {syncParser: true, queryOptions: {database: dbName}}, function (err, result) {
assert.equal (result.data[0][0], 1);
assert.equal (result.data[0][1], 2.22);
assert.equal (result.data[0][2], null);
assert.equal (result.data[0][3], testDateISO);
assert.equal (result.data[1][0], 20);
assert.equal (result.data[1][1], 1.11);
assert.equal (result.data[1][2], "wrqwefqwef");
assert.equal (result.data[1][3], "2017-07-07 12:12:12");
done ();
});
});
stream.write ([1, 2.22, null, testDate]);
stream.write ("20\t1.11\twrqwefqwef\t2017-07-07 12:12:12");
// stream.write ([0, Infinity, null, new Date ()]);
// stream.write ([23, NaN, "yyy", new Date ()]);
stream.end ();
});
it ("creates a table 3", function (done) {
var ch = new ClickHouse ({host: host, port: port, queryOptions: {database: dbName}});
ch.query ("CREATE TABLE t3 (a UInt8, b Float32, x Nullable(String), z DateTime) ENGINE = Memory", function (err, result) {
assert (!err);
done ();
});
});
it ("inserts data from array of objects using stream", function (done) {
var ch = new ClickHouse ({host: host, port: port});
var stream = ch.query ("INSERT INTO t3", {format: "JSONEachRow", queryOptions: {database: dbName}}, function (err, result) {
assert (!err, err);
ch.query ("SELECT * FROM t3", {syncParser: true, queryOptions: {database: dbName}}, function (err, result) {
assert.equal (result.data[0][0], 1);
assert.equal (result.data[0][1], 2.22);
assert.equal (result.data[0][2], null);
assert.equal (result.data[0][3], testDateISO);
assert.equal (result.data[1][0], 20);
assert.equal (result.data[1][1], 1.11);
assert.equal (result.data[1][2], "wrqwefqwef");
assert.equal (result.data[1][3], "2017-07-07 12:12:12");
assert.equal (result.data.length, 2);
done ();
});
});
stream.write ({a: 1, b: 2.22, x: null, z: testDate});
stream.write ({a: 20, b: 1.11, x: "wrqwefqwef", z: "2017-07-07 12:12:12"});
// stream.write ([0, Infinity, null, new Date ()]);
// stream.write ([23, NaN, "yyy", new Date ()]);
stream.end ();
});
it ("inserts from select", function (done) {
var ch = new ClickHouse ({host: host, port: port, queryOptions: {database: dbName}});
var stream = ch.query ("INSERT INTO t3 SELECT * FROM t2", {}, function (err, result) {
assert (!err, err);
ch.query ("SELECT * FROM t3", {syncParser: true, queryOptions: {database: dbName}}, function (err, result) {
assert.equal (result.data[2][0], 1);
assert.equal (result.data[2][1], 2.22);
assert.equal (result.data[2][2], null);
assert.equal (result.data[2][3], testDateISO);
assert.equal (result.data[3][0], 20);
assert.equal (result.data[3][1], 1.11);
assert.equal (result.data[3][2], "wrqwefqwef");
assert.equal (result.data[3][3], "2017-07-07 12:12:12");
assert.equal (result.data.length, 4);
done ();
});
});
// stream.end ();
});
it ("piping data from csv file", function (done) {
this.timeout (5000);
var ch = new ClickHouse ({host: host, port: port});
var csvFileName = __filename.replace ('.js', '.csv');
function processFileStream (fileStream) {
var stream = ch.query ("INSERT INTO t3", {format: "CSV", queryOptions: {database: dbName}}, function (err, result) {
assert (!err, err);
ch.query ("SELECT * FROM t3", {syncParser: true, queryOptions: {database: dbName}}, function (err, result) {
assert (!err, err);
fs.unlink (csvFileName, function () {
done ();
});
});
});
fileStream.pipe (stream);
stream.on ('error', function (err) {
// console.log (err);
});
}
fs.stat (csvFileName, function (err, stat) {
//if (err) {
return generateData ('CSV', csvFileName, function () {
processFileStream (fs.createReadStream (csvFileName));
})
//}
processFileStream (fs.createReadStream (csvFileName));
});
});
it ("piping data from tsv file", function (done) {
this.timeout (5000);
var ch = new ClickHouse ({host: host, port: port});
var tsvFileName = __filename.replace ('.js', '.tsv');
function processFileStream (fileStream) {
var stream = ch.query ("INSERT INTO t3", {format: "TabSeparated", queryOptions: {database: dbName}}, function (err, result) {
assert (!err, err);
ch.query ("SELECT * FROM t3", {syncParser: true, queryOptions: {database: dbName}}, function (err, result) {
assert (!err, err);
fs.unlink (tsvFileName, function () {
done ();
});
});
});
fileStream.pipe (stream);
stream.on ('error', function (err) {
// console.log (err);
});
}
fs.stat (tsvFileName, function (err, stat) {
//if (err) {
return generateData ('TSV', tsvFileName, function () {
processFileStream (fs.createReadStream (tsvFileName));
})
//}
processFileStream (fs.createReadStream (tsvFileName));
});
});
it ("creates a table 4", function (done) {
var ch = new ClickHouse ({host: host, port: port, queryOptions: {database: dbName}});
ch.query ("CREATE TABLE t4 (arrayString Array(String), arrayInt Array(UInt32)) ENGINE = Memory", function (err, result) {
assert (!err);
done ();
});
});
it ("inserts array with format JSON using stream", function (done) {
var ch = new ClickHouse ({host: host, port: port});
var stream = ch.query ("INSERT INTO t4", {queryOptions: {database: dbName}, format: "JSONEachRow"}, function (err, result) {
assert (!err, err);
ch.query ("SELECT * FROM t4", {syncParser: true, queryOptions: {database: dbName}, dataObjects: "JSON"}, function (err, result) {
assert.deepEqual (result.data[0].arrayString, ['first', 'second']);
assert.deepEqual (result.data[0].arrayInt, [1, 0, 100]);
done ();
});
});
stream.write ({
arrayString: ['first', 'second'],
arrayInt: [1, 0, 100]
});
stream.end ();
});
it ("creates a table 5", function () {
var ch = new ClickHouse ({host: host, port: port, queryOptions: {database: dbName}});
return ch.querying ("CREATE TABLE t5 (a UInt8, b Float32, x Nullable(String), z DateTime) ENGINE = Memory");
});
it ("inserts csv with FORMAT clause", function (done) {
var ch = new ClickHouse ({host: host, port: port});
var stream = ch.query ("INSERT INTO t5 FORMAT CSV", {queryOptions: {database: dbName}}, function (err, result) {
assert (!err, err);
ch.query ("SELECT * FROM t5", {syncParser: true, queryOptions: {database: dbName}}, function (err, result) {
assert.equal (result.data[0][0], 0);
assert.equal (result.data[0][1], 0);
assert.equal (result.data[0][2], null);
assert.equal (result.data[0][3], '1970-01-02 00:00:00');
assert.equal (result.data[1][0], 1);
assert.equal (result.data[1][1], 1.5);
assert.equal (result.data[1][2], '1');
assert.equal (result.data[1][3], '2050-01-01 00:00:00');
done ();
});
});
stream.write('0,0,\\N,"1970-01-02 00:00:00"\n1,1.5,"1","2050-01-01 00:00:00"')
stream.end ();
});
it ("select data with FORMAT clause", function () {
var ch = new ClickHouse ({host: host, port: port});
return ch.querying("SELECT * FROM t5 FORMAT Values", {queryOptions: {database: dbName}})
.then((data) => {
assert.equal (data, `(0,0,NULL,'1970-01-02 00:00:00'),(1,1.5,'1','2050-01-01 00:00:00')`)
})
});
it ("select data with GET method and FORMAT clause", function () {
var ch = new ClickHouse ({host: host, port: port, useQueryString: true});
return ch.querying("SELECT * FROM t5 FORMAT Values", {queryOptions: {database: dbName}})
.then((data) => {
assert.equal (data, `(0,0,NULL,'1970-01-02 00:00:00'),(1,1.5,'1','2050-01-01 00:00:00')`)
})
});
it ("select data with GET method and format option", function () {
var ch = new ClickHouse ({host: host, port: port, useQueryString: true});
return ch.querying("SELECT * FROM t5", {queryOptions: {database: dbName}, format: 'Values'})
.then((data) => {
assert.equal (data, `(0,0,NULL,'1970-01-02 00:00:00'),(1,1.5,'1','2050-01-01 00:00:00')`)
})
});
after (function (done) {
if (!dbCreated)
return done;
var ch = new ClickHouse ({host: host, port: port});
ch.query ("DROP DATABASE " + dbName, function (err, result) {
assert (!err);
// console.log (result);
done ();
});
});
});
================================================
FILE: test/06-encode-value.js
================================================
var assert = require ('assert');
var encodeValue = require ('../src/process-db-value').encodeValue;
describe('encode value', function() {
it('escaping string backslashes for TSV', function() {
var value = '{"key1":\t"{\"key2\":\"<some \\"attr\\">With tabulated \\t and line bracking \\n</some>\"}"\n}';
var expected = '{"key1":\\t"{"key2":"<some \\\\"attr\\\\">With tabulated \\\\t and line bracking \\\\n</some>\"}"\\n}';
var actual = encodeValue(undefined, value, 'TSV');
assert.equal(actual, expected);
})
})
================================================
FILE: test/07-compat.js
================================================
const ClickHouse = require("../src/clickhouse");
const assert = require("assert");
const DB_NAME = 'node_clickhouse_test_compat';
const noop = () => {}
describe("api compatibility", () => {
const host = process.env.CLICKHOUSE_HOST || '127.0.0.1';
const port = process.env.CLICKHOUSE_PORT || 8123;
const ch = new ClickHouse({host: host, port: port, queryOptions: {database: DB_NAME}});
before(() => (
new ClickHouse({host: host, port: port})
.querying(`CREATE DATABASE IF NOT EXISTS "${DB_NAME}"`)
))
after(() => (
ch.querying(`DROP DATABASE IF EXISTS "${DB_NAME}"`)
))
describe('should emit "end" event', () => {
before(function() {
console.log(('CREATE TABLE x (date Date) Engine=Memory'))
return ch.querying('CREATE TABLE x (date Date) Engine=Memory')
})
it('on insert done', (done) => {
const chStream = ch.query('INSERT INTO x', { format: 'JSONEachRow' })
chStream.on('data', noop)
chStream.on('error', done)
chStream.on('end', e => done())
chStream.write({date: '2000-01-01'})
chStream.end()
});
describe('on SELECT done. With default format', () => {
it('HTTP GET', (done) => {
const stream = ch.query('SELECT * FROM system.numbers LIMIT 0', { readonly: true });
stream.on ('data', noop)
stream.on ('end', () => done());
stream.end()
});
it('HTTP POST', (done) => {
const stream = ch.query('SELECT * FROM system.numbers LIMIT 0', { readonly: false });
stream.on ('data', noop)
stream.on ('end', () => done());
stream.end()
});
})
})
});
================================================
FILE: test/90-torture.js
================================================
var ClickHouse = require ("../src/clickhouse");
var http = require ('http');
var url = require ('url');
var qs = require ('querystring');
var assert = require ("assert");
var memwatch;
try {
var nodeMajorVersion = parseInt(process.version.substr(1));
if (nodeMajorVersion >= 10) {
memwatch = require('memwatch');
} else {
memwatch = require('memwatch-next');
}
} catch (err) {
if (process.env.TORTURE) {
console.error ("For the torture test you should install memwatch-next (node 8) or @airbnb/memwatch (node >= 10)");
console.error (err);
process.exit (1);
}
}
// replace with it, if you want to run this test suite
var method = process.env.TORTURE ? it : it.skip;
var timeout = 60000;
function checkUsageAndGC (used, baseline) {
function inMB (v) {
return (v/Math.pow (2, 20)).toFixed (1) + 'MB';
}
var usage = 'rss heapTotal heapUsed external'.split (' ').map (function (k) {
var delta = used[k] - baseline[k];
return k + ' ' + inMB (used[k]) + ' / +' + inMB (delta);
});
console.log (usage.join (' '));
gc ();
}
describe ("torturing", function () {
global.gc && gc();
var server,
host = process.env.CLICKHOUSE_HOST || '127.0.0.1',
port = process.env.CLICKHOUSE_PORT || 8123,
lastSuite,
baselineMemoryUsage = process.memoryUsage();
before (function () {
memwatch && memwatch.on ('leak', function (info) {
console.log (info);
});
// console.log (process.memoryUsage());
})
method ("selects 1 million using async parser", function (done) {
this.timeout (timeout);
var ch = new ClickHouse ({host: host, port: port, readonly: true});
var queryCount = 0;
var symbolsTransferred = 0;
function runQuery () {
var stream = ch.query ("SELECT number FROM system.numbers LIMIT 1000000", function (err, result) {
symbolsTransferred += result.transferred;
queryCount ++;
if (queryCount < 10)
return runQuery ();
console.log ('symbols transferred:', symbolsTransferred);
done();
});
stream.on ('error', function (err) {
done (err);
});
stream.on ('data', function (row) {
// just throw away that data
});
}
runQuery ();
lastSuite = 'async';
});
// enable this test separately
it.skip ("selects 1 million using sync parser", function (done) {
this.timeout (timeout);
var ch = new ClickHouse ({host: host, port: port, readonly: true});
var queryCount = 0;
var symbolsTransferred = 0;
function runQuery () {
var stream = ch.query ("SELECT number FROM system.numbers LIMIT 1000000", {syncParser: true}, function (err, result) {
symbolsTransferred += result.transferred;
queryCount ++;
if (queryCount < 10)
return runQuery ();
console.log ('symbols transferred:', symbolsTransferred);
done();
});
stream.on ('error', function (err) {
done (err);
});
stream.on ('data', function (row) {
// just throw away that data
});
}
runQuery ();
lastSuite = 'sync';
});
method ("selects system.columns using async parser #1", function (done) {
this.timeout (timeout);
var ch = new ClickHouse ({host: host, port: port, readonly: true});
var queryCount = 0;
var symbolsTransferred = 0;
function runQuery () {
var stream = ch.query ("SELECT * FROM system.columns", function (err, result) {
symbolsTransferred += result.transferred;
queryCount ++;
if (queryCount < 10000)
return runQuery ();
console.log ('symbols transferred:', symbolsTransferred);
done();
});
stream.on ('error', function (err) {
done (err);
});
stream.on ('data', function (row) {
// just throw away that data
});
}
runQuery ();
lastSuite = 'async';
});
method ("selects system.columns using sync parser #1", function (done) {
this.timeout (timeout);
var ch = new ClickHouse ({host: host, port: port, readonly: true});
var queryCount = 0;
var symbolsTransferred = 0;
function runQuery () {
var stream = ch.query ("SELECT * FROM system.columns", {syncParser: true}, function (err, result) {
symbolsTransferred += result.transferred;
queryCount ++;
if (queryCount < 10000)
return runQuery ();
console.log ('symbols transferred:', symbolsTransferred);
done();
});
stream.on ('error', function (err) {
done (err);
});
stream.on ('data', function (row) {
// just throw away that data
});
}
runQuery ();
lastSuite = 'sync';
});
method ("selects system.columns using async parser #2", function (done) {
this.timeout (timeout);
var ch = new ClickHouse ({host: host, port: port, readonly: true});
var queryCount = 0;
var symbolsTransferred = 0;
function runQuery () {
var stream = ch.query ("SELECT * FROM system.columns", function (err, result) {
symbolsTransferred += result.transferred;
queryCount ++;
if (queryCount < 10000)
return runQuery ();
console.log ('symbols transferred:', symbolsTransferred);
done();
});
stream.on ('error', function (err) {
done (err);
});
stream.on ('data', function (row) {
// just throw away that data
});
}
runQuery ();
lastSuite = 'async';
});
method ("selects system.columns using sync parser #2", function (done) {
this.timeout (timeout);
var ch = new ClickHouse ({host: host, port: port, readonly: true});
var queryCount = 0;
var symbolsTransferred = 0;
function runQuery () {
var stream = ch.query ("SELECT * FROM system.columns", {syncParser: true}, function (err, result) {
symbolsTransferred += result.transferred;
queryCount ++;
if (queryCount < 10000)
return runQuery ();
console.log ('symbols transferred:', symbolsTransferred);
done();
});
stream.on ('error', function (err) {
done (err);
});
stream.on ('data', function (row) {
// just throw away that data
});
}
runQuery ();
lastSuite = 'sync';
});
afterEach (function () {
checkUsageAndGC (process.memoryUsage(), baselineMemoryUsage)
// console.log ('after', lastSuite, process.memoryUsage());
})
});
gitextract_73vqxqjg/
├── .gitignore
├── .travis.yml
├── LICENSE
├── NOTES.md
├── README.md
├── package.json
├── src/
│ ├── clickhouse.js
│ ├── consts.js
│ ├── legacy-support.js
│ ├── parse-error.js
│ ├── process-db-value.js
│ └── streams.js
└── test/
├── 01-simulated.js
├── 02-real-server.js
├── 03-errors.js
├── 04-select.js
├── 05-insert.js
├── 06-encode-value.js
├── 07-compat.js
└── 90-torture.js
SYMBOL INDEX (21 symbols across 8 files)
FILE: src/clickhouse.js
function httpResponseHandler (line 18) | function httpResponseHandler (stream, reqParams, reqData, cb, response) {
function httpRequest (line 172) | function httpRequest (reqParams, reqData, cb) {
function ClickHouse (line 209) | function ClickHouse (options) {
FILE: src/parse-error.js
function parseError (line 1) | function parseError (e) {
FILE: src/process-db-value.js
function encodeValue (line 75) | function encodeValue (quote, v, format) {
function encodeRow (line 114) | function encodeRow (row, format) {
FILE: src/streams.js
function JSONStream (line 11) | function JSONStream (emitter) {
function RecordStream (line 97) | function RecordStream (options) {
FILE: test/03-errors.js
function countCallbacks (line 67) | function countCallbacks (err) {
FILE: test/05-insert.js
function randomDate (line 10) | function randomDate(start, end) {
function generateData (line 14) | function generateData (format, fileName, cb) {
function processFileStream (line 223) | function processFileStream (fileStream) {
function processFileStream (line 265) | function processFileStream (fileStream) {
FILE: test/07-compat.js
constant DB_NAME (line 4) | const DB_NAME = 'node_clickhouse_test_compat';
FILE: test/90-torture.js
function checkUsageAndGC (line 29) | function checkUsageAndGC (used, baseline) {
function runQuery (line 69) | function runQuery () {
function runQuery (line 102) | function runQuery () {
function runQuery (line 134) | function runQuery () {
function runQuery (line 166) | function runQuery () {
function runQuery (line 198) | function runQuery () {
function runQuery (line 230) | function runQuery () {
Condensed preview — 20 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (89K chars).
[
{
"path": ".gitignore",
"chars": 641,
"preview": "# Logs\nlogs\n*.log\nnpm-debug.log*\n\n# Runtime data\npids\n*.pid\n*.seed\n\n# Directory for instrumented libs generated by jscov"
},
{
"path": ".travis.yml",
"chars": 1258,
"preview": "language: node_js\nmatrix:\n include:\n# - node_js: '0.10'\n# env: _CXXAUTO=1\n# - node_js: '0.12'\n# env: _C"
},
{
"path": "LICENSE",
"chars": 1081,
"preview": "The MIT License (MIT)\n\nCopyright (c) 2016 ivan baktsheev\n\nPermission is hereby granted, free of charge, to any person ob"
},
{
"path": "NOTES.md",
"chars": 211,
"preview": "# Notes\n\n## Testing on node 10.x\n\nSeems like memwatch-next is not supported anymore,\nso replace it with @airbnb/node-mem"
},
{
"path": "README.md",
"chars": 12060,
"preview": "Simple and powerful interface for [ClickHouse](https://clickhouse.yandex/) [![travis](https://travis-ci.org/apla/node-cl"
},
{
"path": "package.json",
"chars": 1199,
"preview": "{\n \"name\": \"@apla/clickhouse\",\n \"version\": \"1.6.4\",\n \"description\": \"ClickHouse database interface\",\n \"main\": \"src/c"
},
{
"path": "src/clickhouse.js",
"chars": 10225,
"preview": "var http = require ('http');\nvar https = require('https');\nvar url = require ('url');\nvar qs = require ('querystring'"
},
{
"path": "src/consts.js",
"chars": 271,
"preview": "exports.LIBRARY_SPECIFIC_OPTIONS = new Set([\n // \"Auth\" shorthand\n 'user',\n 'password',\n\n // Database settings go to"
},
{
"path": "src/legacy-support.js",
"chars": 1140,
"preview": "var nodeVer = process.version.substr (1).split ('.');\n\nif (nodeVer[0] >= 6)\n\treturn;\n\nvar legacyModulesInstallCmd = 'npm"
},
{
"path": "src/parse-error.js",
"chars": 983,
"preview": "function parseError (e) {\n\tvar fields = new Error (e.toString ('utf8'));\n\te.toString ('utf8')\n\t\t.split (/\\,\\s+(?=e\\.)/gm"
},
{
"path": "src/process-db-value.js",
"chars": 4469,
"preview": "/*\n\nFormats:\n\nCSV: https://clickhouse.yandex/docs/en/formats/csv.html\n\nDuring parsing, values could be enclosed or not e"
},
{
"path": "src/streams.js",
"chars": 4543,
"preview": "var util = require ('util');\nvar Duplex = require ('stream').Duplex;\n\nvar encodeRow = require ('./process-db-value').e"
},
{
"path": "test/01-simulated.js",
"chars": 4538,
"preview": "var ClickHouse = require (\"../src/clickhouse\");\n\nvar http = require ('http');\nvar url = require ('url');\nvar qs = req"
},
{
"path": "test/02-real-server.js",
"chars": 4342,
"preview": "var ClickHouse = require (\"../src/clickhouse\");\n\nvar http = require ('http');\nvar url = require ('url');\nvar qs = req"
},
{
"path": "test/03-errors.js",
"chars": 3504,
"preview": "var ClickHouse = require (\"../src/clickhouse\");\n\nvar http = require ('http');\nvar url = require ('url');\nvar qs = req"
},
{
"path": "test/04-select.js",
"chars": 9911,
"preview": "var ClickHouse = require (\"../src/clickhouse\");\n\nvar assert = require (\"assert\");\n\ndescribe (\"select data from database\""
},
{
"path": "test/05-insert.js",
"chars": 11999,
"preview": "var ClickHouse = require (\"../src/clickhouse\");\n\nvar assert = require (\"assert\");\nvar fs = require (\"fs\");\nvar crypt"
},
{
"path": "test/06-encode-value.js",
"chars": 525,
"preview": "var assert = require ('assert');\nvar encodeValue = require ('../src/process-db-value').encodeValue;\n\ndescribe('encode va"
},
{
"path": "test/07-compat.js",
"chars": 1647,
"preview": "const ClickHouse = require(\"../src/clickhouse\");\nconst assert = require(\"assert\");\n\nconst DB_NAME = 'node_clickhouse_tes"
},
{
"path": "test/90-torture.js",
"chars": 6070,
"preview": "var ClickHouse = require (\"../src/clickhouse\");\n\nvar http = require ('http');\nvar url = require ('url');\nvar qs = req"
}
]
About this extraction
This page contains the full source code of the apla/node-clickhouse GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 20 files (78.7 KB), approximately 22.2k tokens, and a symbol index with 21 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.