Showing preview only (1,247K chars total). Download the full file or copy to clipboard to get everything.
Repository: tagomoris/shib
Branch: master
Commit: 8c5386931518
Files: 73
Total size: 1.2 MB
Directory structure:
gitextract_i18lfx99/
├── .gitignore
├── .gitmodules
├── README.md
├── app.js
├── bin/
│ ├── dbmigrate.js
│ └── purge.js
├── config.js
├── examples/
│ ├── hive_beeline_client.js
│ ├── hive_client.js
│ ├── hive_methods.js
│ ├── hive_multi_client.js
│ ├── hive_server2_engine.example.js
│ └── hive_server2_results.js
├── lib/
│ └── shib/
│ ├── access_control.js
│ ├── auth/
│ │ ├── dumb.js
│ │ ├── http_basic_auth.js
│ │ └── http_custom_header.js
│ ├── auth.js
│ ├── client.js
│ ├── engine.js
│ ├── engines/
│ │ ├── bigquery/
│ │ │ └── index.js
│ │ ├── dummy/
│ │ │ └── index.js
│ │ ├── dummyengine.js
│ │ ├── hiveserver/
│ │ │ ├── FacebookService.js
│ │ │ ├── ThriftHive.js
│ │ │ ├── ThriftHiveMetastore.js
│ │ │ ├── fb303_types.js
│ │ │ ├── hive_metastore_types.js
│ │ │ ├── hive_service_types.js
│ │ │ ├── index.js
│ │ │ └── queryplan_types.js
│ │ ├── hiveserver2/
│ │ │ ├── TCLIService.js
│ │ │ ├── TCLIService_types.js
│ │ │ └── index.js
│ │ ├── huahin_mrv1/
│ │ │ ├── index.js
│ │ │ └── rest.js
│ │ ├── huahin_yarn/
│ │ │ ├── index.js
│ │ │ └── rest.js
│ │ ├── jobtracker/
│ │ │ ├── client.js
│ │ │ └── index.js
│ │ ├── presto/
│ │ │ └── index.js
│ │ └── yarn/
│ │ ├── client.js
│ │ └── index.js
│ ├── huahin_client.js
│ ├── index.js
│ ├── localdiskstore.js
│ ├── logger.js
│ ├── query.js
│ └── simple_csv_builder.js
├── package.json
├── public/
│ ├── css/
│ │ ├── redmond/
│ │ │ └── jquery-ui-1.8.13.custom.css
│ │ ├── shib.css
│ │ └── ui.dynatree.css
│ ├── index.graphtest.html
│ ├── index.jquery-ui-default.html
│ └── js/
│ └── shib.js
├── src/
│ └── hiveserver/
│ └── if/
│ ├── TCLIService.thrift
│ ├── hive_service.thrift
│ ├── metastore/
│ │ └── if/
│ │ └── hive_metastore.thrift
│ ├── ql/
│ │ └── if/
│ │ └── queryplan.thrift
│ └── share/
│ └── fb303/
│ └── if/
│ └── fb303.thrift
├── test/
│ ├── t_access_control.js
│ ├── t_query.js
│ ├── t_simple_csv_builder.js
│ └── t_thrifthivemock.js
├── testtools/
│ ├── ThriftHiveMock.js
│ ├── check_database_hs2_engine.js
│ ├── check_database_hs2_use.js
│ ├── check_engine.js
│ ├── check_engine_huahin_mrv1.js
│ └── hive_server_mock.js
├── var/
│ └── .datadir
└── views/
└── index.jade
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitignore
================================================
var/database.sqlite3
var/database.sqlite3.v0
var/results
node_modules
npm-debug.log
production.js
*.swp
================================================
FILE: .gitmodules
================================================
================================================
FILE: README.md
================================================
# shib
* http://github.com/tagomoris/shib
* Blog entry [tagomoris.tumblr](http://tagomoris.tumblr.com/post/77890621867/shib-web-client-for-hive-now-supports-presto)
## DESCRIPTION
Shib is web client application for SQL-like query engines, written in Node.js, supporting
* Hive (hiveserver, hiveserver2)
* Facebook Presto
* Google BigQuery
Once configured, we can switch query engines per executions.
Some extension features are supported:
* Setup queries: options to specify queries executed before main query, like 'create functions ...'
* Default Database: option to specify default database for Hive 0.6 or later
### Versions
Latest version of 'shib' is v1.0.2.
'shib' versions are:
* v1.0 series
* metadata of v1.0 is **NOT compatible with v0.3**, so migration required
* v0.3 series
* use latest node (`~> v0.10.26`)
* multi engines/databases support
* presto support
* storages of v0.3.x are compatible with v0.2
* engine-wide database/table access controls
* tagging for executed queries
* authentication / logging for query execution
* access controls based on authentication
* v0.2 series
* current status of master branch
* uses local filesystem instead of KT, depends on latest node (v0.8.x, v0.10.x)
* higher performance, more safe Web UI and updated features
* storages of v0.2 are **NOT complatible with v0.1**
* v0.1 series
* uses KT, depends on node v0.6.x
* see `v0.1` tag
## INSTALL
### Hive/Presto
For Hive queries, shib requires HiveServer or HiveServer2. Setup and run these.
* For HiveServer2
* Configure `hive.server2.authentication` as `NOSASL`
* Strongly recommended to configure `hive.support.concurrency` as `false`
For Presto, shib is tested with Presto version 0.57.
### Node.js
To run shib, you must install node.js (v0.10.x recommended), and export PATH for installed node.
### shib
Clone shib code.
$ git clone git://github.com/tagomoris/shib.git
Install libraries, configure addresses of HiveServer (and other specifications).
$ npm install
$ vi config.js
And run.
$ npm start
Shib listens on port 3000. see http://localhost:3000/
To switch environments for each shib instance, use `NODE_ENV` environment variable. (ex: `production.js` will be used with `NODE_ENV=production`)
$ NODE_ENV=production NODE_PATH=lib node app.js
## Migrate metadata database from v0 to v1
Migration operation required to execute shib v1, with data in v0 era.
1. Stop shib process
2. Update shib code to v1
3. Execute `npm run migrate` (for `var/database.sqlite3` file)
* This operation requires 5 minutes 30 seconds for 220MB database
* Backup v0 database file is `var/database.sqlite3.v0`
4. Start shib
## Configuration
Shib can have 2 or more query executor engines.
### HiveServer2
Basic configuration with HiveServer2 in config.js (or productions.js):
```js
var servers = exports.servers = {
listen: 3000,
fetch_lines: 1000, // lines per fetch in shib
query_timeout: null, // shib waits queries forever
setup_queries: [],
storage: {
datadir: './var'
},
engines: [
{ label: 'mycluster1',
executer: {
name: 'hiveserver2',
host: 'hs2.mycluster1.local',
port: 10000,
username: 'hive',
support_database: true
},
monitor: null
},
],
};
```
`username` should be same as user name that hive job will be executed on. (`password` is not required for NOSASL transport.)
For UDFs, you can specify statements before query executions in `setup_queries`.
```js
var servers = exports.servers = {
listen: 3000,
fetch_lines: 1000,
query_timeout: null,
setup_queries: [
"add jar /path/to/jarfile/foo.jar",
"create temporary function foofunc as 'package.of.udf.FooFunc'",
"create temporary function barfunc as 'package.of.udf.BarFunc'"
],
storage: {
datadir: './var'
},
engines: [
{ label: 'mycluster1',
executer: {
name: 'hiveserver2',
host: 'hs2.mycluster1.local',
port: 10000,
support_database: true
},
monitor: null
},
],
};
```
### HiveServer
Classic HiveServer is available if you want database supports instead of HiveServer2.
```js
var servers = exports.servers = {
listen: 3000,
fetch_lines: 1000,
query_timeout: null,
setup_queries: [],
storage: {
datadir: './var'
},
engines: [
{ label: 'mycluster1',
executer: {
name: 'hiveserver', // HiveServer(1)
host: 'hs1.mycluster1.local',
port: 10000,
support_database: true,
default_database: 'mylogs1'
},
monitor: null
},
],
};
```
### Presto
For Presto, use `presto` executer.
```js
var servers = exports.servers = {
listen: 3000,
fetch_lines: 1000,
query_timeout: 30, // 30 seconds for Presto query timeouts (it will fail)
setup_queries: [],
storage: {
datadir: './var'
},
engines: [
{ label: 'prestocluster1',
executer: {
name: 'presto',
host: 'coordinator.mycluster2.local',
port: 8080,
user: 'shib',
catalog: 'hive', // required configuration argument
support_database: true,
default_database: 'mylogs1'
},
monitor: null
},
],
};
```
### BigQuery
For BigQuery, use `bigquery` executer.
```js
var servers = exports.servers = {
listen: 3000,
fetch_lines: 1000,
query_timeout: 30, // 30 seconds for BigQuery query timeouts (it will fail)
storage: {
datadir: './var'
},
engines: [
{ label: 'bigquery',
executer: {
name: 'bigquery',
default_database: 'mylogs1',
project_id: 'gcp-project-id',
key_filename: '/path/to/keyfile.json'
},
monitor: null
}
]
};
```
For more detail about `project_id` and `key_filename` config, see https://github.com/GoogleCloudPlatform/gcloud-node#authorization .
### Multi clusters and engines
Shib supports 2 or more engines for a cluster, and 2 or more clusters with same engines. These patterns are available.
* HiveServer1, HiveServer2 and Presto for same data source
* 2 or more catalogs for same Presto cluster
* Many clusters which has one of HiveServer, HiveServer2 or Presto
You should write configurations in `engines` how you wants. `fetch_lines`, `query_timeout` and `setup_queries` in each engines overwrites global default of these configurations.
For example, This is configuration example.
* ClusterA has HiveServer2
* listenes port 10000
* customized udfs in `foo.jar` are availabe
* ClusterB has HiveServer
* listenes port 10001
* customized udfs in `foo.jar` are available
* Presto cluster is configured with `hive` catalog and `hive2` catalog
* udfs are not available
```js
var servers = exports.servers = {
listen: 3000,
fetch_lines: 1000,
query_timeout: null,
setup_queries: [
"add jar /path/to/jarfile/foo.jar",
"create temporary function foofunc as 'package.of.udf.FooFunc'",
"create temporary function barfunc as 'package.of.udf.BarFunc'"
],
storage: {
datadir: './var'
},
engines: [
{ label: 'myclusterA',
executer: {
name: 'hiveserver2',
host: 'master.a.cluster.local',
port: 10000,
support_database: true
},
monitor: null
},
{ label: 'myclusterB',
executer: {
name: 'hiveserver',
host: 'master.b.cluster.local',
port: 10001,
support_database: true,
default_database: 'mylogs1'
},
monitor: null
},
{ label: 'prestocluster1',
executer: {
name: 'presto',
host: 'coordinator.p.cluster.local',
port: 8080,
user: 'shib',
catalog: 'hive',
support_database: true,
default_database: 'mylogs1',
query_timeout: 30, // overwrite global config
setup_queries: [] // overwrite global config
},
monitor: null
},
{ label: 'prestocluster2',
executer: {
name: 'presto',
host: 'coordinator.p.cluster.local',
port: 8080,
user: 'shib',
catalog: 'hive2', // one engine config per catalogs
support_database: true,
default_database: 'default',
query_timeout: 30, // overwrite global config
setup_queries: [] // overwrite global config
},
monitor: null
}
],
};
```
### Access Control
Shib have access control list for Databases/Tables. Default is 'allow' for all databases/tables.
Shib's access control rules are:
* configure per executer
* database level default ('allow' or 'deny') without any optional rules makes that database unvisible
* database level default + allow/deny table list makes its tables visible/unvisible
* in this case, this 'database' is visible
* default 'allow' or 'deny' decides visibilities of databases without any optional rules
'unvisible' databases and tables:
* are not shown in tables/partitions list and schema list
* cannot be queried by users (these queries always fails)
Access control options are written in 'executer' like this:
```js
executer: {
name: 'presto',
host: 'coordinator.p.cluster.local',
port: 8080,
catalog: 'hive',
support_database: true,
default_database: 'default',
query_timeout: 30,
setup_queries: [],
access_control: {
databases: {
secret: { default: "deny" },
member: { default: "deny", allow: ["users"] },
test: { default: "allow", deny: ["secretData", "userMaster"] },
},
default: "allow"
}
},
```
For more details, see [wiki: Access Control](https://github.com/tagomoris/shib/wiki/Access-Control).
## Monitors
`monitor` configurations are used to get query status and to kill queries.
### JobTracker (MRv1)
`jobtracker` monitor is available in MRv1 environment (w/ both of hiveserver and hiveserver2).
```js
monitor: {
name: 'jobtracker',
host: 'jobtracker.hostname.local',
port: 50030,
mapred: '/usr/bin/mapred' // 'mapred' in PATH by default
}
```
For this feature, shib should be executed by a user who can execute command `mapred job -kill JOB_ID`.
### YARN (MRv2)
`yarn` monitor is available in MRv2 environment (w/ both of hiveserver and hiveserver2).
```js
monitor: {
name: 'yarn',
host: 'resourcemanager.hostname.local',
port: 8088
}
```
In this case, shib kills query with Resource Manager REST API.
If you specify yarn command description, shib kills query with `yarn application -kill APP_ID`.
```js
monitor: {
name: 'yarn',
host: 'resourcemanager.hostname.local',
port: 8088,
yarn: '/usr/bin/yarn'
}
```
### Huahin Manager (obsolete)
For monitors in CDH4 + MRv1 environment, Huahin manager is available.
To show map/reduce status, and/or to kill actual map/reduce jobs behind hive query, shib can use Huahin Manager. Current version supports only 'Huahin Manager CDH4 + MRv1' only.
http://huahinframework.org/huahin-manager/
Configure `monitor` argument like below instead of `null`.
```js
monitor: {
name : 'huahin_mrv1',
host: 'localhost',
port: 9010
}
```
## Authentication
Shib have authentication to log who execute queries and to control accesses:
setup_queries_auth option means to specify queries executed before main query when authentication is required.
```js
var servers = exports.servers = {
listen: 3000,
fetch_lines: 1000, // lines per fetch in shib
query_timeout: null, // shib waits queries forever
setup_queries: [],
setup_queries_auth: ["set hive.mapred.mode=strict"],
storage: {
datadir: './var'
},
auth: {
type: 'http_basic_auth',
url: 'http://your.internal.protected.service.example.com/',
realm: '@your.service.example.com'
},
engines: [
{ label: 'mycluster1',
executer: {
name: 'hiveserver2',
host: 'hs2.mycluster1.local',
port: 10000,
username: 'hive',
support_database: true
},
monitor: null
},
],
};
```
For more details, see [wiki: Authentication](https://github.com/tagomoris/shib/wiki/Authentication).
## Miscellaneous configurations
### Disable "history" tab
Specify `disable_history: true` on `servers`.
```js
var servers = exports.servers = {
listen: 3000,
fetch_lines: 1000,
query_timeout: null, // seconds. (null:shib will wait query response infinitely).
setup_queries: [],
disable_history: true,
storage: {
datadir: './var'
},
```
## As HTTP Proxy for query engines
POST query string into `/execute` with some parameters.
```
curl -s -X POST -F 'querystring=SELECT COUNT(*) AS cnt FROM yourtable WHERE field="value"' http://shib.server.local:3000/execute | jq .
{
"queryid": "69927e67c5b1d5f665697943cc4867ec",
"results": [],
"dbname": "default",
"engine": "hiveserver",
"querystring": "SELECT COUNT(*) AS cnt FROM yourtable WHERE field=\"value\""
}
```
Specify `engineLabel` and `dbname` for non-default query engines and databases:
```
curl -s -X POST -F "engineLabel=presto" -F "dbname=testing" -F "querystring=SELECT COUNT(*) AS cnt FROM yourtable WHERE field='value'" http://shib.server.local:3000/execute
```
If you want not to add your query into history tab, specify 'scheduled':
```
curl -s -X POST -F "scheduled=true" -F "querystring=SELECT COUNT(*) AS cnt FROM yourtable WHERE field='value'" http://shib.server.local:3000/execute
```
Then, fetch query's status whenever you want.
```
curl -s http://shib.server.local:3000/status/69927e67c5b1d5f665697943cc4867ec
executed
```
Or get whole query object.
```
curl -s http://shib.server.local:3000/query/69927e67c5b1d5f665697943cc4867ec | jq .
{
"queryid": "69927e67c5b1d5f665697943cc4867ec",
"results": [
{
"resultid": "969629614dff69411a2f4f1733c9616a",
"executed_at": "Wed Feb 26 2014 16:02:00 GMT+0900 (JST)"
}
],
"dbname": "default",
"engine": "hiveserver",
"querystring": "SELECT COUNT(*) AS cnt FROM yourtable WHERE field=\"value\""
}
```
If this query object has `executed` status, or a member of `results`, you can fetch its result by `resultid`.
```
# if you want elasped time or bytes or lines or ....
curl -s http://shib.server.local:3000/result/969629614dff69411a2f4f1733c9616a | jq .
{
"schema": [
{
"type": "bigint",
"name": "cnt"
}
],
"completed_msec": 1393398893759,
"completed_at": "Wed Feb 26 2014 16:14:53 GMT+0900 (JST)",
"completed_time": null,
"bytes": 6,
"queryid": "69927e67c5b1d5f665697943cc4867ec",
"executed_time": null,
"executed_at": "Wed Feb 26 2014 16:14:52 GMT+0900 (JST)",
"executed_msec": 1393398892752,
"resultid": "969629614dff69411a2f4f1733c9616a",
"state": "done",
"error": "",
"lines": 2
}
# raw result data as TSV (fast)
curl -s http://shib.server.local:3000/download/tsv/969629614dff69411a2f4f1733c9616a | jq .
CNT
1234567
# or CSV (slow)
curl -s http://shib.server.local:3000/download/csv/969629614dff69411a2f4f1733c9616a | jq .
"CNT"
"1234567"
```
These HTTP requests/response are same with that javascript on browser does.
* * * * *
## TODO
* Paches are welcome!
## License
Copyright 2011- TAGOMORI Satoshi (tagomoris)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: app.js
================================================
var express = require('express'),
jade = require('jade'),
async = require('async'),
fs = require('fs'),
app = express();
var RECENT_FETCHES = 50;
var SHOW_RESULT_HEAD_LINES = 20;
var InvalidQueryError = require('shib/query').InvalidQueryError;
var SimpleCSVBuilder = require('shib/simple_csv_builder').SimpleCSVBuilder;
var shib = require('shib');
var config_package = {};
if (process.env.NODE_ENV) {
config_package = require('./' + process.env.NODE_ENV);
} else {
config_package = require('./config');
}
var servers = config_package.servers;
shib.init(servers);
function shutdown(signal){
shib.logger().info('Shutdown by signal', {signal: signal});
process.exit();
};
process.on('SIGINT', function(){ shutdown('SIGINT'); });
process.on('SIGHUP', function(){ shutdown('SIGHUP'); });
process.on('SIGQUIT', function(){ shutdown('SIGQUIT'); });
process.on('SIGTERM', function(){ shutdown('SIGTERM'); });
var runningQueries = {};
function error_handle(req, res, err){
shib.logger().error('Error in app', err);
if (err.stack)
shib.logger().error(err.stack);
res.send(err, 500);
};
function shibclient(req){
return shib.client({credential: shib.auth().credential(req)});
}
app.configure(function(){
app.use(express.static(__dirname + '/public'));
app.use(express.methodOverride());
app.use(express.urlencoded());
app.use(express.json());
app.use(express.bodyParser());
app.use(app.router);
app.set('view options', {layout: false});
app.set('port', (servers.listen || process.env.PORT || 3000));
});
app.use(function(err, req, res, next){
if (!err) {
next();
}
else {
shib.logger().error('ServerError', err);
err.stack.split("\n").forEach(function(line){
shib.logger().error(line);
});
res.send(500, 'Server Error');
}
});
app.configure('development', function(){
app.use(express.static(__dirname + '/public'));
app.use(express.errorHandler({ dumpExceptions: true, showStack: true }));
});
app.configure('production', function(){
app.use(express.logger('default'));
var onehour = 3600;
app.use(express.static(__dirname + '/public', { maxAge: onehour }));
app.use(express.errorHandler());
});
app.get('/', function(req, res){
var client = shibclient(req);
res.render(__dirname + '/views/index.jade');
client.end();
});
app.post('/auth', function(req, res){
var auth = shib.auth();
auth.check(req, function(err, result){
if (err) { error_handle(req, res, err); return; }
if (result) {
// if auth module is dumb, username becomes String('undefined') and password becomes String('undefined')
var authInfo = auth.crypto(result.username);
res.send(200, {authInfo: authInfo, realm: auth.realm, enabled: auth.enabled});
} else {
res.send(403, {authInfo: null, realm: auth.realm, enabled: auth.enabled});
}
});
});
app.get('/q/:queryid', function(req, res){
// Only this request handler is for permalink request from browser URL bar.
var client = shibclient(req);
res.render(__dirname + '/views/index.jade');
client.end();
});
app.get('/t/:tag', function(req, res){
// Only this request handler is for permalink request from browser URL bar.
var client = shibclient(req);
res.render(__dirname + '/views/index.jade');
client.end();
});
app.get('/runnings', function(req, res){
var runnings = [];
for (var queryid in runningQueries) {
var times = (function(){
var secs = Math.floor(((new Date()) - runningQueries[queryid]) / 1000);
if (secs < 60)
return secs + ' seconds';
var mins = Math.floor(secs / 60);
if (mins < 60)
return mins + ' minutes';
var hours = Math.floor(mins / 60);
return hours + ' hours';
})();
runnings.push([queryid, times]);
}
res.send(runnings);
});
var enginesCache = null;
app.get('/engines', function(req, res){
/*
// "monitor" means support 'status' (and 'kill') or not.
{
pairs: [ [engine_label, dbname], [engine_label, dbname], ... ],
monitor: { label: bool }
}
*/
// If authentications are always required, database list cannot be cached.
if (shib.auth().require_always) {
shibclient(req).engineInfo(function(err, info){
if (err) { error_handle(req, res, err); this.end(); return; }
res.send(info);
this.end();
});
return;
}
var info = enginesCache;
if (info) {
res.send(info);
}
else {
shibclient(req).engineInfo(function(err, info){
if (err) { error_handle(req, res, err); this.end(); return; }
enginesCache = info;
res.send(info);
this.end();
});
}
});
app.get('/tables', function(req, res){
var client = shibclient(req);
var engineLabel = req.query.engine;
var database = req.query.db;
client.tables(engineLabel, database, function(err, result){
if (err) { error_handle(req, res, err); this.end(); return; }
res.send(result);
client.end();
});
});
app.get('/partitions', function(req, res){
var engineLabel = req.query.engine;
var database = req.query.db;
var tablename = req.query.key;
if (/^[a-z0-9_]+$/i.exec(tablename) == null) {
error_handle(req, res, {message: 'invalid tablename for show partitions: ' + tablename});
return;
}
var client = shibclient(req);
client.partitions(engineLabel, database, tablename, function(err, results){
if (err) { error_handle(req, res, err); client.end(); return; }
res.send(results);
client.end();
});
});
var describe_node_template = 'tr\n td= colname\n td= coltype\n td= colcomment\n';
app.get('/describe', function(req, res){
var engineLabel = req.query.engine;
var database = req.query.db;
var tablename = req.query.key;
if (/^[a-z0-9_]+$/i.exec(tablename) == null) {
error_handle(req, res, {message: 'invalid tablename for show partitions: ' + tablename});
return;
}
var fn = jade.compile(describe_node_template);
var client = shibclient(req);
client.describe(engineLabel, database, tablename, function(err, result){
if (err) { error_handle(req, res, err); client.end(); return; }
var response_html = '<tr><th>col_name</th><th>type</th><th>comment</th></tr>';
result.forEach(function(cols){
response_html += fn.call(this, {colname: cols[0], coltype: cols[1], colcomment: cols[2]});
});
res.send([{title: '<table>' + response_html + '</table>'}]);
client.end();
});
});
app.get('/summary_bulk', function(req, res){
var client = shibclient(req);
if (shib.setting('disable_history')) {
res.send({disabled: true});
return;
}
var history_queries;
var history = []; /* ["201302", "201301", "201212", "201211"] */
var history_ids = {}; /* {"201302":[query_ids], "201301":[query_ids], ...} */
var query_ids = []; /* [query_ids of all months] */
var fetchRecent = function(cb){
client.recentQueries(RECENT_FETCHES, function(err, list){ // list: [Query, ...]
if (err) { cb(err); return; }
history_queries = list;
cb(null);
});
};
var bundleMonths = function(cb){ // [{yyyymm:...,queryid:....}] => {yyyymm:[queryids], yyyymm:[queryids]}
var pad = function(n){return n < 10 ? '0'+n : n;};
var historyKey = function(date){
return '' + date.getFullYear() + pad(date.getMonth() + 1);
};
history_queries.forEach(function(query){
var month = historyKey(query.datetime);
if (history.indexOf(month) < 0)
history.push(month);
if (! history_ids[month])
history_ids[month] = [];
history_ids[month].push(query.queryid);
query_ids.push(query.queryid);
});
history.sort().reverse();
cb(null);
};
var queryUnique = function(cb){
var exist_ids = {};
query_ids = query_ids.filter(function(id){ if (exist_ids[id]) return false; exist_ids[id] = true; return true;});
cb(null);
};
async.series([fetchRecent, bundleMonths, queryUnique], function(err, results){
if (err) { error_handle(req, res, err); return; }
res.send({history: history, history_ids: history_ids, query_ids: query_ids});
});
});
// generate pseudo query object (simulate v0 query)
function pseudo_query_data(query){
var q = {};
q['queryid'] = query.queryid;
q['engine'] = query.engine;
q['dbname'] = query.dbname;
q['querystring'] = query.querystring;
q['results'] = [];
if (query.state !== "running") {
q['results'] = [{resultid: query.resultid, executed_at: new Date(query.datetime).toLocaleString()}];
}
return q;
}
app.post('/execute', function(req, res){
var auth = shib.auth();
var client = shibclient(req);
var engineLabel = req.body.engineLabel;
var dbname = req.body.dbname;
if (!engineLabel || !dbname) {
engineLabel = enginesCache.pairs[0][0];
dbname = enginesCache.pairs[0][1];
}
var querystring = req.body.querystring;
var scheduled = req.body.scheduled;
var userdata = auth.userdata(req);
if (auth.require_always && !userdata) {
shib.logger().warn('This shib requires authentication to execute queries, but there are no authInfo', {query: querystring});
res.send(403, "Authentication failed");
return;
}
if (userdata) {
shib.logger().info('User try to execute query', {username: userdata.username, query: querystring});
}
client.createQuery(engineLabel, dbname, querystring, scheduled, userdata, function(err, query){
if (err) {
if (err.error) {
err = err.error;
}
if (err instanceof InvalidQueryError) {
shib.logger().warn('Invalid query submitted', {query: querystring, err: err.message});
res.send(400, err.message);
this.end();
return;
}
error_handle(req, res, err); return;
}
res.send(pseudo_query_data(query));
var queryid = query.queryid;
this.execute(query, {
prepare: function(){ runningQueries[queryid] = new Date(); },
stopCheck: function(){ return (! runningQueries[queryid]); },
stop: function(){ client.end(); },
success: function(){ delete runningQueries[queryid]; client.end(); },
error: function(){ delete runningQueries[queryid]; client.end(); }
});
});
});
app.post('/giveup', function(req, res){
var targetid = req.body.queryid;
var client = shibclient(req);
client.query(targetid, function(err, query){
client.giveup(query, function(err, query) {
if (err) {error_handle(req, res, err); client.end(); return;}
client.end(true); // half close
delete runningQueries[query.queryid];
res.send(pseudo_query_data(query));
});
});
});
app.post('/delete', function(req, res){
var targetid = req.body.queryid;
var client = shibclient(req);
delete runningQueries[targetid];
client.deleteQuery(targetid, function(err){
if (err) {error_handle(req, res, err); client.end(); return;}
res.send({result:'ok'});
client.end();
});
});
//TODO: any APIs w/o pseudo_query_data ?
app.get('/query/:queryid', function(req, res){
shibclient(req).query(req.params.queryid, function(err, query){
if (err) { error_handle(req, res, err); this.end(); return; }
if (query === null) {
res.send('query not found', 404);
this.end();
return;
}
res.send(pseudo_query_data(query));
this.end();
});
});
app.post('/queries', function(req, res){
shibclient(req).queries(req.body.ids, function(err, queries){
if (err) { error_handle(req, res, err); this.end(); return; }
res.send({queries: queries.map(function(q){ return pseudo_query_data(q); })});
this.end();
});
});
app.get('/tags/:queryid', function(req, res){
shibclient(req).tags(req.params.queryid, function(err, tags){
if (err) { error_handle(req, res, err); this.end(); return; }
res.send(tags);
this.end();
});
});
app.post('/addtag', function(req, res){
var tag = req.body.tag;
if (tag.length < 1 || tag.length > 16) {
error_handle(req, res, {message: 'invalid tag length [1-16]'});
return;
}
shibclient(req).addTag(req.body.queryid, tag, function(err){
if (err) { error_handle(req, res, err); this.end(); return; }
res.send({result:'ok'});
this.end();
});
});
app.post('/deletetag', function(req, res){
shibclient(req).deleteTag(req.body.queryid, req.body.tag, function(err){
if (err) { error_handle(req, res, err); this.end(); return; }
res.send({result:'ok'});
this.end();
});
});
app.get('/tagged/:tag', function(req, res){
shibclient(req).taggedQueries(req.params.tag, function(err, queryids){
if (err) { error_handle(req, res, err); this.end(); return; }
res.send(queryids);
this.end();
});
});
app.get('/taglist', function(req, res){
shibclient(req).tagList(function(err, tags){
if (err) { error_handle(req, res, err); this.end(); return; }
res.send(tags);
this.end();
});
});
// convert query.state into status label (simulate v0 Client.prototype.status)
function pseudo_status(query_state){
/*
running: newest-and-only query running, and result not stored yet.
executed (done): newest query executed, and result stored.
error: newest query executed, but done with error.
*/
switch (query_state) {
case 'running': return "running";
case 'error': return "error";
default: return "executed";
}
}
app.get('/status/:queryid', function(req, res){
shibclient(req).query(req.params.queryid, function(err, query){
if (err) { error_handle(req, res, err); this.end(); return; }
if (query === null) {
res.send('query not found', 404);
this.end();
return;
}
res.send(pseudo_status(query.state));
this.end();
});
});
app.get('/detailstatus/:queryid', function(req, res){
shibclient(req).query(req.params.queryid, function(err, query){
if (err) { error_handle(req, res, err); this.end(); return; }
this.detailStatus(query, function(err, data){
if (err) { error_handle(req, res, err); this.end(); return; }
if (data === null)
res.send('query not found', 404);
else
res.send(data);
this.end();
});
});
});
// generate pseudo result object (simulate v0 result)
function pseudo_result_data(query){
var r = query.result;
r['resultid'] = query.resultid;
r['executed_at'] = new Date(query.datetime).toLocaleString();
if (r['completed_at'])
r['completed_at'] = new Date(r['completed_at']).toLocaleString();
r['state'] = query.state;
return r;
}
app.get('/lastresult/:queryid', function(req, res){
shibclient(req).query(req.params.queryid, function(err, query){
if (err) { error_handle(req, res, err); this.end(); return; }
if (query === null) {
res.send('query not found', 404);
this.end();
return;
}
res.send(pseudo_result_data(query));
this.end();
});
});
app.get('/result/:resultid', function(req, res){
shibclient(req).getQueryByResultId(req.params.resultid, function(err, query){
if (err) { error_handle(req, res, err); this.end(); return; }
if (query === null) {
res.send('query not found', 404);
this.end();
return;
}
res.send(pseudo_result_data(query));
this.end();
});
});
app.post('/results', function(req, res){
/*
* obsolete: bad implementation for API compatibility
*/
var client = shibclient(req);
var fetchers = (req.body.ids || []).map(function(resultid){
return function(cb){
client.getQueryByResultId(resultid, function(err, query){
if (query === null) {
res.send('query not found', 404);
this.end();
return;
}
if (err) { cb(err); return; }
cb(null, pseudo_result_data(query));
});
};
});
async.series(fetchers, function(err, results){
if (err) { error_handle(req, res, err); client.end(); return; }
res.send({results: results});
client.end();
});
});
app.get('/show/full/:resultid', function(req, res){
var client = shibclient(req);
var file = client.generatePath(req.params.resultid);
if (! fs.existsSync(file)) {
res.send(null);
res.end();
client.end();
return;
}
res.sendfile(file)
});
app.get('/show/head/:resultid', function(req, res){
var client = shibclient(req);
var file = client.generatePath(req.params.resultid);
if (! fs.existsSync(file)) {
res.send(null);
res.end();
client.end();
return;
}
var rStream = fs.createReadStream(file);
var readline = require('readline');
var rl = readline.createInterface(rStream, {});
var line_number = 0;
rl.on('line', function(line) {
if (line_number < SHOW_RESULT_HEAD_LINES) {
res.write(line + '\n');
line_number++;
} else {
rl.close();
}
});
rl.on('close', function() {
res.end();
client.end();
});
res.on('resume', function() {
rl.resume();
});
});
app.get('/download/tsv/:resultid', function(req, res){
shibclient(req).getQueryByResultId(req.params.resultid, function(err, query){
if (err) { error_handle(req, res, err); this.end(); return; }
if (query === null) {
res.send(null);
this.end();
return;
}
res.attachment(req.params.resultid + '.tsv');
res.set('X-Shib-Query-ID', query.queryid);
res.set('X-Shib-Result-ID', query.resultid);
res.set('X-Shib-Executed-At', new Date(query.datetime).getTime());
res.set('X-Shib-Completed-At', query.result.completed_msec || 0);
var client = shib.client();
var file = client.generatePath(req.params.resultid);
if (! fs.existsSync(file)) {
res.send(null);
res.end();
client.end();
return;
}
res.sendfile(file)
});
});
app.get('/download/csv/:resultid', function(req, res){
shibclient(req).getQueryByResultId(req.params.resultid, function(err, query){
if (err) { error_handle(req, res, err); this.end(); return; }
if (query === null) {
res.send(null);
this.end();
return;
}
res.attachment(req.params.resultid + '.csv');
res.set('X-Shib-Query-ID', query.queryid);
res.set('X-Shib-Result-ID', query.resultid);
res.set('X-Shib-Executed-At', new Date(query.datetime).getTime());
res.set('X-Shib-Completed-At', query.result.completed_msec || 0);
var client = shib.client();
var file = client.generatePath(req.params.resultid);
if (! fs.existsSync(file)) {
res.send(null);
res.end();
client.end();
return;
}
var rStream = fs.createReadStream(file);
var readline = require('readline');
var rl = readline.createInterface(rStream, {});
rl.on('line', function(line){
res.write(SimpleCSVBuilder.build(line.split('\t')));
});
rl.on('close', function(){
res.end();
client.end();
});
res.on('resume', function(){
rl.resume();
});
});
});
shib.auth(); // to initialize and check auth module
shib.client().end(); // to initialize sqlite3 database
shib.logger().info('Starting shib.');
if (! shib.auth().require_always){
var d = require('domain').create();
d.on('error', function(err){
// Failed to update engine cache
// This may occur by communication errors w/ servers
if (err.code === 'ECONNREFUSED') {
var e = err.domainEmitter;
shib.logger().error('Connection refused', {host: e.host, port: e.port});
} else {
shib.logger().error('In domain creation', err);
}
process.exit();
});
d.run(function(){
var enginesCacheUpdate = function(){
var AccessControl = require('shib/access_control').AccessControl;
// generate cache w/ all engines and databases forcely
// this cache isn't used for the situation configured as auth:require_always
shib.client({credential: AccessControl.defaultAllowDelegator()}).engineInfo(function(err, info){
if (!err && info)
enginesCache = info;
this.end();
});
};
setInterval(enginesCacheUpdate, 60*60*1000); // cache update per hour
enginesCacheUpdate();
});
}
app.listen(app.get('port'));
================================================
FILE: bin/dbmigrate.js
================================================
var async = require('async')
, fs = require('fs')
, sqlite3 = require('sqlite3');
var target_db_path = process.argv[2];
if (! target_db_path) {
console.log("USAGE: npm run migrate");
console.log(" OR: npm run dbmigrate -- DATABASE_FILE_PATH");
console.log(" : node bin/dbmigrate.js DATABASE_FILE_PATH (if your npm --version is 1.x)");
process.exit(0);
}
var migrate_db_path = target_db_path + ".migrate";
/*
var SQLITE_TABLE_DEFINITIONS_v0 = [
'CREATE TABLE IF NOT EXISTS queries (id VARCHAR(32) NOT NULL PRIMARY KEY, json TEXT NOT NULL)',
'CREATE TABLE IF NOT EXISTS results (id VARCHAR(32) NOT NULL PRIMARY KEY, json TEXT NOT NULL)',
'CREATE TABLE IF NOT EXISTS history (id INTEGER PRIMARY KEY AUTOINCREMENT, yyyymm VARCHAR(6) NOT NULL, queryid VARCHAR(32) NOT NULL)',
'CREATE TABLE IF NOT EXISTS tags (id INTEGER PRIMARY KEY AUTOINCREMENT, queryid VARCHAR(32) NOT NULL, tag VARCHAR(16) NOT NULL)'
];
*/
var SQLITE_TABLE_DEFINITIONS_v1 = [
'CREATE TABLE IF NOT EXISTS queries (autoid INTEGER PRIMARY KEY AUTOINCREMENT, id VARCHAR(32) NOT NULL UNIQUE, datetime TEXT NOT NULL, scheduled INTEGER DEFAULT NULL, engine TEXT DEFAULT NULL, dbname DEFAULT NULL, expression TEXT NOT NULL, state VARCHAR(32) NOT NULL, resultid VARCHAR(32) NOT NULL UNIQUE, result DEFAULT NULL)',
'CREATE TABLE IF NOT EXISTS tags (id INTEGER PRIMARY KEY AUTOINCREMENT, queryid VARCHAR(32) NOT NULL, tag VARCHAR(16) NOT NULL)'
];
var on_connect = function(cb){
async.series(SQLITE_TABLE_DEFINITIONS_v1.map(function(sql){
return function(callback) {
migrate.run(sql, function(error){
if (error) { callback(error.message); return; }
callback(null);
});
};
}), function(err,results){
if (err)
throw "failed to initialize new db file: " + migrate_db_path;
cb();
});
};
var original;
var migrate;
var open_original = function(cb){
original = new sqlite3.Database(target_db_path, sqlite3.OPEN_READONLY, function(err){
if (err) { cb(err); return; }
cb(null);
});
};
var close_original = function(cb){
original.close(function(){ cb(null); });
};
var open_migrate = function(cb){
migrate = new sqlite3.Database(migrate_db_path, (sqlite3.OPEN_READWRITE | sqlite3.OPEN_CREATE), function(err){
if (err) { cb(err); return; }
on_connect(function(){ cb(null); });
});
};
var close_migrate = function(cb){
migrate.close(function(){ cb(null); });
};
var migrate_tags = function(cb){
original.all('SELECT queryid, tag FROM tags ORDER BY id', function(err, rows){
if (err) {
cb(err);
return;
}
async.series(rows.map(function(row){
return function(cb){
migrate.run('INSERT INTO tags (queryid, tag) VALUES (?,?)', [row.queryid, row.tag], function(err){
cb(err);
});
};
}), function(err, results){
cb(err);
});
});
};
var history = {}; // queryid => true
var store_history = function(cb){
// 'CREATE TABLE IF NOT EXISTS history (id INTEGER PRIMARY KEY AUTOINCREMENT, yyyymm VARCHAR(6) NOT NULL, queryid VARCHAR(32) NOT NULL)',
original.all('SELECT yyyymm, queryid FROM history', function(err, rows){
if (err) { cb(err); return; }
var size = rows.length - 1;
for (var i = 0 ; i < size ; i++){
history[rows[i].queryid] = true;
}
cb(null);
});
};
var results = {}; // resultid -> obj
var store_results = function(cb){
original.all('SELECT id, json FROM results', function(err, rows){
if (err) { cb(err); return; }
var size = rows.length - 1;
var row = null;
for (var i = 0 ; i < size ; i++){
row = rows[i];
results[row.id] = row.json;
};
cb(null);
});
};
var migrate_queries = function(cb){
original.all('SELECT id, json FROM queries', function(err, rows_original){
if (err) {
cb(err);
return;
}
var rows = rows_original.filter(function(row){
var json = row.json;
if (json) {
var r = JSON.parse(json).results.concat().pop();
if (r && results[r.resultid])
return true;
}
return false;
}).map(function(row){
var obj = JSON.parse(row.json);
var result = obj.results.pop();
var result_obj = JSON.parse(results[result.resultid]);
var state = result_obj['state'];
delete result_obj['state'];
delete result_obj['queryid'];
delete result_obj['resultid'];
return {
id: row.id,
scheduled: (history[row.id] ? 1 : null),
engine: obj.engine,
dbname: obj.dbname,
expression: obj.querystring,
state: state,
resultid: result.resultid,
result_json: JSON.stringify(result_obj),
date: new Date(result.executed_at).toJSON()
};
});
rows.sort(function(a, b){ return a.date - b.date; });
async.series(rows.map(function(obj){
return function(cb) {
migrate.run(
'INSERT INTO queries (id,datetime,scheduled,engine,dbname,expression,state,resultid,result) VALUES (?,?,?,?,?,?,?,?,?)',
[obj.id, obj.date, obj.scheduled, obj.engine, obj.dbname, obj.expression, obj.state, obj.resultid, obj.result_json],
function(err){ cb(err); }
);
};
}), function(err, results){ cb(err); });
});
};
async.series([
open_original,
open_migrate,
migrate_tags,
store_history,
store_results,
migrate_queries,
close_original,
close_migrate
], function(err, results){
if (err) {
console.log(err);
throw "failed to migrate database...";
}
fs.renameSync(target_db_path, target_db_path + ".v0");
fs.renameSync(migrate_db_path, target_db_path);
});
================================================
FILE: bin/purge.js
================================================
var async = require('async')
, fs = require('fs')
, sqlite3 = require('sqlite3');
if (process.argv.length < 4) {
console.log("USAGE: npm run purge -- DAYS_PURGE_BEFORE");
console.log(" OR: node bin/purge.js DATABASE_FILE_PATH DAYS_PURGE_BEFORE (if your npm --version is 1.x)");
console.log("");
console.log("NOTICE: result data files under DATADIR/results should be removed by yourself.");
console.log(" (ex: find var/results -mtime ... | xargs rm)");
process.exit(0);
}
var target_db_path = process.argv[2];
var purge_days_before = parseInt(process.argv[3]);
if (purge_days_before < 3) {
console.log("ERROR: DAYS_PURGE_BEFORE must be larger than 7, but: " + process.argv[3]);
process.exit(1);
}
var SQLITE_TABLE_DEFINITIONS_v1 = [
'CREATE TABLE IF NOT EXISTS queries (autoid INTEGER PRIMARY KEY AUTOINCREMENT, id VARCHAR(32) NOT NULL UNIQUE, datetime TEXT NOT NULL, engine TEXT DEFAULT NULL, dbname DEFAULT NULL, expression TEXT NOT NULL, state VARCHAR(32) NOT NULL, resultid VARCHAR(32) NOT NULL UNIQUE, result DEFAULT NULL)',
'CREATE TABLE IF NOT EXISTS tags (id INTEGER PRIMARY KEY AUTOINCREMENT, queryid VARCHAR(32) NOT NULL, tag VARCHAR(16) NOT NULL)'
];
var db = null;
var open_db = function(cb){
db = new sqlite3.Database(target_db_path, (sqlite3.OPEN_READWRITE | sqlite3.OPEN_CREATE), function(err){
cb(err);
});
};
var close_db = function(cb){
db.close(function(){ cb(null); });
};
var delete_rows = function(cb){
var purge_threshold_datetime = new Date(new Date() - purge_days_before * 1000 * 86400).toJSON();
db.run('DELETE FROM queries WHERE datetime < ?', [purge_threshold_datetime], function(err){ cb(err); });
};
var vacuum = function(cb){
db.run('VACUUM', function(err){ cb(err); });
};
async.series([
open_db,
delete_rows,
vacuum,
close_db
], function(err, results){
if (err) {
console.log(err);
throw "failed to purge old data and vacuum...";
}
});
================================================
FILE: config.js
================================================
var servers = exports.servers = {
listen: 3000,
fetch_lines: 1000,
query_timeout: null, // seconds. (null:shib will wait query response infinitely).
setup_queries: [],
storage: {
datadir: './var'
},
executer: {
name: 'hiveserver', // or 'hiveserver2', 'presto', 'bigquery'
host: 'localhost',
port: 10000,
support_database: true,
default_database: 'default'
/*
// If you use 'bigquery' executer, set following values
project_id: 'project_id',
key_filename: '/path/to/keyfile.json'
*/
},
monitor: null
/*
monitor: {
name : 'huahin_mrv1', // or 'presto'
host: 'localhost',
port: 9010
}
*/
};
================================================
FILE: examples/hive_beeline_client.js
================================================
// NODE_PATH=lib node example/hive_beeline_client.js
var thrift = require('node-thrift')
, ttransport = require('node-thrift/lib/thrift/transport')
, TCLIService = require('shib/engines/hiveserver2/TCLIService')
, TTypes = require('shib/engines/hiveserver2/TCLIService_types');
var MaxRows = 10000;
var fetchOperationResult = function(op, err, res, callback) {
console.log({seq:'fetchOperationResult', op:op, err:err, res:res});
if (err) { callback(err); return; }
if (!res || !res['status'] || res.status['statusCode'] !== 0) {
callback({err:'Failed to operate "' + op + '"', res:res, status: (res && res['status'] && res.status['statusCode'])});
return;
}
var oph = res.operationHandle;
var POLLING_INTERVAL = 50; // 50ms
var poller = function(){
console.log({seq:'GetOperationStatus', op:op});
client.GetOperationStatus(new TTypes.TGetOperationStatusReq({operationHandle:oph}), function(err, res){
console.log({seq:'GetOperationStatus', op:op, err:err, res:res});
if (err) { callback(err); return; }
var statusCode = res && res['status'] && res.status.statusCode;
if (statusCode === TTypes.TStatusCode['STILL_EXECUTING_STATUS']) {
setTimeout(poller, POLLING_INTERVAL);
return;
}
if (statusCode !== TTypes.TStatusCode['SUCCESS_STATUS'] &&
statusCode !== TTypes.TStatusCode['STATUS_WITH_INFO_STATUS']) {
if (statusCode === TTypes.TStatusCode['ERROR_STATUS']) {
callback({err:'ERROR_STATUS', status:res.status});
} else if (statusCode === TTypes.TStatusCode['INVALID_HANDLE_STATUS']) {
callback({err:'INVALID_HANDLE_STATUS', status:res.status});
} else {
callback({err:'UNKNOWN STATUS:' + statusCode, status:res.status});
}
client.CloseOperation(new TTypes.TCloseOperationReq({operationHandle:oph}), function(err, res){
return err || res && res['status'];
});
}
/* success */
if (res.status.statusCode === TTypes.TStatusCode['STATUS_WITH_INFO_STATUS']) {
console.log({req:'withInfoStatus', res:res}); // TODO: what is WITH_INFO_STATUS ?
}
client.FetchResults(new TTypes.TFetchResultsReq({operationHandle:oph, maxRows:MaxRows}), function(err, res){
if (err) { callback(err); return; }
console.log({seq:'FetchResults', op:op, err:err, res:res});
if (res.hasMoreRows)
throw "Operation hasMoreRow, type:" + oph.operationType;
var results = res.results;
client.CloseOperation(new TTypes.TCloseOperationReq({operationHandle:oph}), function(err, res){
callback(null, results);
return;
});
});
});
};
setTimeout(poller, POLLING_INTERVAL);
};
/* hive.server2.authentication='NOSASL' or SASLTransport ... */
var connection = thrift.createConnection("server.name.local", 10001, {transport: ttransport.TBufferedTransport, timeout: 600*1000});
var client = thrift.createClient(TCLIService, connection);
connection.on('error', function(err){ console.error(err); });
connection.addListener("connect", function(){ console.log("connected"); });
client.OpenSession(new TTypes.TOpenSessionReq({username: '', password: ''}), function(err,res){
var sessionHandle = res.sessionHandle;
var treq = new TTypes.TGetTablesReq({sessionHandle:sessionHandle});
client.GetTables(treq, function(e,r){ fetchOperationResult('GetTables', e, r, function(err, res){
console.log([err, res]);
console.log(JSON.stringify(res.rows, null, 4));
var sreq = new TTypes.TGetSchemasReq({sessionHandle:sessionHandle});
client.GetSchemas(sreq, function(e,r){ fetchOperationResult('GetSchemas', e, r, function(err, res){
console.log([err, res]);
console.log(JSON.stringify(res.rows, null, 4));
var creq = new TTypes.TCloseSessionReq({sessionHandle:sessionHandle});
client.CloseSession(creq, function(e,r){ connection.end(); });
}); });
}); });
/*
var q = 'select service, count(*) as cnt from access_log where service="servic_ename" and yyyymmdd="20130210" group by service';
var req = new TTypes.TExecuteStatementReq({sessionHandle: sessionHandle, statement: q});
client.ExecuteStatement(req, function(e,r){ fetchOperationResult('ExecuteStatement', e, r, function(err,rows){
console.log({err:err, rows:rows});
var creq = new TTypes.TCloseSessionReq({sessionHandle:sessionHandle});
client.CloseSession(creq, function(e,r){ connection.end(); });
}); });
*/
});
/*
http://www.slideshare.net/schubertzhang/hiveserver2/7
ExecuteStatement
TExecuteStatementReq(sessionHandle, statement, confOverlay)
TExecuteStatementResp(status, operationHandle)
GetResultSetMetadata
TGetResultSetMetadataReq(operationHandle)
TGetResultSetMetadataResp(status, schema)
FetchResults
TFetchResultsReq(operationHandle, orientation = 0, maxRows)
TFetchResultsResp(status, hasMoreRows, results)
GetCatalogs : "Catalog: do nothing so far"
TGetCatalogsReq(sessionHandle)
TGetCatalogsResp(status, operationHandle)
GetSchemas
TGetSchemasReq(sessionHandle, catalogName, schemaName)
TGetSchemasResp(status, operationHandle)
GetTableTypes
TGetTableTypesReq(sessionHandle)
TGetTableTypesResp(status, operationHandle)
GetTables
TGetTablesReq(sessionHandle, catalogName, schemaName, tableName, tableTypes)
TGetTablesResp(status, operationHandle)
GetColumns
TGetColumnsReq(sessionHandle, catalogName, schemaName, tableName, columnName)
TGetColumnsResp(status, operationHandle)
GetFunctions
TGetFunctionsReq(sessionHandle, catalogName, schemaName, functionName)
TGetFunctionsResp(status, operationHandle)
OpenSession
TOpenSessionReq(username, password, configuration)
TOpenSessionResp(status, serverProtocol, sessionHandle, configuration)
CloseSession
TCloseSessionReq(sessionHandle)
GetInfo
TGetInfoReq(sessionHandle, infoType)
TGetInfoResp(status, infoValue)
GetTypeInfo
TGetTypeInfoReq(sessionHandle)
TGetTypeInfoResp(status, operationHandle)
GetOperationStatus
TGetOperationStatusReq(operationHandle)
TGetOperationStatusResp(status, operationState)
GetCancelOperation
TCancelOperationReq(operationHandle)
TCancelOperationResp(status)
CloseOperation
TCloseOperationReq(operationHandle)
TCloseOperationResp(status)
オマケ: SessionHandle と OperationHandle
TSessionHandle(sessionId)
TOperationHandle(operationId, operationType, hasResultSet, modifiedRowCount)
*/
================================================
FILE: examples/hive_client.js
================================================
var thrift = require('thrift'),
ttransport = require('thrift/transport'),
ThriftHive = require('./gen-nodejs/ThriftHive');
var connection = thrift.createConnection("localhost", 10000, {transport: ttransport.TBufferedTransport, timeout: 600*1000}),
client = thrift.createClient(ThriftHive, connection);
connection.on('error', function(err) {
console.error(err);
});
connection.addListener("connect", function() {
client.execute('select count(*) from p', function(err){
console.error("pos");
if (err) { console.error("error on execute(): " + err); process.exit(1); }
client.fetchAll(function(err, data){
if (err){ console.error("error on fetchAll(): " + err); process.exit(1); }
console.error(data);
connection.end();
process.exit(0);
});
});
});
================================================
FILE: examples/hive_methods.js
================================================
var thrift = require('thrift'),
ttransport = require('thrift/transport'),
ThriftHive = require('gen-nodejs/ThriftHive');
var connection = thrift.createConnection("localhost", 10000, {transport: ttransport.TBufferedTransport, timeout: 600*1000}),
client = thrift.createClient(ThriftHive, connection);
connection.on('error', function(err) {
console.error(err);
});
connection.addListener("connect", function() {
client.getClusterStatus(function(err, data){
console.log("getClusterStatus:", data);
client.execute('select x, count(*) as cnt from p group by x sort by cnt limit 10', function(err){
if (err) { console.error("error on execute(): " + err); process.exit(1); }
client.getQueryPlan(function(err, data){
console.log("getQueryPlan:", data);
console.log("queryplan queryAttributes:", data.queries[0].queryAttributes);
console.log("queryplan stageGraph:", data.queries[0].stageGraph);
console.log("queryplan stageGraph adjacencyList children:", data.queries[0].stageGraph.adjacencyList[0].children);
console.log("queryplan stageList:", data.queries[0].stageList);
console.log("queryplan stageList taskList:", data.queries[0].stageList[0].taskList[0]);
console.log("queryplan stageList taskList operatorGraph adjacencyList:", data.queries[0].stageList[0].taskList[0].operatorGraph.adjacencyList);
client.getSchema(function(err, data){
console.log("getSchema:", data);
client.getThriftSchema(function(err,data){
console.log("getThriftSchema:", data);
client.fetchAll(function(err, data){
if (err){ console.error("error on fetchAll(): " + err); process.exit(1); }
console.log("fetchAll:", data);
connection.end();
process.exit(0);
});
});
});
});
});
});
});
================================================
FILE: examples/hive_multi_client.js
================================================
var thrift = require('thrift'),
ttransport = require('thrift/transport'),
ThriftHive = require('gen-nodejs/ThriftHive');
var connection1 = thrift.createConnection("localhost", 10000, {transport: ttransport.TBufferedTransport, timeout: 1*1000}),
client1 = thrift.createClient(ThriftHive, connection1);
var connection2 = thrift.createConnection("localhost", 10000, {transport: ttransport.TBufferedTransport, timeout: 1*1000}),
client2 = thrift.createClient(ThriftHive, connection2);
connection1.on('error', function(err){ console.error(err); });
connection2.on('error', function(err){ console.error(err); });
var query1 = 'select x, count(*) from p group by x';
var query2 = 'select x, count(*) as cnt, "hoge" from p group by x sort by cnt desc limit 30';
var done1 = false;
var done2 = false;
var run_test = function(conn, client, query, label, wait, callback) {
var func = function(){
console.log(label, "executing.");
client.execute(query, function(err){
console.log(label, "executed.");
if (err){ console.error(label, "error on execute():", err); }
client.fetchAll(function(err, data){
console.log(label, "fetched.");
if (err){ console.error(label, "error on fetchAll():", err); }
console.log(label, "result:", data);
callback();
});
});
};
conn.addListener("connect", function(){
console.log(label, "connected.");
setTimeout(func, wait);
});
};
run_test(connection1, client1, query1, "conn1:", 5000, function(){ done1 = true; });
run_test(connection2, client2, query2, "conn2:", 1000, function(){ done2 = true; });
setInterval(function(){
if (done1 && done2){
connection1.end();
connection2.end();
process.exit(0);
}
}, 1000);
================================================
FILE: examples/hive_server2_engine.example.js
================================================
var engine = require('shib/engines/hiveserver2');
var executer = new engine.Executer({"name":"hiveserver2", "host":"hiveserver2.server.local", "port":10001});
executer.execute(
'jobname',
'select service, count(*) as cnt, false, 0.01, 1, array(1,2,3) as a from access_log where (service="blog" or service="news") and yyyymmdd="20121122" group by service',
function(err, fetcher){
console.log({op:'execute', err:err, fetcher:fetcher});
if (err) return;
fetcher.schema(function(err,schema){
if (err) {
console.log(JSON.stringify(err, null, " "));
return;
}
console.log(JSON.stringify(schema, null, " "));
fetcher.fetch(null, function(err, rows){
console.log({op:'fetch', err:err, rows:rows});
console.log(JSON.stringify(rows, null, " "));
});
});
}
);
executer.execute(
null,
'show tables',
function(err, fetcher){
console.log({op:'execute', err:err, fetcher:fetcher});
if (err) {
console.log(JSON.stringify(err, null, " "));
return;
}
fetcher.schema(function(err,res){
if (err) {
console.log(JSON.stringify(err, null, " "));
return;
}
console.log(JSON.stringify(res, null, " "));
fetcher.fetch(null, function(err, rows){
console.log({op:'fetch', err:err, rows:rows});
console.log(JSON.stringify(rows, null, " "));
});
});
}
);
executer.execute(
null,
'show tables',
function(err, fetcher){
console.log({op:'execute', err:err, fetcher:fetcher});
fetcher.fetch(null, function(err, rows){
console.log({op:'fetch', err:err, rows:rows});
});
}
);
executer.execute(
null,
'show tables',
function(err, fetcher){
console.log({op:'execute', err:err, fetcher:fetcher});
fetcher.schema(function(err, schema){
console.log({op:'schema', err:err, schema:schema});
fetcher.fetch(null, function(err, rows){
console.log({op:'fetch', err:err, rows:rows});
});
});
}
);
================================================
FILE: examples/hive_server2_results.js
================================================
/*
* GetTables
* rows: [ table_items ]
* table_items: [ ???(catalog?), database(schema?), tablename, tabletype, ??? ]
*/
/*
[
{
"colVals": [
{
"boolVal": null,
"byteVal": null,
"i16Val": null,
"i32Val": null,
"i64Val": null,
"doubleVal": null,
"stringVal": {
"value": ""
}
},
{
"boolVal": null,
"byteVal": null,
"i16Val": null,
"i32Val": null,
"i64Val": null,
"doubleVal": null,
"stringVal": {
"value": "default"
}
},
{
"boolVal": null,
"byteVal": null,
"i16Val": null,
"i32Val": null,
"i64Val": null,
"doubleVal": null,
"stringVal": {
"value": "access_log"
}
},
{
"boolVal": null,
"byteVal": null,
"i16Val": null,
"i32Val": null,
"i64Val": null,
"doubleVal": null,
"stringVal": {
"value": "MANAGED_TABLE"
}
},
{
"boolVal": null,
"byteVal": null,
"i16Val": null,
"i32Val": null,
"i64Val": null,
"doubleVal": null,
"stringVal": {
"value": null
}
}
]
},
{
"colVals": [
{
"boolVal": null,
"byteVal": null,
"i16Val": null,
"i32Val": null,
"i64Val": null,
"doubleVal": null,
"stringVal": {
"value": ""
}
},
{
"boolVal": null,
"byteVal": null,
"i16Val": null,
"i32Val": null,
"i64Val": null,
"doubleVal": null,
"stringVal": {
"value": "default"
}
},
{
"boolVal": null,
"byteVal": null,
"i16Val": null,
"i32Val": null,
"i64Val": null,
"doubleVal": null,
"stringVal": {
"value": "pageviews"
}
},
{
"boolVal": null,
"byteVal": null,
"i16Val": null,
"i32Val": null,
"i64Val": null,
"doubleVal": null,
"stringVal": {
"value": "MANAGED_TABLE"
}
},
{
"boolVal": null,
"byteVal": null,
"i16Val": null,
"i32Val": null,
"i64Val": null,
"doubleVal": null,
"stringVal": {
"value": null
}
}
]
},
{
"colVals": [
{
"boolVal": null,
"byteVal": null,
"i16Val": null,
"i32Val": null,
"i64Val": null,
"doubleVal": null,
"stringVal": {
"value": ""
}
},
{
"boolVal": null,
"byteVal": null,
"i16Val": null,
"i32Val": null,
"i64Val": null,
"doubleVal": null,
"stringVal": {
"value": "default"
}
},
{
"boolVal": null,
"byteVal": null,
"i16Val": null,
"i32Val": null,
"i64Val": null,
"doubleVal": null,
"stringVal": {
"value": "hourly_log"
}
},
{
"boolVal": null,
"byteVal": null,
"i16Val": null,
"i32Val": null,
"i64Val": null,
"doubleVal": null,
"stringVal": {
"value": "MANAGED_TABLE"
}
},
{
"boolVal": null,
"byteVal": null,
"i16Val": null,
"i32Val": null,
"i64Val": null,
"doubleVal": null,
"stringVal": {
"value": null
}
}
]
}
]
*/
/*
* GetSchemas
*/
/*
[
{
"colVals": [
{
"boolVal": null,
"byteVal": null,
"i16Val": null,
"i32Val": null,
"i64Val": null,
"doubleVal": null,
"stringVal": {
"value": "default"
}
},
{
"boolVal": null,
"byteVal": null,
"i16Val": null,
"i32Val": null,
"i64Val": null,
"doubleVal": null,
"stringVal": {
"value": ""
}
}
]
}
]
*/
================================================
FILE: lib/shib/access_control.js
================================================
var AccessControl = exports.AccessControl = function(config){
this.default_rule = config['default'] || "allow";
this.databases = config['databases'] || {};
};
AccessControl.prototype.allowed = function(tablename, dbname){
var dbconf = this.databases[dbname];
if (! dbconf)
return (this.default_rule === "allow");
return this.checkDBPrivilege(dbconf, tablename);
};
AccessControl.prototype.visible = function(dbname){
if (! this.databases[dbname])
return (this.default_rule === "allow");
// this.databases[dbname] exists
var dbrule = this.databases[dbname];
if (dbrule['default'] === "deny" && (dbrule.allow || []).length < 1)
// specified db configured db-scope denied
return false;
// visible tables are exists -> db visible
return true;
};
AccessControl.prototype.checkDBPrivilege = function(dbconf, tablename){
if (dbconf['default'] === "allow") {
var deny = dbconf.deny || [];
return ( deny.indexOf(tablename) === -1 );
} else { // default deny
var allow = dbconf.allow || [];
return ( allow.indexOf(tablename) >= 0 );
}
};
AccessControl.defaultDenyDelegator = function(){
var delegator = new function(){
this.allowed = function(t,d){return false;};
this.visible = function(d){return false;};
};
return delegator;
};
AccessControl.defaultAllowDelegator = function(){
var delegator = new function(){
this.allowed = function(t,d){return true;};
this.visible = function(d){return true;};
};
return delegator;
};
================================================
FILE: lib/shib/auth/dumb.js
================================================
var Auth = exports.Auth = function(args){
};
Auth.prototype.check = function(req, callback) {
callback(null, {username: req.body.username, password: req.body.password});
};
Auth.prototype.acl_delegators = function(required_always, username, req) {
// allowed(tablename, dbname), visible(dbname)
return [function(t,d){return true;}, function(d){return true;}];
};
================================================
FILE: lib/shib/auth/http_basic_auth.js
================================================
var url = require('url')
, http = require('http')
, https = require('https');
var AccessControl = require('shib/access_control').AccessControl;
var Auth = exports.Auth = function(args, logger){
this.logger = logger;
var parsed = {};
if (args.url) {
parsed = url.parse(args.url);
/*
{ protocol: 'http:',
slashes: true,
auth: null,
host: 'localhost',
port: null,
hostname: 'localhost',
hash: null,
search: null,
query: null,
pathname: '/path',
path: '/path',
href: 'http://localhost/path' }
*/
}
this._url = parsed.href || args.url;
this._method = args.method || 'GET';
this._hostname = args.host || parsed.hostname;
this._port = args.port || parsed.port || 80;
this._path = args.path || '/'; // including query string
var protocol = args.protocol || parsed.protocol || 'http';
this._client = http;
if (protocol === 'https' || protocol === 'https:')
this._client = https;
if (! this._hostname || ! this._path)
throw "basic auth target host/path not speicifed";
this._acl_config = args.access_control;
};
Auth.prototype.check = function(req, callback) {
var username = req.body.username;
var password = req.body.password;
if (!username || !password) {
callback(null, false);
return;
}
var auth_string = username + ':' + password;
var options = {
hostname: this._hostname,
port: this._port,
method: this._method,
path: this._path,
auth: auth_string
};
var authreq = this._client.request(options, function(res){
var success = false;
if (res.statusCode == 200) {
success = {username: username, password: password};
}
var waiting = true;
res.on('data', function(){
if (waiting) {
waiting = false;
callback(null, success);
}
});
});
authreq.on('error', function(e){ callback(e); });
authreq.end();
};
/*
var auth = {
type: 'http_basic_auth',
url: '....',
realm: '',
access_control: {
users: {
normal_user_name: {
databases: {
public: { default: "allow" }
},
default: "deny"
},
super_user_name: {
default: "allow"
}
},
default: "allow" // default
},
};
*/
Auth.prototype.acl_delegators = function(req, username, options) {
var acl_config = null;
if (username && this._acl_config && this._acl_config['users'] && this._acl_config['users'][username])
acl_config = this._acl_config['users'][username];
if (!acl_config) {
if (options.require_always || this._acl_config && this._acl_config['default'] === 'deny')
return [function(t,d){return false;}, function(d){return false;}];
return [function(t,d){return true;}, function(d){return true;}];
}
var acl = new AccessControl(acl_config);
// allowed(tablename, dbname), visible(dbname)
return [function(t,d){return acl.allowed(t,d);}, function(d){return acl.visible(d);}];
};
================================================
FILE: lib/shib/auth/http_custom_header.js
================================================
var AccessControl = require('shib/access_control').AccessControl;
/*
var auth = {
require_always: true,
type: 'http_custom_header',
realm: 'linecorp rev',
username: 'X-Shib-Auth-User',
groupname: 'X-Shib-Auth-Group',
access_control: {
users: {
tagomoris: {
default: "deny",
databases: {
default: { default: "allow" },
legy: { default: "deny", allow: ["access_logs"] }
}
},
hogepos: {
default: "allow"
}
},
groups: {
supermember: {
default: "allow"
},
limitedmember: {
default: "deny",
databases: {
default: { default: "allow" }
}
}
},
order: ["group", "user"], //default
default: "deny" //default
}
};
*/
var Auth = exports.Auth = function(args, logger){
this.logger = logger;
this.realm = args.realm;
if (! args.require_always)
throw "Auth 'http_custom_headers' does not permit 'require_always: false'";
if (! args.username)
throw "Auth 'http_custom_headers' requires username";
this._username = args.username;
this._groupname = args.groupname;
this._acl_config = args.access_control;
};
Auth.prototype.check = function(req, callback) {
var username = req.get(this._username);
var groupname = null;
if (this._groupname)
groupname = req.get(this._groupname);
if (!username && !groupname) {
callback(null, false);
return;
}
callback(null, {username: username});
};
// On this auth plugin, HTTP custom headers is the most important credential,
// rather than Shib auth headers.
Auth.prototype.acl_delegators = function(req, username, options) {
// [ allowed(tablename, dbname), visible(dbname) ]
if (!username)
username = req.get(this._username);
if (!username || username !== req.get(this._username))
return [function(t,d){return false;}, function(d){return false;}];
var groupname = null;
if (this._groupname)
groupname = req.get(this._groupname);
var aclconfig = this._acl_config;
var order = (aclconfig['order'] || ["group","user"]);
var acl = null;
order.forEach(function(order_item){
if (acl)
return; // continue
if (order_item === "group") {
if (! groupname)
return; // continue
var gdata = (aclconfig['groups'] || {})[groupname];
if (gdata)
acl = new AccessControl(gdata);
} else { // user
var udata = (aclconfig['users'] || {})[username];
if (udata)
acl = new AccessControl(udata);
}
});
if (acl)
return [function(t,d){return acl.allowed(t,d);}, function(d){return acl.visible(d);}];
if (aclconfig['default'] === "allow")
return [function(t,d){return true;}, function(d){return true;}];
// default deny
return [function(t,d){return false;}, function(d){return false;}];
};
================================================
FILE: lib/shib/auth.js
================================================
var crypto = require('crypto');
var dumb = require('./auth/dumb')
, http_basic_auth = require('./auth/http_basic_auth')
, http_custom_header = require('./auth/http_custom_header');
var CRYPTO_DEFAULT_CIPHER = 'aes192';
var passphrase = null;
var secret = null;
var Auth = exports.Auth = function(config, logger){
this.logger = logger;
if (!passphrase)
passphrase = crypto.pseudoRandomBytes(32).toString('binary');
if (!secret)
secret = crypto.pseudoRandomBytes(4).toString('hex');
this._provider = null;
this.enabled = false;
this.require_always = config.require_always || false;
this.realm = config.realm || '';
if (config.type === null || config.type === undefined){
this._provider = new dumb.Auth();
} else if (config.type === 'http_basic_auth'){
this._provider = new http_basic_auth.Auth(config, logger);
this.enabled = true;
} else if (config.type === 'http_custom_header'){
this._provider = new http_custom_header.Auth(config, logger);
this.enabled = true;
} else {
throw "unknown auth type name:" + config.type;
}
};
Auth.prototype.check = function(req, callback){
this._provider.check(req, callback);
};
Auth.prototype.credential = function(req){
var user = this.userdata(req);
var username = null;
if (user)
username = user.username;
return new Credential(this._provider.acl_delegators(req, username, {require_always: this.require_always}));
};
Auth.prototype.userdata = function(req){
var data = req.body.authInfo || req.get('X-Shib-AuthInfo');
if (!data)
return null;
return this.decrypto(data);
};
Auth.prototype.crypto = function(username){
var cipher = crypto.createCipher(CRYPTO_DEFAULT_CIPHER, passphrase);
var data = secret + ':' + String(username);
var buf0 = cipher.update(data, 'utf8', 'hex');
var buf1 = cipher.final('hex');
return buf0 + buf1;
};
Auth.prototype.decrypto = function(data){
var user;
var decipher = crypto.createDecipher(CRYPTO_DEFAULT_CIPHER, passphrase);
var r0 = decipher.update(data, 'hex', 'utf8');
try {
var r1 = decipher.final('utf8');
var ary = (r0 + r1).split(':');
if (ary[0] === secret) {
user = ary[1];
}
} catch (x) {
// TypeError: error:06065064:digital envelope routines:EVP_DecryptFinal_ex:bad decrypt
// decrypt error
}
if (!user)
return null;
return {username: user};
};
var Credential = function(acl_delegators){
this.allowed_delegator = acl_delegators[0];
this.visible_delegator = acl_delegators[1];
};
Credential.prototype.allowed = function(tablename, dbname){
return this.allowed_delegator(tablename, dbname);
};
Credential.prototype.visible = function(dbname){
return this.visible_delegator(dbname);
};
================================================
FILE: lib/shib/client.js
================================================
var Query = require('./query').Query;
var async = require('async');
var localdiskstore = require('./localdiskstore'),
LocalDiskStoreError = localdiskstore.LocalStoreError;
var engine = require('./engine');
var FETCH_LINES_DEFAULT = 1000;
var Client = exports.Client = function(args, logger, credential){
var self = this;
this.logger = logger;
this._conf = args;
this._localstore = undefined;
/*
* engine configurations defaults
* fetch_lines: 1000,
* query_timeout: null,
* setup_queries: [],
* setup_queries_auth: [],
* max_result_file_byte_size: null,
* default_database: 'default',
*
* these may be overwritten by each engine configurations
*/
this._default_fetch_lines = this._conf.fetch_lines || FETCH_LINES_DEFAULT;
this._default_query_timeout = this._conf.query_timeout || null;
this._default_setup_queries = this._conf.setup_queries || [];
this._default_setup_queries_auth = this._conf.setup_queries_auth || [];
this._default_max_result_file_byte_size = this._conf.max_result_file_byte_size || null;
this._default_default_database = this._conf.default_database || 'default';
this._default_engine_label = undefined;
this._engine_configs = {};
this._engines = {};
if (this._conf.executer) {
this._default_engine_label = 'default';
var engine_conf = {
label: 'default',
executer: this._conf.executer,
monitor: this._conf.monitor
};
this._engine_configs['default'] = engine_conf;
this._conf.engines = [engine_conf];
}
else if (this._conf.engines) {
this._default_engine_label = this._conf.engines[0].label;
this._conf.engines.forEach(function(e){
if (! e.label)
throw "label missing in engines configuration!";
if (! e.executer)
throw "executer not found in engines configuration!";
self._engine_configs[e.label] = e;
});
}
else
throw "engines configuration missing";
this.localStore(); // to initialize sqlite3 database
this.auth_credential = credential;
};
function error_callback(name, t, callback, err, data){
if (! callback) return;
if (data && data['ERROR'])
err.message += ' ERROR:' + data['ERROR'];
callback.apply(t, [err]);
};
Client.prototype.localStore = function(){
if (this._localstore) {
return this._localstore;
}
this._localstore = new localdiskstore.LocalDiskStore(this._conf.storage.datadir, this.logger);
return this._localstore;
};
Client.prototype.engineLabels = function(){
var self = this;
var labels = [];
this._conf.engines.forEach(function(e){
labels.push( e.label );
});
return labels;
};
Client.prototype.engineInfo = function(callback){
var self = this;
var labels = this.engineLabels();
var response = { monitor: {} };
var funclist = [];
// default engine is head of engineLabels
// default database is head of each database lists of engines
// we MUST keep orders of engineLabels and database lists
var dblists = {};
labels.forEach(function(label){
response.monitor[label] = self.engine(label).supports('status');
funclist.push(function(cb){
self.databases(label, function(err,dblist){
if (err) { cb(err); return; }
dblists[label] = dblist;
cb(null);
});
});
});
async.parallel(funclist, function(err, results){
if (err) { error_callback('engineInfo', self, callback, err); return; }
var pairs = [];
labels.forEach(function(label){
var part = dblists[label].map(function(dbname){ return [ label, dbname ]; });
pairs = pairs.concat(part);
});
response['pairs'] = pairs;
callback.apply(self, [err, response]);
});
};
Client.prototype.engine = function(label){
if (!label)
label = this._default_engine_label;
if (this._engines[label])
return this._engines[label];
var conf = this._engine_configs[label];
if (! conf)
return null; // unconfigured engine label
var executer_conf = conf.executer;
var monitor_conf = conf.monitor;
var options = {
query_timeout: this._default_query_timeout,
fetch_lines: this._default_fetch_lines,
setup_queries: this._default_setup_queries,
setup_queries_auth: this._default_setup_queries_auth,
auth_credential: this.auth_credential
};
if (executer_conf.fetch_lines)
options.fetch_lines = executer_conf.fetch_lines;
if (executer_conf.query_timeout)
options.query_timeout = executer_conf.query_timeout;
if (executer_conf.setup_queries)
options.setup_queries = executer_conf.setup_queries;
if (executer_conf.setup_queries_auth)
options.setup_queries_auth = executer_conf.setup_queries_auth;
if (! executer_conf.default_database)
executer_conf.default_database = this._default_default_database;
this._engines[label] = new engine.Engine(label, executer_conf, monitor_conf, options, this.logger);
return this._engines[label];
};
Client.prototype.end = function(half){
var client = this;
if (this._localstore) {
this._localstore.close();
this._localstore = undefined;
}
if (half)
return;
if (this._engines) {
for (var label in this._engines) {
this._engines[label].close();
}
this._engines = undefined;
}
};
Client.prototype.recentQueries = function(num, callback){
var client = this;
this.localStore().recentQueries(num, function(err, list){
if (err) { error_callback('recentQueries', client, callback, err); return; }
callback.apply(client, [err, list]);
});
};
Client.prototype.getQuery = function(queryid, callback){
var client = this;
this.localStore().query(queryid, function(err, data){
if (err) { error_callback('getQuery', client, callback, err, data); return; }
callback.apply(client, [err, data]);
});
};
Client.prototype.query = Client.prototype.getQuery;
Client.prototype.getQueries = function(queryids, callback){
var client = this;
this.localStore().queries(queryids, function(err, list){
if (err) { error_callback('getQueries', client, callback, err, list); return; }
callback.apply(client, [err, list]);
});
};
Client.prototype.queries = Client.prototype.getQueries;
Client.prototype.updateQuery = function(query, callback) {
var client = this;
this.localStore().updateQuery(query, function(err){
if (err) { error_callback('updateQuery', client, callback, err); return; }
if (callback)
callback.apply(client, [err]);
});
};
Client.prototype.createQuery = function(engineLabel, dbname, querystring, scheduled, userdata, callback){
var client = this;
var seed;
var query;
try {
var rand = Math.floor(Math.random() * 10000);
seed = (new Date()).toTimeString() + ";" + rand.toString(); // for queryid
var auth;
if(userdata) {
auth = true;
} else {
auth = false;
}
query = new Query({querystring:querystring, engine: engineLabel, dbname: dbname, scheduled: scheduled, seed: seed, auth: auth});
}
catch (e) {
error_callback('createQuery catch', client, callback, e);
return;
}
client.query(query.queryid, function(err, savedquery){
if (!err && savedquery) { callback.apply(client, [err, savedquery]); return; }
this.localStore().insertQuery(query, function(err){
if (err) { error_callback('createQuery', client, callback, err); return; }
callback.apply(client, [err, query]);
});
});
};
/*
* delete query, tag and resultData
*/
Client.prototype.deleteQuery = function(queryid, callback){
var client = this;
this.query(queryid, function(err, query){
if (err) { callback(err); return; }
if (query === null) { error_callback('deleteQuery', client, callback, new Error("queryid=" + queryid + " is not found in DB.")); return; }
var resultid = query.resultid;
async.series([
function(cb){ client.localStore().deleteQuery(queryid, function(err){ cb(err); }); },
function(cb){ client.localStore().deleteTagForQuery(queryid, function(err){ cb(err); }); },
function(cb){ client.localStore().deleteResultData(resultid, function(err){ cb(err); }); }
], function(err, results){ callback(err); });
});
};
Client.prototype.getQueryByResultId = function(resultid, callback){
var client = this;
this.localStore().queryByResultId(resultid, function(err, data){
if (err) { error_callback('getQuery', client, callback, err, data); return; }
callback.apply(client, [err, data]);
});
};
Client.prototype.tagList = function(callback){
var client = this;
this.localStore().tagList(function(err, tags){
if (err) { error_callback('tagList', client, callback, err); return; }
callback.apply(client, [err, tags]);
});
};
Client.prototype.tags = function(queryid, callback){
var client = this;
this.localStore().tags(queryid, function(err, tags){
if (err) { error_callback('tags', client, callback, err); return; }
callback.apply(client, [err, tags]);
});
};
Client.prototype.addTag = function(queryid, tag, callback){
var client = this;
this.localStore().addTag(queryid, tag, function(err){
if (err) { error_callback('addTag', client, callback, err); return; }
callback.apply(client, [err]);
});
};
Client.prototype.deleteTag = function(queryid, tag, callback){
var client = this;
this.localStore().deleteTag(queryid, tag, function(err){
if (err) { error_callback('deleteTag', client, callback, err); return; }
callback.apply(client, [err]);
});
};
Client.prototype.taggedQueries = function(tag, callback){
var client = this;
this.localStore().taggedQueries(tag, function(err, queryids){
if (err) { error_callback('taggedQueries', client, callback, err); return; }
callback.apply(client, [err, queryids]);
});
};
Client.prototype.getResultData = function(resultid, callback){
var client = this;
this.localStore().readResultData(resultid, function(err, data){
if (err) { error_callback('getResultData', client, callback, err, data); return; }
var list = [];
data.split("\n").forEach(function(line){
if (line == "")
return;
list.push(line.split("\t"));
});
callback.apply(client, [err, list]);
});
};
Client.prototype.resultData = Client.prototype.getResultData;
Client.prototype.getRawResultData = function(resultid, callback){
var client = this;
this.localStore().readResultData(resultid, function(err, data){
if (err) { error_callback('getRawResultData', client, callback, err, data); return; }
callback.apply(client, [err, data]);
});
};
Client.prototype.rawResultData = Client.prototype.getRawResultData;
Client.prototype.appendResultData = function(resultid, data, callback){
var client = this;
this.localStore().appendResultData(resultid, data.join("\n") + "\n", function(err){
if (err) { error_callback('appendResultData', client, callback, err); return; }
if (callback)
callback.apply(client, [err]);
});
};
Client.prototype.databases = function(engineLabel, callback){
var client = this;
var engine = client.engine(engineLabel);
if (! engine) {
callback.apply(client, [{message: "RELOAD page: unknown engine label, " + engineLabel}]);
return;
}
if (! engine.supports('databases')) {
callback.apply(client, [null, ['(default)']]);
return;
}
engine.databases(function(error, data){
callback.apply(client, [error, data]);
});
};
Client.prototype.tables = function(engineLabel, dbname, callback){
var client = this;
var engine = client.engine(engineLabel);
if (! engine) {
callback.apply(client, [{message: "RELOAD page: unknown engine label, " + engineLabel}]);
return;
}
engine.tables(dbname, function(error, data){
callback.apply(client, [error, data]);
});
};
/* get partitions of specified table */
Client.prototype.partitions = function(engineLabel, dbname, tablename, callback){
var client = this;
var engine = client.engine(engineLabel);
if (! engine) {
callback.apply(client, [{message: "RELOAD page: unknown engine label, " + engineLabel}]);
return;
}
engine.partitions(dbname, tablename, function(error, data){
// engine().partitions() returns
// [ 'part1=val1/part2=val2/part3=val3', .... ]
var partition_nodes = [];
var treenodes = {};
var create_node = function(partition, hasChildren){
if (treenodes[partition])
return treenodes[partition];
var parts = partition.split('/');
var leafName = parts.pop();
var node = {title: leafName};
if (hasChildren) {
node.children = [];
}
if (parts.length > 0) {
var parent = create_node(parts.join('/'), true);
parent.children.push(node);
}
else {
partition_nodes.push(node);
}
treenodes[partition] = node;
return node;
};
(data || []).forEach(function(partition){
create_node(partition);
});
callback.apply(client, [error, partition_nodes]);
});
};
/* get table schema info */
Client.prototype.describe = function(engineLabel, dbname, tablename, callback){
var client = this;
var engine = client.engine(engineLabel);
if (! engine) {
callback.apply(client, [{message: "RELOAD page: unknown engine label, " + engineLabel}]);
return;
}
engine.describe(dbname, tablename, function(error, data){
callback.apply(client, [error, data]);
});
};
Client.prototype.giveup = function(query, callback){
var client = this;
query.markAsExecuted({message: 'specified as "give up"'});
client.updateQuery(query);
if (callback)
callback.apply(client, [null, query]);
if (! query.engine){
client.end(); // self close after half close
return;
}
var engine = client.engine(query.engine);
if (! engine) {
callback.apply(client, [{message: "RELOAD page: unknown engine label, " + query.engine}]);
client.end(); // self close after half close
return;
}
engine.status(query.queryid, function(err, jobdata){
if (err){ // engine doesn't support 'status' or errors with other reason
// cannot kill with no status information
error_callback('giveup', client, callback, err);
client.end(); // self close after half close
return;
}
if (jobdata) {// killed job terminates hive query and executer calls callback with error "killed by user"
engine.kill(jobdata.jobid, function(err){
if (err) {
client.logger.error("Error on killing job", {jobid: jobdata.jobid, error: err});
}
client.end(); // self close after half close
});
}
});
};
Client.prototype.execute = function(query, args){
if (! args)
args = {};
var client = this;
var executed_time = new Date();
//query.state is already running, and resultid/result is set
if (args.prepare)
args.prepare(query);
var schemaRow = null;
var resultLines = 0;
var resultBytes = 0;
var onerror = null;
var engine = client.engine(query.engine);
if (! engine) {
query.markAsExecuted({message: "RELOAD page: unknown engine label, " + query.engine});
if (args.error)
args.error();
return;
}
engine.execute(query.queryid, query.dbname, query.composed(), query.auth, {
stopcheck: args.stopCheck,
stop: args.stop,
complete: function(err){
if (err){ onerror = err; }
query.markAsExecuted(onerror, resultLines, resultBytes);
client.updateQuery(query);
if (onerror && args.error)
args.error();
else if (args.success)
args.success();
},
error: function(err){
query.markAsExecuted(err);
client.updateQuery(query);
if (args.error)
args.error();
},
timeout: function(err){
var queryid = query.queryid;
query.markAsExecuted(err);
client.updateQuery(query, function(err){ client.end(true); }); // early half close to close database
// DO NOT operate database after here
if (engine.supports('status') && engine.supports('kill')) {
engine.status(queryid, function(err, jobdata){
if (err){
if (args.error)
args.error();
return;
}
if (jobdata) {// killed job terminates hive query and executer calls callback with error "killed by user"
engine.kill(jobdata.jobid, function(err){
if (err) { client.logger.error("Error on killing job", {jobid: jobdata.jobid, error: err}); }
if (args.error)
args.error();
});
}
});
}
else {
if (args.error)
args.error();
}
},
schema: function(error, data){
if (error){ onerror = error; return; }
query.addSchema(data);
// set to write to top of result data
schemaRow = data.map(function(f){return f.name.toUpperCase();}).join('\t');
},
fetch: function(error, rows, cb){
if (error){ onerror = error; cb(error); return; }
if (schemaRow) {
rows.unshift(schemaRow);
schemaRow = null;
}
client.appendResultData(query.resultid, rows, function(err){
if (err) {
client.logger.error('failed to append result data', err);
cb(err);
throw new LocalDiskStoreError("failed to append result data");
}
resultLines += rows.length;
resultBytes += rows.reduce(function(prev,v){return prev + v.length + 1;}, 0);
if (this._default_max_result_file_byte_size != null && resultBytes > this._default_max_result_file_byte_size) {
cb({message:"Result file size exceeds " + this._default_max_result_file_byte_size + " bytes."})
} else {
cb();
}
});
}
});
};
Client.prototype.detailStatus = function(query, callback){
var client = this;
var engine = client.engine(query.engine);
if (! engine) {
error_callback('detailStatus', client, callback, {message: "RELOAD page: unknown engine label, " + query.engine}); return;
return;
}
engine.status(query.queryid, function(err, status){
if (err) { error_callback('detailStatus', client, callback, err); return; }
callback.apply(client, [null, status]);
});
};
Client.prototype.generatePath = function(key){
return this.localStore().generatePath(key);
};
================================================
FILE: lib/shib/engine.js
================================================
var dummyengine = require('./engines/dummyengine')
, dummy = require('./engines/dummy')
, hiveserver = require('./engines/hiveserver')
, hiveserver2 = require('./engines/hiveserver2')
, presto = require('./engines/presto')
, jobtracker = require('./engines/jobtracker')
, yarn = require('./engines/yarn')
, huahin_mrv1 = require('./engines/huahin_mrv1')
, bigquery = require('./engines/bigquery');
var Query = require('./query').Query
, AccessControl = require('./access_control').AccessControl;
var Engine = exports.Engine = function(label, executer_conf, monitor_conf, options, logger) {
// query_timeout is for timeout between 'setup' and just after 'execute' callback call.
this.label = label;
this.logger = logger;
// this._executer = dummyengine.Executer();
this._query_timeout = options.query_timeout;
this._setup_queries = options.setup_queries;
this._setup_queries_auth = options.setup_queries_auth;
this._fetch_lines = options.fetch_lines;
this._auth_credential = options.auth_credential;
this._executer = null;
this._monitor = null;
if (! executer_conf || !executer_conf.name) {
throw "executer configuration or executer name not found";
}
var executer = null;
var monitor = null;
switch (executer_conf.name) {
case 'hiveserver':
executer = hiveserver.Executer;
break;
case 'hiveserver2':
executer = hiveserver2.Executer;
// monitor = hiveserver2.Monitor;
break;
case 'presto':
executer = presto.Executer;
monitor = presto.Monitor; // default monitor
break;
case 'bigquery':
executer = bigquery.Executer;
monitor = bigquery.Monitor; // default monitor
break;
case 'dummy':
executer = dummy.Executer;
monitor = dummy.Monitor;
break;
default:
throw "unknown executer name:" + executer_conf.name;
}
if (! executer_conf.default_database)
throw "BUG: default_database missing in executer_conf";
this._default_dbname = executer_conf.default_database;
this._executer = new executer(executer_conf, this.logger);
this._acl_config = executer_conf.access_control || {};
if (monitor_conf) {
if (! monitor_conf.name)
throw "monitor name not found";
switch (monitor_conf.name) {
case 'jobtracker':
monitor = jobtracker.Monitor;
break;
case 'yarn':
monitor = yarn.Monitor;
break;
case 'huahin_mrv1':
monitor = huahin_mrv1.Monitor;
break;
case 'presto':
monitor = presto.Monitor;
break;
case 'bigquery':
monitor = bigquery.Monitor;
break;
case 'dummy':
monitor = dummy.Monitor;
break;
default:
throw "unknown monitor name:" + monitor_conf.name;
}
this._monitor = new monitor(monitor_conf, this.logger);
} else if (monitor) {
// default monitor same with executor
this._monitor = new monitor(executer_conf, this.logger);
} else {
this._monitor = new dummyengine.Monitor();
}
};
Engine.prototype.supports = function(operation) {
switch (operation) {
case 'jobname':
case 'setup':
case 'databases':
case 'tables':
case 'partitions':
case 'describe':
case 'execute':
return this._executer.supports(operation); // executer MUST support setup() and execute()
case 'status':
case 'kill':
return this._monitor.supports(operation); // monitor CAN support status() and kill()
}
throw "invalid operation for engines:" + operation;
};
Engine.prototype.acl = function(){
if (this._acl_config['delegate'] === 'auth') {
if (! this._auth_credential) {
this.logger.warn("auth credential not found: default denied for all requests");
return AccessControl.defaultDenyDelegator();
}
return this._auth_credential;
}
return new AccessControl(this._acl_config);
};
Engine.prototype.close = function(){
this._executer.end();
this._executer = undefined;
this._monitor.end();
this._monitor = undefined;
};
Engine.prototype.default_database_name = function(){
if (this.supports('database'))
return this._default_dbname;
return null;
};
Engine.prototype.databases = function(callback){
var self = this;
if (this.supports('databases')) {
this._executer.databases(function(err, dbnamelist){
if (err) { self.logger.error('databases', err); callback(err); return; }
var dbs = [self._default_dbname];
dbnamelist.forEach(function(dbname){
if (dbname === self._default_dbname)
return;
if (self.acl().visible(dbname))
dbs.push(dbname);
});
callback(null, dbs);
});
} else {
callback({message:'Failed to get database list, not supported'});
}
};
Engine.prototype.tables = function(dbname, callback){
var self = this;
if (this.supports('tables')) {
if (! this.acl().visible(dbname)) {
callback(null, []);
return;
}
this._executer.tables(dbname, function(err, data){
var tables = [];
data.forEach(function(tablename){
if (self.acl().allowed(tablename, dbname))
tables.push(tablename);
});
callback(null, tables);
});
} else {
callback({message:'Failed to get table list, not supported'});
}
};
Engine.prototype.partitions = function(dbname, tablename, callback){
if (this.supports('partitions')) {
if (! this.acl().allowed(tablename, dbname)) {
callback({message:'access denied'});
return;
}
this._executer.partitions(dbname, tablename, callback);
} else {
callback({message:'Failed to get partition list, not supported'});
}
};
Engine.prototype.describe = function(dbname, tablename, callback){
if (this.supports('describe')) {
if (! this.acl().allowed(tablename, dbname)) {
callback({message:'access denied'});
return;
}
this._executer.describe(dbname, tablename, callback);
} else {
callback({message:'Failed to get table schema, not supported'});
}
};
/*
options: {
schema: function(error, data){},
callback: function(error, rows){}, // default callback (exclusive with error/success/fetch/complete)
stopcheck: function(query){},
stop: function(){},
error: function(error){},
fetch: function(error, rows){},
complete: function(error){},
success: function(){} // without fetchNum, fetchAll() -> success_callback(result_rows) (or callback(null,result_rows) )
}
*/
Engine.prototype.execute = function(queryid, dbname, query, auth, options) {
var self = this;
var executer = this._executer;
if (options.call) { // 3rd argument is single callback function
var callback = options;
options = {callback: callback};
}
var schema_callback = options.schema || function(err,data){};
var stopcheck = options.stopcheck || function(){return false;};
var success_callback = options.success || options.callback || function(err,data){};
var stop_callback = options.stop || options.callback || function(err,data){};
var error_callback = options.error || options.callback || function(err){};
// query_timeout is for timeout between 'setup' and just after 'execute' callback call.
var timeout_callback = options.timeout || error_callback || function(err){};
var setups = this._setup_queries || [];
if(auth) {
setups = setups.concat(this._setup_queries_auth)
}
var fetchnum = this._fetch_lines;
var fetch_callback = null;
var complete_callback = null;
if (fetchnum) {
if (!options.fetch || !options.complete)
throw "missing fetch or complete callback for fetch api";
fetch_callback = options.fetch;
complete_callback = options.complete;
}
var timeout_watch = null;
var disable_timeout_handler = function(){
if(timeout_watch && timeout_watch.state) {
timeout_watch.state = false;
clearTimeout(timeout_watch.timer);
}
};
if (this._query_timeout) {
var timeout_seconds = this._query_timeout * 1000;
var timer = setTimeout(function(){
if (timeout_watch && timeout_watch.state) {
timeout_watch.expired = true;
timeout_callback({message: 'query is expired with configured timeout seconds (' + String(timeout_seconds) + ')'});
}
}, timeout_seconds);
var state = true; // not expired
timeout_watch = { timer:timer, state:state, expired:false };
}
var table_db_pairs = Query.parseTableNames(query);
var access_denied = false;
table_db_pairs.forEach(function(pair){
if (! self.acl().allowed(pair[0], (pair[1] || dbname)))
access_denied = true;
});
if (access_denied) {
disable_timeout_handler();
error_callback({message: "access denied for db/table"});
return;
}
executer.setup(setups, function(err){
var jobname = executer.jobname(queryid);
executer.execute(jobname, dbname, query, function(err, fetcher){
if (timeout_watch && timeout_watch.expired) {
// this query has expired, and timeout_callback() has already been executed
return;
}
if (stopcheck()) { disable_timeout_handler(); stop_callback({message: "stopped by stopcheck"}); return; }
if (err) { disable_timeout_handler(); error_callback(err); return; }
fetcher.schema(function(err, data){
if (err) { disable_timeout_handler(); error_callback(err); return; }
if (timeout_watch && timeout_watch.expired) {
// this query has expired, and timeout_callback() has already been executed
return;
}
disable_timeout_handler();
if (schema_callback)
schema_callback(err, data);
// fetch all result rows at once
if (! fetchnum) {
fetcher.fetch(null, function(err, rows){ // fetch all records
if (err) { error_callback(err); return; }
success_callback(null, rows);
});
return;
}
var has_errors = false;
// fetch N-rows step by step
var continuous_fetch = function(err){
if (err) {
has_errors = true;
self.logger.warn("fetch killed with upper layer error", {err: err});
complete_callback({message:"Fetching exits with errors. " + err.message});
return;
}
fetcher.fetch(fetchnum, function(err, rows){
if (err) {
has_errors = true;
self.logger.warn("fetch error", {err: err});
}
if (stopcheck()) { stop_callback({message: "stopped by stopcheck"}); return; }
if (rows == null || rows.length < 1 || (rows.length == 1 && rows[0].length < 1)) {
// end of fetched rows
if (has_errors)
complete_callback({message:"HiveQL exits with errors"});
else
complete_callback(null);
return;
}
fetch_callback(err, rows, continuous_fetch);
});
};
continuous_fetch(null);
});
});
});
};
Engine.prototype.status = function(queryid, callback) {
var jobname = this._executer.jobname(queryid);
if (this.supports('status')) {
this._monitor.status(jobname, callback);
} else {
callback({message:'Failed to get job(' + jobname + ') status, not supported'});
}
};
Engine.prototype.kill = function(jobid, callback) {
if (this.supports('kill')) {
this._monitor.kill(jobid, callback);
} else {
callback({message:'Failed to kill job(jobid:' + jobid + '), not supported'});
}
};
================================================
FILE: lib/shib/engines/bigquery/index.js
================================================
var gcloud = require('gcloud')
, util = require('util');
// check interval for real queries, not system queries
var BLOCK_CHECK_INTERVAL = 1000; // 1sec
var MAX_RESULTS_FOR_EACH_FETCH = 1000;
var jobname_jobid_map = {};
var Executer = exports.Executer = function(conf, logger){
if (conf.name !== 'bigquery')
throw "executer name mismatch for bigquery:" + conf.name;
if (!conf.project_id)
throw "project_id MUST be specified for bigquery executer";
if (!conf.key_filename)
throw "key_filename MUST be specified for bigquery executer";
this.logger = logger;
this._client = gcloud.bigquery({
projectId: conf.project_id,
keyFilename: conf.key_filename
});
};
Executer.prototype.end = function(){
// Nothing to do for HTTP API :-)
};
Executer.prototype.supports = function(operation){
switch (operation) { // "executer" methods
case 'jobname':
case 'setup':
case 'databases':
case 'tables':
case 'describe':
case 'execute':
return true;
}
throw "unknown operation name (for bigquery):" + operation;
};
Executer.prototype.jobname = function(queryid) {
return 'shib-bigquery-' + queryid;
};
Executer.prototype.setup = function(setups, callback){
callback(null);
};
Executer.prototype.databases = function(callback){
var results = []
, pageToken = null
, self = this;
var processDatasets = function(datasets) {
datasets.forEach(function(row) {
var dbname = row.metadata.datasetReference.datasetId;
results.push(dbname);
});
};
var fetchDatesets = function() {
self._client.getDatasets({ maxResults: MAX_RESULTS_FOR_EACH_FETCH, pageToken: pageToken }, function(err, datasets, nextQuery){
if (err) { callback(err); return; }
processDatasets(datasets);
if (nextQuery) {
pageToken = nextQuery.pageToken;
setTimeout(fetchDatesets, 0);
} else {
callback(null, results);
}
});
};
fetchDatesets();
};
Executer.prototype.tables = function(dbname, callback){
var results = []
, pageToken = null
, self = this
, dataset = self._client.dataset(dbname);
var processTables = function(tables) {
tables.forEach(function(row) {
var table = row.metadata.tableReference.tableId;
results.push(table);
});
};
var fetchTables = function() {
dataset.getTables({ maxResults: MAX_RESULTS_FOR_EACH_FETCH, pageToken: pageToken }, function(err, tables, nextQuery){
if (err) { callback(err); return; }
processTables(tables);
if (nextQuery) {
pageToken = nextQuery.pageToken;
setTimeout(fetchTables, 0);
} else {
callback(null, results);
}
});
};
fetchTables();
};
Executer.prototype.partitions = function(dbname, tablename, callback){
// bigquery engine does not support 'partitions'
callback(null, []);
};
Executer.prototype.describe = function(dbname, tablename, callback){
var dataset = this._client.dataset(dbname);
var table = dataset.table(tablename);
table.getMetadata(function(err, metadata){
if (err) { callback(err); return; }
var results = [];
var fields = metadata.schema.fields;
fields.forEach(function(field){
var name = field.name;
var type = field.type;
var comment = field.description ? field.description : '';
results.push([name, type, comment]);
});
callback(null, results);
});
};
Executer.prototype.execute = function(jobname, dbname, query, callback){
var client = this._client;
var fetcher = new Fetcher(client, jobname);
var error_callback = function(e){
delete jobname_jobid_map[jobname];
if (!fetcher._rpcError) // only first error is stored
fetcher._rpcError = e;
};
client.startQuery(query, function(err, job) {
if (err) { error_callback(err); return; }
jobname_jobid_map[jobname] = job.id;
});
callback(null, fetcher);
};
var Fetcher = function(client, jobname){
this._client = client;
this._jobname = jobname;
this._hasResults = false;
this._noMoreResults = false;
this._rpcError = null;
this._nextQuery = null;
this._cache = { data: [], schema: null };
this._processColumn = function(data){
var columns = [];
if (data.length > 0) {
var first = data[0];
Object.keys(first).forEach(function(key){
columns.push({ name: key });
});
}
this._cache.schema = columns;
this._hasResults = true;
};
this._processData = function(data){
var buf = []
, len = data.length
, schema = this._cache.schema
, schemaLen = schema.length;
for ( var i = 0 ; i < len ; i++ ) {
var row = [];
for (var j = 0 ; j < schemaLen; j++ ) {
row.push(data[i][schema[j].name]);
}
buf.push( row.join('\t') );
}
this._push( buf );
};
this._push = function(data) {
this._cache.data = this._cache.data.concat(data);
};
this._waitComplete = function(callback) {
if (this._hasResults || this._rpcError) {
if (this._rpcError)
callback(this._rpcError);
else
callback(null);
return;
}
var self = this;
var check = function() {
var jobId = jobname_jobid_map[self._jobname];
if (jobId) {
var job = self._client.job(jobId);
job.getQueryResults({
maxResults: MAX_RESULTS_FOR_EACH_FETCH,
pageToken: self._nextQuery ? self._nextQuery.pageToken : null
}, function (err, rows, nextQuery) {
if (err) { callback(err); return; }
if (!self._hasResults) {
self._processColumn(rows);
}
self._processData(rows);
if (!nextQuery) {
self._nextQuery = null;
self._noMoreResults = true;
delete jobname_jobid_map[self._jobname];
return callback(null);
}
self._nextQuery = nextQuery;
setTimeout(check, BLOCK_CHECK_INTERVAL);
});
} else {
setTimeout(check, BLOCK_CHECK_INTERVAL);
}
};
check();
};
this.schema = function(callback){
/*
* schema(callback): callback(err, schema)
* schema: {fieldSchemas: [{name:'fieldname1'}, {name:'fieldname2'}, {name:'fieldname3'}, ...]}
* //?? schema: [{name:'fieldname1'}, {name:'fieldname2'}, ...]
*/
var self = this;
this._waitComplete(function(err){
if (err) { callback(err); return; }
// self._cache.schema: [ { name: "username", type: "varchar" }, { name: "cnt", type: "bigint" } ]
callback(null, self._cache.schema);
});
};
this.fetch = function(num, callback){
if (!num) {
this._fetchAll(callback);
return;
}
var self = this;
if (self._cache.data.length < 1 && self._noMoreResults) {
// if (rows === null || rows.length < 1 || (rows.length == 1 && rows[0].length < 1)) {
// end of fetched rows
callback(null, null);
return;
}
var buf = [];
var fill = function() {
var chunk = self._cache.data.splice(0, num - buf.length);
if (chunk.length < 1) {
if (self._noMoreResults) {
var tmpbuf = buf;
buf = [];
callback(null, tmpbuf); // if tmpbuf is empty, this is end of fetching
return;
}
else {
setTimeout(fill, BLOCK_CHECK_INTERVAL);
return;
}
}
buf = buf.concat(chunk);
if (buf.length >= num || self._noMoreResults) {
var fullchunk = buf;
buf = [];
callback(null, fullchunk);
}
else
setTimeout(fill, BLOCK_CHECK_INTERVAL);
};
this._waitComplete(function(err){
if (err) { callback(err); return; }
fill();
});
};
this._fetchAll = function(callback) {
var self = this;
var check = function() {
if (self._rpcError)
callback(self._rpcError);
else if (self._noMoreResults)
callback(null, self._cache.data);
else
setTimeout(check, BLOCK_CHECK_INTERVAL);
};
check();
};
};
var Monitor = exports.Monitor = function(conf){
if (conf.name !== 'bigquery')
throw "executer name mismatch for bigquery:" + conf.name;
if (!conf.project_id)
throw "project_id MUST be specified for bigquery executer";
if (!conf.key_filename)
throw "key_filename MUST be specified for bigquery executer";
this._client = gcloud.bigquery({
projectId: conf.project_id,
keyFilename: conf.key_filename
});
};
Monitor.prototype.end = function(){
};
Monitor.prototype.supports = function(operation){
switch (operation) { // "monitor" methods
case 'status':
case 'kill':
return true;
}
throw "unknown operation name (for bigquery.Monitor):" + operation;
};
function convertStatus(jobname, status) {
if (status === undefined) {
return null;
}
var retval = {};
// https://cloud.google.com/bigquery/docs/reference/v2/jobs
retval['jobid'] = status['jobReference']['jobId'];
retval['name'] = jobname;
retval['priority'] = status['configuration']['priority'] || 'INTERACTIVE';
retval['state'] = status['status']['state'];
retval['trackingURL'] = status['selfLink'];
retval['startTime'] = new Date(parseInt(status['statistics']['creationTime']));
retval['complete'] = (retval['state'] === 'DONE' ? 100 : 0);
return retval;
}
Monitor.prototype.status = function(jobname, callback){
var jobId = jobname_jobid_map[jobname];
if (!jobId) {
callback({message:"job already expired (maybe completed)"}, null);
return;
}
var job = this._client.job(jobId);
job.getMetadata(function(err, data){
if (err) { callback(err); return; }
callback(null, convertStatus(jobname, data));
});
};
Monitor.prototype.kill = function(query_id, callback){
callback(null);
};
================================================
FILE: lib/shib/engines/dummy/index.js
================================================
/*
* This engine is for development on local environment
*/
var Executer = exports.Executer = function(conf, logger){
if (conf.name !== 'dummy')
throw "executer name mismatch for dummy:" + conf.name;
this.logger = logger;
};
Executer.prototype.end = function(){
};
Executer.prototype.supports = function(operation){
switch (operation) { // "executer" methods
case 'jobname':
case 'setup':
case 'databases':
case 'tables':
case 'partitions':
case 'describe':
case 'execute':
return true;
}
throw "unknown operation name (for dummy):" + operation;
};
Executer.prototype.jobname = function(queryid) {
return 'dummy-' + queryid;
};
Executer.prototype.setup = function(setups, callback){
callback(null);
};
Executer.prototype.databases = function(callback){
callback(null, [ ['default'], ['dummy1'] ]);
};
Executer.prototype.tables = function(dbname, callback){
callback(null, [ ['t1'], ['t2'] ]);
};
Executer.prototype.partitions = function(dbname, tablename, callback){
callback(null, ['f1=1/f2=1', 'f1=1/f2=2', 'f1=2/f2=1', 'f1=2/f2=2']);
};
Executer.prototype.describe = function(dbname, tablename, callback){
callback(null, [ ['f1', 'string', ''], ['f2', 'string', ''], ['id', 'bigint', ''], ['json', 'string', ''] ]);
};
Executer.prototype.execute = function(jobname, dbname, query, callback){
callback(null, new Fetcher(this));
};
/*
* Fetcher
*
* schema(callback): callback(err, schema)
* schema: [ { name: 'fieldname1', type: 'typename' }, ...]
*
* fetch(num, callback): callback(err, rows)
* num: rows to fetch (null == all)
* rows: ["_col1value_\t_col2value_\t_col3value_", "_col1value_\t_col2value_\t_col3value_", ...]
* no more rows exists if (rows === null || rows.length < 1 || (rows.length == 1 && rows[0].length < 1))
*/
var Fetcher = function(client){
this.schema = function(callback){
callback(null, [ {name:"f1", type:"string"}, {name:"f2", type:"string"}, {name:"id", type:"bigint"}, {name:"json", type:"string"} ]);
};
var finished = false;
this.fetch = function(num, callback){
// always returns only 4 row after 10 seconds
if (finished) {
callback(null, []);
return;
}
setTimeout(function(){
callback(null, ["1\t1\t1000\t{}", "1\t2\t1001\t{}", "2\t1\t1002\t{}", "2\t2\t1003\t{}"]);
finished = true;
}, 10000);
};
};
var Monitor = exports.Monitor = function(conf){
if (conf.name !== 'dummy')
throw "executer name mismatch for dummy:" + conf.name;
};
Monitor.prototype.end = function(){
};
Monitor.prototype.supports = function(operation){
switch (operation) { // "monitor" methods
case 'status':
case 'kill':
return true;
}
throw "unknown operation name (for dummy.Monitor):" + operation;
};
Monitor.prototype.status = function(jobname, callback){
callback(null, {
jobid: "dummy-id-000001",
name: jobname,
priority: "unknown",
state: "RUNNING",
trackingURL: "http://localhost/dummy/",
startTime: new Date().toLocaleString(),
mapComplete: null,
reduceComplete: null
});
};
Monitor.prototype.kill = function(query_id, callback){
callback(null);
};
================================================
FILE: lib/shib/engines/dummyengine.js
================================================
var Executer = exports.Executer = function(){
this.supports = function(operation){ return true; };
};
Executer.prototype.jobname = function(queryid) {
return 'dummy-shib-' + queryid;
};
Executer.prototype.setup = function(setups, callback){
callback(null);
};
Executer.prototype.databases = function(callback){
callback(null, []); // ['db1', 'db2']
};
Executer.prototype.tables = function(dbname, callback){
callback(null, []); // ['table1', 'table2']
};
Executer.prototype.partitions = function(dbname, tablename, callback){
callback(null, []); // ['f1=va1/f2=vb1', 'f1=va1/f2=vb2']
};
Executer.prototype.describe = function(dbname, tablename, callback){
callback(null, []); // [ [ 'fieldname', 'type', 'comment' ], ... ]
};
Executer.prototype.execute = function(jobname, dbname, query, callback){
callback(null, new Fetcher());
};
Executer.prototype.end = function(){};
var Fetcher = function(){
this.schema = function(callback){ callback(null, []); };
this.fetch = function(num, callback){ callback(null, []); };
};
/*
* Fetcher
*
* schema(callback): callback(err, schema)
* schema: [ { name: 'fieldname1', type: 'typename' }, ...]
*
* fetch(num, callback): callback(err, rows)
* num: rows to fetch (null == all)
* rows: ["_col1value_\t_col2value_\t_col3value_", "_col1value_\t_col2value_\t_col3value_", ...]
* no more rows exists if (rows === null || rows.length < 1 || (rows.length == 1 && rows[0].length < 1))
*/
var Monitor = exports.Monitor = function(){
this.supports = function(operation){ return false; };
};
Monitor.prototype.status = function(jobname, callback){
callback(null, {});
};
Monitor.prototype.kill = function(jobid, callback){
callback(null);
};
Monitor.prototype.end = function(){};
================================================
FILE: lib/shib/engines/hiveserver/FacebookService.js
================================================
//
// Autogenerated by Thrift Compiler (0.8.0)
//
// DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
//
var Thrift = require('node-thrift').Thrift;
var ttypes = require('./fb303_types');
//HELPER FUNCTIONS AND STRUCTURES
var FacebookService_getName_args = function(args) {
};
FacebookService_getName_args.prototype = {};
FacebookService_getName_args.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
input.skip(ftype);
input.readFieldEnd();
}
input.readStructEnd();
return;
};
FacebookService_getName_args.prototype.write = function(output) {
output.writeStructBegin('FacebookService_getName_args');
output.writeFieldStop();
output.writeStructEnd();
return;
};
var FacebookService_getName_result = function(args) {
this.success = null;
if (args) {
if (args.success !== undefined) {
this.success = args.success;
}
}
};
FacebookService_getName_result.prototype = {};
FacebookService_getName_result.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
switch (fid)
{
case 0:
if (ftype == Thrift.Type.STRING) {
this.success = input.readString();
} else {
input.skip(ftype);
}
break;
case 0:
input.skip(ftype);
break;
default:
input.skip(ftype);
}
input.readFieldEnd();
}
input.readStructEnd();
return;
};
FacebookService_getName_result.prototype.write = function(output) {
output.writeStructBegin('FacebookService_getName_result');
if (this.success) {
output.writeFieldBegin('success', Thrift.Type.STRING, 0);
output.writeString(this.success);
output.writeFieldEnd();
}
output.writeFieldStop();
output.writeStructEnd();
return;
};
var FacebookService_getVersion_args = function(args) {
};
FacebookService_getVersion_args.prototype = {};
FacebookService_getVersion_args.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
input.skip(ftype);
input.readFieldEnd();
}
input.readStructEnd();
return;
};
FacebookService_getVersion_args.prototype.write = function(output) {
output.writeStructBegin('FacebookService_getVersion_args');
output.writeFieldStop();
output.writeStructEnd();
return;
};
var FacebookService_getVersion_result = function(args) {
this.success = null;
if (args) {
if (args.success !== undefined) {
this.success = args.success;
}
}
};
FacebookService_getVersion_result.prototype = {};
FacebookService_getVersion_result.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
switch (fid)
{
case 0:
if (ftype == Thrift.Type.STRING) {
this.success = input.readString();
} else {
input.skip(ftype);
}
break;
case 0:
input.skip(ftype);
break;
default:
input.skip(ftype);
}
input.readFieldEnd();
}
input.readStructEnd();
return;
};
FacebookService_getVersion_result.prototype.write = function(output) {
output.writeStructBegin('FacebookService_getVersion_result');
if (this.success) {
output.writeFieldBegin('success', Thrift.Type.STRING, 0);
output.writeString(this.success);
output.writeFieldEnd();
}
output.writeFieldStop();
output.writeStructEnd();
return;
};
var FacebookService_getStatus_args = function(args) {
};
FacebookService_getStatus_args.prototype = {};
FacebookService_getStatus_args.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
input.skip(ftype);
input.readFieldEnd();
}
input.readStructEnd();
return;
};
FacebookService_getStatus_args.prototype.write = function(output) {
output.writeStructBegin('FacebookService_getStatus_args');
output.writeFieldStop();
output.writeStructEnd();
return;
};
var FacebookService_getStatus_result = function(args) {
this.success = null;
if (args) {
if (args.success !== undefined) {
this.success = args.success;
}
}
};
FacebookService_getStatus_result.prototype = {};
FacebookService_getStatus_result.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
switch (fid)
{
case 0:
if (ftype == Thrift.Type.I32) {
this.success = input.readI32();
} else {
input.skip(ftype);
}
break;
case 0:
input.skip(ftype);
break;
default:
input.skip(ftype);
}
input.readFieldEnd();
}
input.readStructEnd();
return;
};
FacebookService_getStatus_result.prototype.write = function(output) {
output.writeStructBegin('FacebookService_getStatus_result');
if (this.success) {
output.writeFieldBegin('success', Thrift.Type.I32, 0);
output.writeI32(this.success);
output.writeFieldEnd();
}
output.writeFieldStop();
output.writeStructEnd();
return;
};
var FacebookService_getStatusDetails_args = function(args) {
};
FacebookService_getStatusDetails_args.prototype = {};
FacebookService_getStatusDetails_args.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
input.skip(ftype);
input.readFieldEnd();
}
input.readStructEnd();
return;
};
FacebookService_getStatusDetails_args.prototype.write = function(output) {
output.writeStructBegin('FacebookService_getStatusDetails_args');
output.writeFieldStop();
output.writeStructEnd();
return;
};
var FacebookService_getStatusDetails_result = function(args) {
this.success = null;
if (args) {
if (args.success !== undefined) {
this.success = args.success;
}
}
};
FacebookService_getStatusDetails_result.prototype = {};
FacebookService_getStatusDetails_result.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
switch (fid)
{
case 0:
if (ftype == Thrift.Type.STRING) {
this.success = input.readString();
} else {
input.skip(ftype);
}
break;
case 0:
input.skip(ftype);
break;
default:
input.skip(ftype);
}
input.readFieldEnd();
}
input.readStructEnd();
return;
};
FacebookService_getStatusDetails_result.prototype.write = function(output) {
output.writeStructBegin('FacebookService_getStatusDetails_result');
if (this.success) {
output.writeFieldBegin('success', Thrift.Type.STRING, 0);
output.writeString(this.success);
output.writeFieldEnd();
}
output.writeFieldStop();
output.writeStructEnd();
return;
};
var FacebookService_getCounters_args = function(args) {
};
FacebookService_getCounters_args.prototype = {};
FacebookService_getCounters_args.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
input.skip(ftype);
input.readFieldEnd();
}
input.readStructEnd();
return;
};
FacebookService_getCounters_args.prototype.write = function(output) {
output.writeStructBegin('FacebookService_getCounters_args');
output.writeFieldStop();
output.writeStructEnd();
return;
};
var FacebookService_getCounters_result = function(args) {
this.success = null;
if (args) {
if (args.success !== undefined) {
this.success = args.success;
}
}
};
FacebookService_getCounters_result.prototype = {};
FacebookService_getCounters_result.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
switch (fid)
{
case 0:
if (ftype == Thrift.Type.MAP) {
var _size0 = 0;
var _rtmp34;
this.success = {};
var _ktype1 = 0;
var _vtype2 = 0;
_rtmp34 = input.readMapBegin();
_ktype1 = _rtmp34.ktype;
_vtype2 = _rtmp34.vtype;
_size0 = _rtmp34.size;
for (var _i5 = 0; _i5 < _size0; ++_i5)
{
if (_i5 > 0 ) {
if (input.rstack.length > input.rpos[input.rpos.length -1] + 1) {
input.rstack.pop();
}
}
var key6 = null;
var val7 = null;
key6 = input.readString();
val7 = input.readI64();
this.success[key6] = val7;
}
input.readMapEnd();
} else {
input.skip(ftype);
}
break;
case 0:
input.skip(ftype);
break;
default:
input.skip(ftype);
}
input.readFieldEnd();
}
input.readStructEnd();
return;
};
FacebookService_getCounters_result.prototype.write = function(output) {
output.writeStructBegin('FacebookService_getCounters_result');
if (this.success) {
output.writeFieldBegin('success', Thrift.Type.MAP, 0);
output.writeMapBegin(Thrift.Type.STRING, Thrift.Type.I64, Thrift.objectLength(this.success));
for (var kiter8 in this.success)
{
if (this.success.hasOwnProperty(kiter8))
{
var viter9 = this.success[kiter8];
output.writeString(kiter8);
output.writeI64(viter9);
}
}
output.writeMapEnd();
output.writeFieldEnd();
}
output.writeFieldStop();
output.writeStructEnd();
return;
};
var FacebookService_getCounter_args = function(args) {
this.key = null;
if (args) {
if (args.key !== undefined) {
this.key = args.key;
}
}
};
FacebookService_getCounter_args.prototype = {};
FacebookService_getCounter_args.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
switch (fid)
{
case 1:
if (ftype == Thrift.Type.STRING) {
this.key = input.readString();
} else {
input.skip(ftype);
}
break;
case 0:
input.skip(ftype);
break;
default:
input.skip(ftype);
}
input.readFieldEnd();
}
input.readStructEnd();
return;
};
FacebookService_getCounter_args.prototype.write = function(output) {
output.writeStructBegin('FacebookService_getCounter_args');
if (this.key) {
output.writeFieldBegin('key', Thrift.Type.STRING, 1);
output.writeString(this.key);
output.writeFieldEnd();
}
output.writeFieldStop();
output.writeStructEnd();
return;
};
var FacebookService_getCounter_result = function(args) {
this.success = null;
if (args) {
if (args.success !== undefined) {
this.success = args.success;
}
}
};
FacebookService_getCounter_result.prototype = {};
FacebookService_getCounter_result.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
switch (fid)
{
case 0:
if (ftype == Thrift.Type.I64) {
this.success = input.readI64();
} else {
input.skip(ftype);
}
break;
case 0:
input.skip(ftype);
break;
default:
input.skip(ftype);
}
input.readFieldEnd();
}
input.readStructEnd();
return;
};
FacebookService_getCounter_result.prototype.write = function(output) {
output.writeStructBegin('FacebookService_getCounter_result');
if (this.success) {
output.writeFieldBegin('success', Thrift.Type.I64, 0);
output.writeI64(this.success);
output.writeFieldEnd();
}
output.writeFieldStop();
output.writeStructEnd();
return;
};
var FacebookService_setOption_args = function(args) {
this.key = null;
this.value = null;
if (args) {
if (args.key !== undefined) {
this.key = args.key;
}
if (args.value !== undefined) {
this.value = args.value;
}
}
};
FacebookService_setOption_args.prototype = {};
FacebookService_setOption_args.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
switch (fid)
{
case 1:
if (ftype == Thrift.Type.STRING) {
this.key = input.readString();
} else {
input.skip(ftype);
}
break;
case 2:
if (ftype == Thrift.Type.STRING) {
this.value = input.readString();
} else {
input.skip(ftype);
}
break;
default:
input.skip(ftype);
}
input.readFieldEnd();
}
input.readStructEnd();
return;
};
FacebookService_setOption_args.prototype.write = function(output) {
output.writeStructBegin('FacebookService_setOption_args');
if (this.key) {
output.writeFieldBegin('key', Thrift.Type.STRING, 1);
output.writeString(this.key);
output.writeFieldEnd();
}
if (this.value) {
output.writeFieldBegin('value', Thrift.Type.STRING, 2);
output.writeString(this.value);
output.writeFieldEnd();
}
output.writeFieldStop();
output.writeStructEnd();
return;
};
var FacebookService_setOption_result = function(args) {
};
FacebookService_setOption_result.prototype = {};
FacebookService_setOption_result.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
input.skip(ftype);
input.readFieldEnd();
}
input.readStructEnd();
return;
};
FacebookService_setOption_result.prototype.write = function(output) {
output.writeStructBegin('FacebookService_setOption_result');
output.writeFieldStop();
output.writeStructEnd();
return;
};
var FacebookService_getOption_args = function(args) {
this.key = null;
if (args) {
if (args.key !== undefined) {
this.key = args.key;
}
}
};
FacebookService_getOption_args.prototype = {};
FacebookService_getOption_args.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
switch (fid)
{
case 1:
if (ftype == Thrift.Type.STRING) {
this.key = input.readString();
} else {
input.skip(ftype);
}
break;
case 0:
input.skip(ftype);
break;
default:
input.skip(ftype);
}
input.readFieldEnd();
}
input.readStructEnd();
return;
};
FacebookService_getOption_args.prototype.write = function(output) {
output.writeStructBegin('FacebookService_getOption_args');
if (this.key) {
output.writeFieldBegin('key', Thrift.Type.STRING, 1);
output.writeString(this.key);
output.writeFieldEnd();
}
output.writeFieldStop();
output.writeStructEnd();
return;
};
var FacebookService_getOption_result = function(args) {
this.success = null;
if (args) {
if (args.success !== undefined) {
this.success = args.success;
}
}
};
FacebookService_getOption_result.prototype = {};
FacebookService_getOption_result.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
switch (fid)
{
case 0:
if (ftype == Thrift.Type.STRING) {
this.success = input.readString();
} else {
input.skip(ftype);
}
break;
case 0:
input.skip(ftype);
break;
default:
input.skip(ftype);
}
input.readFieldEnd();
}
input.readStructEnd();
return;
};
FacebookService_getOption_result.prototype.write = function(output) {
output.writeStructBegin('FacebookService_getOption_result');
if (this.success) {
output.writeFieldBegin('success', Thrift.Type.STRING, 0);
output.writeString(this.success);
output.writeFieldEnd();
}
output.writeFieldStop();
output.writeStructEnd();
return;
};
var FacebookService_getOptions_args = function(args) {
};
FacebookService_getOptions_args.prototype = {};
FacebookService_getOptions_args.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
input.skip(ftype);
input.readFieldEnd();
}
input.readStructEnd();
return;
};
FacebookService_getOptions_args.prototype.write = function(output) {
output.writeStructBegin('FacebookService_getOptions_args');
output.writeFieldStop();
output.writeStructEnd();
return;
};
var FacebookService_getOptions_result = function(args) {
this.success = null;
if (args) {
if (args.success !== undefined) {
this.success = args.success;
}
}
};
FacebookService_getOptions_result.prototype = {};
FacebookService_getOptions_result.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
switch (fid)
{
case 0:
if (ftype == Thrift.Type.MAP) {
var _size10 = 0;
var _rtmp314;
this.success = {};
var _ktype11 = 0;
var _vtype12 = 0;
_rtmp314 = input.readMapBegin();
_ktype11 = _rtmp314.ktype;
_vtype12 = _rtmp314.vtype;
_size10 = _rtmp314.size;
for (var _i15 = 0; _i15 < _size10; ++_i15)
{
if (_i15 > 0 ) {
if (input.rstack.length > input.rpos[input.rpos.length -1] + 1) {
input.rstack.pop();
}
}
var key16 = null;
var val17 = null;
key16 = input.readString();
val17 = input.readString();
this.success[key16] = val17;
}
input.readMapEnd();
} else {
input.skip(ftype);
}
break;
case 0:
input.skip(ftype);
break;
default:
input.skip(ftype);
}
input.readFieldEnd();
}
input.readStructEnd();
return;
};
FacebookService_getOptions_result.prototype.write = function(output) {
output.writeStructBegin('FacebookService_getOptions_result');
if (this.success) {
output.writeFieldBegin('success', Thrift.Type.MAP, 0);
output.writeMapBegin(Thrift.Type.STRING, Thrift.Type.STRING, Thrift.objectLength(this.success));
for (var kiter18 in this.success)
{
if (this.success.hasOwnProperty(kiter18))
{
var viter19 = this.success[kiter18];
output.writeString(kiter18);
output.writeString(viter19);
}
}
output.writeMapEnd();
output.writeFieldEnd();
}
output.writeFieldStop();
output.writeStructEnd();
return;
};
var FacebookService_getCpuProfile_args = function(args) {
this.profileDurationInSec = null;
if (args) {
if (args.profileDurationInSec !== undefined) {
this.profileDurationInSec = args.profileDurationInSec;
}
}
};
FacebookService_getCpuProfile_args.prototype = {};
FacebookService_getCpuProfile_args.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
switch (fid)
{
case 1:
if (ftype == Thrift.Type.I32) {
this.profileDurationInSec = input.readI32();
} else {
input.skip(ftype);
}
break;
case 0:
input.skip(ftype);
break;
default:
input.skip(ftype);
}
input.readFieldEnd();
}
input.readStructEnd();
return;
};
FacebookService_getCpuProfile_args.prototype.write = function(output) {
output.writeStructBegin('FacebookService_getCpuProfile_args');
if (this.profileDurationInSec) {
output.writeFieldBegin('profileDurationInSec', Thrift.Type.I32, 1);
output.writeI32(this.profileDurationInSec);
output.writeFieldEnd();
}
output.writeFieldStop();
output.writeStructEnd();
return;
};
var FacebookService_getCpuProfile_result = function(args) {
this.success = null;
if (args) {
if (args.success !== undefined) {
this.success = args.success;
}
}
};
FacebookService_getCpuProfile_result.prototype = {};
FacebookService_getCpuProfile_result.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
switch (fid)
{
case 0:
if (ftype == Thrift.Type.STRING) {
this.success = input.readString();
} else {
input.skip(ftype);
}
break;
case 0:
input.skip(ftype);
break;
default:
input.skip(ftype);
}
input.readFieldEnd();
}
input.readStructEnd();
return;
};
FacebookService_getCpuProfile_result.prototype.write = function(output) {
output.writeStructBegin('FacebookService_getCpuProfile_result');
if (this.success) {
output.writeFieldBegin('success', Thrift.Type.STRING, 0);
output.writeString(this.success);
output.writeFieldEnd();
}
output.writeFieldStop();
output.writeStructEnd();
return;
};
var FacebookService_aliveSince_args = function(args) {
};
FacebookService_aliveSince_args.prototype = {};
FacebookService_aliveSince_args.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
input.skip(ftype);
input.readFieldEnd();
}
input.readStructEnd();
return;
};
FacebookService_aliveSince_args.prototype.write = function(output) {
output.writeStructBegin('FacebookService_aliveSince_args');
output.writeFieldStop();
output.writeStructEnd();
return;
};
var FacebookService_aliveSince_result = function(args) {
this.success = null;
if (args) {
if (args.success !== undefined) {
this.success = args.success;
}
}
};
FacebookService_aliveSince_result.prototype = {};
FacebookService_aliveSince_result.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
switch (fid)
{
case 0:
if (ftype == Thrift.Type.I64) {
this.success = input.readI64();
} else {
input.skip(ftype);
}
break;
case 0:
input.skip(ftype);
break;
default:
input.skip(ftype);
}
input.readFieldEnd();
}
input.readStructEnd();
return;
};
FacebookService_aliveSince_result.prototype.write = function(output) {
output.writeStructBegin('FacebookService_aliveSince_result');
if (this.success) {
output.writeFieldBegin('success', Thrift.Type.I64, 0);
output.writeI64(this.success);
output.writeFieldEnd();
}
output.writeFieldStop();
output.writeStructEnd();
return;
};
var FacebookService_reinitialize_args = function(args) {
};
FacebookService_reinitialize_args.prototype = {};
FacebookService_reinitialize_args.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
input.skip(ftype);
input.readFieldEnd();
}
input.readStructEnd();
return;
};
FacebookService_reinitialize_args.prototype.write = function(output) {
output.writeStructBegin('FacebookService_reinitialize_args');
output.writeFieldStop();
output.writeStructEnd();
return;
};
var FacebookService_reinitialize_result = function(args) {
};
FacebookService_reinitialize_result.prototype = {};
FacebookService_reinitialize_result.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
input.skip(ftype);
input.readFieldEnd();
}
input.readStructEnd();
return;
};
FacebookService_reinitialize_result.prototype.write = function(output) {
output.writeStructBegin('FacebookService_reinitialize_result');
output.writeFieldStop();
output.writeStructEnd();
return;
};
var FacebookService_shutdown_args = function(args) {
};
FacebookService_shutdown_args.prototype = {};
FacebookService_shutdown_args.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
input.skip(ftype);
input.readFieldEnd();
}
input.readStructEnd();
return;
};
FacebookService_shutdown_args.prototype.write = function(output) {
output.writeStructBegin('FacebookService_shutdown_args');
output.writeFieldStop();
output.writeStructEnd();
return;
};
var FacebookService_shutdown_result = function(args) {
};
FacebookService_shutdown_result.prototype = {};
FacebookService_shutdown_result.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
input.skip(ftype);
input.readFieldEnd();
}
input.readStructEnd();
return;
};
FacebookService_shutdown_result.prototype.write = function(output) {
output.writeStructBegin('FacebookService_shutdown_result');
output.writeFieldStop();
output.writeStructEnd();
return;
};
var FacebookServiceClient = exports.Client = function(output, pClass) {
this.output = output;
this.pClass = pClass;
this.seqid = 0;
this._reqs = {};
};
FacebookServiceClient.prototype = {};
FacebookServiceClient.prototype.getName = function(callback) {
this.seqid += 1;
this._reqs[this.seqid] = callback;
this.send_getName();
};
FacebookServiceClient.prototype.send_getName = function() {
var output = new this.pClass(this.output);
output.writeMessageBegin('getName', Thrift.MessageType.CALL, this.seqid);
var args = new FacebookService_getName_args();
args.write(output);
output.writeMessageEnd();
return this.output.flush();
};
FacebookServiceClient.prototype.recv_getName = function(input,mtype,rseqid) {
var callback = this._reqs[rseqid] || function() {};
delete this._reqs[rseqid];
if (mtype == Thrift.MessageType.EXCEPTION) {
var x = new Thrift.TApplicationException();
x.read(input);
input.readMessageEnd();
return callback(x);
}
var result = new FacebookService_getName_result();
result.read(input);
input.readMessageEnd();
if (null !== result.success) {
return callback(null, result.success);
}
return callback('getName failed: unknown result');
};
FacebookServiceClient.prototype.getVersion = function(callback) {
this.seqid += 1;
this._reqs[this.seqid] = callback;
this.send_getVersion();
};
FacebookServiceClient.prototype.send_getVersion = function() {
var output = new this.pClass(this.output);
output.writeMessageBegin('getVersion', Thrift.MessageType.CALL, this.seqid);
var args = new FacebookService_getVersion_args();
args.write(output);
output.writeMessageEnd();
return this.output.flush();
};
FacebookServiceClient.prototype.recv_getVersion = function(input,mtype,rseqid) {
var callback = this._reqs[rseqid] || function() {};
delete this._reqs[rseqid];
if (mtype == Thrift.MessageType.EXCEPTION) {
var x = new Thrift.TApplicationException();
x.read(input);
input.readMessageEnd();
return callback(x);
}
var result = new FacebookService_getVersion_result();
result.read(input);
input.readMessageEnd();
if (null !== result.success) {
return callback(null, result.success);
}
return callback('getVersion failed: unknown result');
};
FacebookServiceClient.prototype.getStatus = function(callback) {
this.seqid += 1;
this._reqs[this.seqid] = callback;
this.send_getStatus();
};
FacebookServiceClient.prototype.send_getStatus = function() {
var output = new this.pClass(this.output);
output.writeMessageBegin('getStatus', Thrift.MessageType.CALL, this.seqid);
var args = new FacebookService_getStatus_args();
args.write(output);
output.writeMessageEnd();
return this.output.flush();
};
FacebookServiceClient.prototype.recv_getStatus = function(input,mtype,rseqid) {
var callback = this._reqs[rseqid] || function() {};
delete this._reqs[rseqid];
if (mtype == Thrift.MessageType.EXCEPTION) {
var x = new Thrift.TApplicationException();
x.read(input);
input.readMessageEnd();
return callback(x);
}
var result = new FacebookService_getStatus_result();
result.read(input);
input.readMessageEnd();
if (null !== result.success) {
return callback(null, result.success);
}
return callback('getStatus failed: unknown result');
};
FacebookServiceClient.prototype.getStatusDetails = function(callback) {
this.seqid += 1;
this._reqs[this.seqid] = callback;
this.send_getStatusDetails();
};
FacebookServiceClient.prototype.send_getStatusDetails = function() {
var output = new this.pClass(this.output);
output.writeMessageBegin('getStatusDetails', Thrift.MessageType.CALL, this.seqid);
var args = new FacebookService_getStatusDetails_args();
args.write(output);
output.writeMessageEnd();
return this.output.flush();
};
FacebookServiceClient.prototype.recv_getStatusDetails = function(input,mtype,rseqid) {
var callback = this._reqs[rseqid] || function() {};
delete this._reqs[rseqid];
if (mtype == Thrift.MessageType.EXCEPTION) {
var x = new Thrift.TApplicationException();
x.read(input);
input.readMessageEnd();
return callback(x);
}
var result = new FacebookService_getStatusDetails_result();
result.read(input);
input.readMessageEnd();
if (null !== result.success) {
return callback(null, result.success);
}
return callback('getStatusDetails failed: unknown result');
};
FacebookServiceClient.prototype.getCounters = function(callback) {
this.seqid += 1;
this._reqs[this.seqid] = callback;
this.send_getCounters();
};
FacebookServiceClient.prototype.send_getCounters = function() {
var output = new this.pClass(this.output);
output.writeMessageBegin('getCounters', Thrift.MessageType.CALL, this.seqid);
var args = new FacebookService_getCounters_args();
args.write(output);
output.writeMessageEnd();
return this.output.flush();
};
FacebookServiceClient.prototype.recv_getCounters = function(input,mtype,rseqid) {
var callback = this._reqs[rseqid] || function() {};
delete this._reqs[rseqid];
if (mtype == Thrift.MessageType.EXCEPTION) {
var x = new Thrift.TApplicationException();
x.read(input);
input.readMessageEnd();
return callback(x);
}
var result = new FacebookService_getCounters_result();
result.read(input);
input.readMessageEnd();
if (null !== result.success) {
return callback(null, result.success);
}
return callback('getCounters failed: unknown result');
};
FacebookServiceClient.prototype.getCounter = function(key, callback) {
this.seqid += 1;
this._reqs[this.seqid] = callback;
this.send_getCounter(key);
};
FacebookServiceClient.prototype.send_getCounter = function(key) {
var output = new this.pClass(this.output);
output.writeMessageBegin('getCounter', Thrift.MessageType.CALL, this.seqid);
var args = new FacebookService_getCounter_args();
args.key = key;
args.write(output);
output.writeMessageEnd();
return this.output.flush();
};
FacebookServiceClient.prototype.recv_getCounter = function(input,mtype,rseqid) {
var callback = this._reqs[rseqid] || function() {};
delete this._reqs[rseqid];
if (mtype == Thrift.MessageType.EXCEPTION) {
var x = new Thrift.TApplicationException();
x.read(input);
input.readMessageEnd();
return callback(x);
}
var result = new FacebookService_getCounter_result();
result.read(input);
input.readMessageEnd();
if (null !== result.success) {
return callback(null, result.success);
}
return callback('getCounter failed: unknown result');
};
FacebookServiceClient.prototype.setOption = function(key, value, callback) {
this.seqid += 1;
this._reqs[this.seqid] = callback;
this.send_setOption(key, value);
};
FacebookServiceClient.prototype.send_setOption = function(key, value) {
var output = new this.pClass(this.output);
output.writeMessageBegin('setOption', Thrift.MessageType.CALL, this.seqid);
var args = new FacebookService_setOption_args();
args.key = key;
args.value = value;
args.write(output);
output.writeMessageEnd();
return this.output.flush();
};
FacebookServiceClient.prototype.recv_setOption = function(input,mtype,rseqid) {
var callback = this._reqs[rseqid] || function() {};
delete this._reqs[rseqid];
if (mtype == Thrift.MessageType.EXCEPTION) {
var x = new Thrift.TApplicationException();
x.read(input);
input.readMessageEnd();
return callback(x);
}
var result = new FacebookService_setOption_result();
result.read(input);
input.readMessageEnd();
callback(null)
};
FacebookServiceClient.prototype.getOption = function(key, callback) {
this.seqid += 1;
this._reqs[this.seqid] = callback;
this.send_getOption(key);
};
FacebookServiceClient.prototype.send_getOption = function(key) {
var output = new this.pClass(this.output);
output.writeMessageBegin('getOption', Thrift.MessageType.CALL, this.seqid);
var args = new FacebookService_getOption_args();
args.key = key;
args.write(output);
output.writeMessageEnd();
return this.output.flush();
};
FacebookServiceClient.prototype.recv_getOption = function(input,mtype,rseqid) {
var callback = this._reqs[rseqid] || function() {};
delete this._reqs[rseqid];
if (mtype == Thrift.MessageType.EXCEPTION) {
var x = new Thrift.TApplicationException();
x.read(input);
input.readMessageEnd();
return callback(x);
}
var result = new FacebookService_getOption_result();
result.read(input);
input.readMessageEnd();
if (null !== result.success) {
return callback(null, result.success);
}
return callback('getOption failed: unknown result');
};
FacebookServiceClient.prototype.getOptions = function(callback) {
this.seqid += 1;
this._reqs[this.seqid] = callback;
this.send_getOptions();
};
FacebookServiceClient.prototype.send_getOptions = function() {
var output = new this.pClass(this.output);
output.writeMessageBegin('getOptions', Thrift.MessageType.CALL, this.seqid);
var args = new FacebookService_getOptions_args();
args.write(output);
output.writeMessageEnd();
return this.output.flush();
};
FacebookServiceClient.prototype.recv_getOptions = function(input,mtype,rseqid) {
var callback = this._reqs[rseqid] || function() {};
delete this._reqs[rseqid];
if (mtype == Thrift.MessageType.EXCEPTION) {
var x = new Thrift.TApplicationException();
x.read(input);
input.readMessageEnd();
return callback(x);
}
var result = new FacebookService_getOptions_result();
result.read(input);
input.readMessageEnd();
if (null !== result.success) {
return callback(null, result.success);
}
return callback('getOptions failed: unknown result');
};
FacebookServiceClient.prototype.getCpuProfile = function(profileDurationInSec, callback) {
this.seqid += 1;
this._reqs[this.seqid] = callback;
this.send_getCpuProfile(profileDurationInSec);
};
FacebookServiceClient.prototype.send_getCpuProfile = function(profileDurationInSec) {
var output = new this.pClass(this.output);
output.writeMessageBegin('getCpuProfile', Thrift.MessageType.CALL, this.seqid);
var args = new FacebookService_getCpuProfile_args();
args.profileDurationInSec = profileDurationInSec;
args.write(output);
output.writeMessageEnd();
return this.output.flush();
};
FacebookServiceClient.prototype.recv_getCpuProfile = function(input,mtype,rseqid) {
var callback = this._reqs[rseqid] || function() {};
delete this._reqs[rseqid];
if (mtype == Thrift.MessageType.EXCEPTION) {
var x = new Thrift.TApplicationException();
x.read(input);
input.readMessageEnd();
return callback(x);
}
var result = new FacebookService_getCpuProfile_result();
result.read(input);
input.readMessageEnd();
if (null !== result.success) {
return callback(null, result.success);
}
return callback('getCpuProfile failed: unknown result');
};
FacebookServiceClient.prototype.aliveSince = function(callback) {
this.seqid += 1;
this._reqs[this.seqid] = callback;
this.send_aliveSince();
};
FacebookServiceClient.prototype.send_aliveSince = function() {
var output = new this.pClass(this.output);
output.writeMessageBegin('aliveSince', Thrift.MessageType.CALL, this.seqid);
var args = new FacebookService_aliveSince_args();
args.write(output);
output.writeMessageEnd();
return this.output.flush();
};
FacebookServiceClient.prototype.recv_aliveSince = function(input,mtype,rseqid) {
var callback = this._reqs[rseqid] || function() {};
delete this._reqs[rseqid];
if (mtype == Thrift.MessageType.EXCEPTION) {
var x = new Thrift.TApplicationException();
x.read(input);
input.readMessageEnd();
return callback(x);
}
var result = new FacebookService_aliveSince_result();
result.read(input);
input.readMessageEnd();
if (null !== result.success) {
return callback(null, result.success);
}
return callback('aliveSince failed: unknown result');
};
FacebookServiceClient.prototype.reinitialize = function(callback) {
this.seqid += 1;
this._reqs[this.seqid] = callback;
this.send_reinitialize();
};
FacebookServiceClient.prototype.send_reinitialize = function() {
var output = new this.pClass(this.output);
output.writeMessageBegin('reinitialize', Thrift.MessageType.CALL, this.seqid);
var args = new FacebookService_reinitialize_args();
args.write(output);
output.writeMessageEnd();
return this.output.flush();
};
FacebookServiceClient.prototype.shutdown = function(callback) {
this.seqid += 1;
this._reqs[this.seqid] = callback;
this.send_shutdown();
};
FacebookServiceClient.prototype.send_shutdown = function() {
var output = new this.pClass(this.output);
output.writeMessageBegin('shutdown', Thrift.MessageType.CALL, this.seqid);
var args = new FacebookService_shutdown_args();
args.write(output);
output.writeMessageEnd();
return this.output.flush();
};
var FacebookServiceProcessor = exports.Processor = function(handler) {
this._handler = handler
}
FacebookServiceProcessor.prototype.process = function(input, output) {
var r = input.readMessageBegin();
if (this['process_' + r.fname]) {
return this['process_' + r.fname].call(this, r.rseqid, input, output);
} else {
input.skip(Thrift.Type.STRUCT);
input.readMessageEnd();
var x = new Thrift.TApplicationException(Thrift.TApplicationExceptionType.UNKNOWN_METHOD, 'Unknown function ' + r.fname);
output.writeMessageBegin(r.fname, Thrift.MessageType.Exception, r.rseqid);
x.write(output);
output.writeMessageEnd();
output.flush();
}
}
FacebookServiceProcessor.prototype.process_getName = function(seqid, input, output) {
var args = new FacebookService_getName_args();
args.read(input);
input.readMessageEnd();
var result = new FacebookService_getName_result();
this._handler.getName(function (success) {
result.success = success;
output.writeMessageBegin("getName", Thrift.MessageType.REPLY, seqid);
result.write(output);
output.writeMessageEnd();
output.flush();
})
}
FacebookServiceProcessor.prototype.process_getVersion = function(seqid, input, output) {
var args = new FacebookService_getVersion_args();
args.read(input);
input.readMessageEnd();
var result = new FacebookService_getVersion_result();
this._handler.getVersion(function (success) {
result.success = success;
output.writeMessageBegin("getVersion", Thrift.MessageType.REPLY, seqid);
result.write(output);
output.writeMessageEnd();
output.flush();
})
}
FacebookServiceProcessor.prototype.process_getStatus = function(seqid, input, output) {
var args = new FacebookService_getStatus_args();
args.read(input);
input.readMessageEnd();
var result = new FacebookService_getStatus_result();
this._handler.getStatus(function (success) {
result.success = success;
output.writeMessageBegin("getStatus", Thrift.MessageType.REPLY, seqid);
result.write(output);
output.writeMessageEnd();
output.flush();
})
}
FacebookServiceProcessor.prototype.process_getStatusDetails = function(seqid, input, output) {
var args = new FacebookService_getStatusDetails_args();
args.read(input);
input.readMessageEnd();
var result = new FacebookService_getStatusDetails_result();
this._handler.getStatusDetails(function (success) {
result.success = success;
output.writeMessageBegin("getStatusDetails", Thrift.MessageType.REPLY, seqid);
result.write(output);
output.writeMessageEnd();
output.flush();
})
}
FacebookServiceProcessor.prototype.process_getCounters = function(seqid, input, output) {
var args = new FacebookService_getCounters_args();
args.read(input);
input.readMessageEnd();
var result = new FacebookService_getCounters_result();
this._handler.getCounters(function (success) {
result.success = success;
output.writeMessageBegin("getCounters", Thrift.MessageType.REPLY, seqid);
result.write(output);
output.writeMessageEnd();
output.flush();
})
}
FacebookServiceProcessor.prototype.process_getCounter = function(seqid, input, output) {
var args = new FacebookService_getCounter_args();
args.read(input);
input.readMessageEnd();
var result = new FacebookService_getCounter_result();
this._handler.getCounter(args.key, function (success) {
result.success = success;
output.writeMessageBegin("getCounter", Thrift.MessageType.REPLY, seqid);
result.write(output);
output.writeMessageEnd();
output.flush();
})
}
FacebookServiceProcessor.prototype.process_setOption = function(seqid, input, output) {
var args = new FacebookService_setOption_args();
args.read(input);
input.readMessageEnd();
var result = new FacebookService_setOption_result();
this._handler.setOption(args.key, args.value, function (success) {
result.success = success;
output.writeMessageBegin("setOption", Thrift.MessageType.REPLY, seqid);
result.write(output);
output.writeMessageEnd();
output.flush();
})
}
FacebookServiceProcessor.prototype.process_getOption = function(seqid, input, output) {
var args = new FacebookService_getOption_args();
args.read(input);
input.readMessageEnd();
var result = new FacebookService_getOption_result();
this._handler.getOption(args.key, function (success) {
result.success = success;
output.writeMessageBegin("getOption", Thrift.MessageType.REPLY, seqid);
result.write(output);
output.writeMessageEnd();
output.flush();
})
}
FacebookServiceProcessor.prototype.process_getOptions = function(seqid, input, output) {
var args = new FacebookService_getOptions_args();
args.read(input);
input.readMessageEnd();
var result = new FacebookService_getOptions_result();
this._handler.getOptions(function (success) {
result.success = success;
output.writeMessageBegin("getOptions", Thrift.MessageType.REPLY, seqid);
result.write(output);
output.writeMessageEnd();
output.flush();
})
}
FacebookServiceProcessor.prototype.process_getCpuProfile = function(seqid, input, output) {
var args = new FacebookService_getCpuProfile_args();
args.read(input);
input.readMessageEnd();
var result = new FacebookService_getCpuProfile_result();
this._handler.getCpuProfile(args.profileDurationInSec, function (success) {
result.success = success;
output.writeMessageBegin("getCpuProfile", Thrift.MessageType.REPLY, seqid);
result.write(output);
output.writeMessageEnd();
output.flush();
})
}
FacebookServiceProcessor.prototype.process_aliveSince = function(seqid, input, output) {
var args = new FacebookService_aliveSince_args();
args.read(input);
input.readMessageEnd();
var result = new FacebookService_aliveSince_result();
this._handler.aliveSince(function (success) {
result.success = success;
output.writeMessageBegin("aliveSince", Thrift.MessageType.REPLY, seqid);
result.write(output);
output.writeMessageEnd();
output.flush();
})
}
FacebookServiceProcessor.prototype.process_reinitialize = function(seqid, input, output) {
var args = new FacebookService_reinitialize_args();
args.read(input);
input.readMessageEnd();
this._handler.reinitialize()
}
FacebookServiceProcessor.prototype.process_shutdown = function(seqid, input, output) {
var args = new FacebookService_shutdown_args();
args.read(input);
input.readMessageEnd();
this._handler.shutdown()
}
================================================
FILE: lib/shib/engines/hiveserver/ThriftHive.js
================================================
//
// Autogenerated by Thrift Compiler (0.8.0)
//
// DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
//
var Thrift = require('node-thrift').Thrift;
var fb303_ttypes = require('./fb303_types')
var hive_metastore_ttypes = require('./hive_metastore_types')
var queryplan_ttypes = require('./queryplan_types')
var ThriftHiveMetastore = require('./ThriftHiveMetastore')
var ThriftHiveMetastoreClient = ThriftHiveMetastore.Client
var ThriftHiveMetastoreProcessor = ThriftHiveMetastore.Processor
var ttypes = require('./hive_service_types');
//HELPER FUNCTIONS AND STRUCTURES
var ThriftHive_execute_args = function(args) {
this.query = null;
if (args) {
if (args.query !== undefined) {
this.query = args.query;
}
}
};
ThriftHive_execute_args.prototype = {};
ThriftHive_execute_args.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
switch (fid)
{
case 1:
if (ftype == Thrift.Type.STRING) {
this.query = input.readString();
} else {
input.skip(ftype);
}
break;
case 0:
input.skip(ftype);
break;
default:
input.skip(ftype);
}
input.readFieldEnd();
}
input.readStructEnd();
return;
};
ThriftHive_execute_args.prototype.write = function(output) {
output.writeStructBegin('ThriftHive_execute_args');
if (this.query) {
output.writeFieldBegin('query', Thrift.Type.STRING, 1);
output.writeString(this.query);
output.writeFieldEnd();
}
output.writeFieldStop();
output.writeStructEnd();
return;
};
var ThriftHive_execute_result = function(args) {
this.ex = null;
if (args) {
if (args.ex !== undefined) {
this.ex = args.ex;
}
}
};
ThriftHive_execute_result.prototype = {};
ThriftHive_execute_result.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
switch (fid)
{
case 1:
if (ftype == Thrift.Type.STRUCT) {
this.ex = new ttypes.HiveServerException();
this.ex.read(input);
} else {
input.skip(ftype);
}
break;
case 0:
input.skip(ftype);
break;
default:
input.skip(ftype);
}
input.readFieldEnd();
}
input.readStructEnd();
return;
};
ThriftHive_execute_result.prototype.write = function(output) {
output.writeStructBegin('ThriftHive_execute_result');
if (this.ex) {
output.writeFieldBegin('ex', Thrift.Type.STRUCT, 1);
this.ex.write(output);
output.writeFieldEnd();
}
output.writeFieldStop();
output.writeStructEnd();
return;
};
var ThriftHive_fetchOne_args = function(args) {
};
ThriftHive_fetchOne_args.prototype = {};
ThriftHive_fetchOne_args.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
input.skip(ftype);
input.readFieldEnd();
}
input.readStructEnd();
return;
};
ThriftHive_fetchOne_args.prototype.write = function(output) {
output.writeStructBegin('ThriftHive_fetchOne_args');
output.writeFieldStop();
output.writeStructEnd();
return;
};
var ThriftHive_fetchOne_result = function(args) {
this.success = null;
this.ex = null;
if (args) {
if (args.success !== undefined) {
this.success = args.success;
}
if (args.ex !== undefined) {
this.ex = args.ex;
}
}
};
ThriftHive_fetchOne_result.prototype = {};
ThriftHive_fetchOne_result.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
switch (fid)
{
case 0:
if (ftype == Thrift.Type.STRING) {
this.success = input.readString();
} else {
input.skip(ftype);
}
break;
case 1:
if (ftype == Thrift.Type.STRUCT) {
this.ex = new ttypes.HiveServerException();
this.ex.read(input);
} else {
input.skip(ftype);
}
break;
default:
input.skip(ftype);
}
input.readFieldEnd();
}
input.readStructEnd();
return;
};
ThriftHive_fetchOne_result.prototype.write = function(output) {
output.writeStructBegin('ThriftHive_fetchOne_result');
if (this.success) {
output.writeFieldBegin('success', Thrift.Type.STRING, 0);
output.writeString(this.success);
output.writeFieldEnd();
}
if (this.ex) {
output.writeFieldBegin('ex', Thrift.Type.STRUCT, 1);
this.ex.write(output);
output.writeFieldEnd();
}
output.writeFieldStop();
output.writeStructEnd();
return;
};
var ThriftHive_fetchN_args = function(args) {
this.numRows = null;
if (args) {
if (args.numRows !== undefined) {
this.numRows = args.numRows;
}
}
};
ThriftHive_fetchN_args.prototype = {};
ThriftHive_fetchN_args.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
switch (fid)
{
case 1:
if (ftype == Thrift.Type.I32) {
this.numRows = input.readI32();
} else {
input.skip(ftype);
}
break;
case 0:
input.skip(ftype);
break;
default:
input.skip(ftype);
}
input.readFieldEnd();
}
input.readStructEnd();
return;
};
ThriftHive_fetchN_args.prototype.write = function(output) {
output.writeStructBegin('ThriftHive_fetchN_args');
if (this.numRows) {
output.writeFieldBegin('numRows', Thrift.Type.I32, 1);
output.writeI32(this.numRows);
output.writeFieldEnd();
}
output.writeFieldStop();
output.writeStructEnd();
return;
};
var ThriftHive_fetchN_result = function(args) {
this.success = null;
this.ex = null;
if (args) {
if (args.success !== undefined) {
this.success = args.success;
}
if (args.ex !== undefined) {
this.ex = args.ex;
}
}
};
ThriftHive_fetchN_result.prototype = {};
ThriftHive_fetchN_result.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
switch (fid)
{
case 0:
if (ftype == Thrift.Type.LIST) {
var _size0 = 0;
var _rtmp34;
this.success = [];
var _etype3 = 0;
_rtmp34 = input.readListBegin();
_etype3 = _rtmp34.etype;
_size0 = _rtmp34.size;
for (var _i5 = 0; _i5 < _size0; ++_i5)
{
var elem6 = null;
elem6 = input.readString();
this.success.push(elem6);
}
input.readListEnd();
} else {
input.skip(ftype);
}
break;
case 1:
if (ftype == Thrift.Type.STRUCT) {
this.ex = new ttypes.HiveServerException();
this.ex.read(input);
} else {
input.skip(ftype);
}
break;
default:
input.skip(ftype);
}
input.readFieldEnd();
}
input.readStructEnd();
return;
};
ThriftHive_fetchN_result.prototype.write = function(output) {
output.writeStructBegin('ThriftHive_fetchN_result');
if (this.success) {
output.writeFieldBegin('success', Thrift.Type.LIST, 0);
output.writeListBegin(Thrift.Type.STRING, this.success.length);
for (var iter7 in this.success)
{
if (this.success.hasOwnProperty(iter7))
{
iter7 = this.success[iter7];
output.writeString(iter7);
}
}
output.writeListEnd();
output.writeFieldEnd();
}
if (this.ex) {
output.writeFieldBegin('ex', Thrift.Type.STRUCT, 1);
this.ex.write(output);
output.writeFieldEnd();
}
output.writeFieldStop();
output.writeStructEnd();
return;
};
var ThriftHive_fetchAll_args = function(args) {
};
ThriftHive_fetchAll_args.prototype = {};
ThriftHive_fetchAll_args.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
input.skip(ftype);
input.readFieldEnd();
}
input.readStructEnd();
return;
};
ThriftHive_fetchAll_args.prototype.write = function(output) {
output.writeStructBegin('ThriftHive_fetchAll_args');
output.writeFieldStop();
output.writeStructEnd();
return;
};
var ThriftHive_fetchAll_result = function(args) {
this.success = null;
this.ex = null;
if (args) {
if (args.success !== undefined) {
this.success = args.success;
}
if (args.ex !== undefined) {
this.ex = args.ex;
}
}
};
ThriftHive_fetchAll_result.prototype = {};
ThriftHive_fetchAll_result.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
switch (fid)
{
case 0:
if (ftype == Thrift.Type.LIST) {
var _size8 = 0;
var _rtmp312;
this.success = [];
var _etype11 = 0;
_rtmp312 = input.readListBegin();
_etype11 = _rtmp312.etype;
_size8 = _rtmp312.size;
for (var _i13 = 0; _i13 < _size8; ++_i13)
{
var elem14 = null;
elem14 = input.readString();
this.success.push(elem14);
}
input.readListEnd();
} else {
input.skip(ftype);
}
break;
case 1:
if (ftype == Thrift.Type.STRUCT) {
this.ex = new ttypes.HiveServerException();
this.ex.read(input);
} else {
input.skip(ftype);
}
break;
default:
input.skip(ftype);
}
input.readFieldEnd();
}
input.readStructEnd();
return;
};
ThriftHive_fetchAll_result.prototype.write = function(output) {
output.writeStructBegin('ThriftHive_fetchAll_result');
if (this.success) {
output.writeFieldBegin('success', Thrift.Type.LIST, 0);
output.writeListBegin(Thrift.Type.STRING, this.success.length);
for (var iter15 in this.success)
{
if (this.success.hasOwnProperty(iter15))
{
iter15 = this.success[iter15];
output.writeString(iter15);
}
}
output.writeListEnd();
output.writeFieldEnd();
}
if (this.ex) {
output.writeFieldBegin('ex', Thrift.Type.STRUCT, 1);
this.ex.write(output);
output.writeFieldEnd();
}
output.writeFieldStop();
output.writeStructEnd();
return;
};
var ThriftHive_getSchema_args = function(args) {
};
ThriftHive_getSchema_args.prototype = {};
ThriftHive_getSchema_args.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
input.skip(ftype);
input.readFieldEnd();
}
input.readStructEnd();
return;
};
ThriftHive_getSchema_args.prototype.write = function(output) {
output.writeStructBegin('ThriftHive_getSchema_args');
output.writeFieldStop();
output.writeStructEnd();
return;
};
var ThriftHive_getSchema_result = function(args) {
this.success = null;
this.ex = null;
if (args) {
if (args.success !== undefined) {
this.success = args.success;
}
if (args.ex !== undefined) {
this.ex = args.ex;
}
}
};
ThriftHive_getSchema_result.prototype = {};
ThriftHive_getSchema_result.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
switch (fid)
{
case 0:
if (ftype == Thrift.Type.STRUCT) {
this.success = new hive_metastore_ttypes.Schema();
this.success.read(input);
} else {
input.skip(ftype);
}
break;
case 1:
if (ftype == Thrift.Type.STRUCT) {
this.ex = new ttypes.HiveServerException();
this.ex.read(input);
} else {
input.skip(ftype);
}
break;
default:
input.skip(ftype);
}
input.readFieldEnd();
}
input.readStructEnd();
return;
};
ThriftHive_getSchema_result.prototype.write = function(output) {
output.writeStructBegin('ThriftHive_getSchema_result');
if (this.success) {
output.writeFieldBegin('success', Thrift.Type.STRUCT, 0);
this.success.write(output);
output.writeFieldEnd();
}
if (this.ex) {
output.writeFieldBegin('ex', Thrift.Type.STRUCT, 1);
this.ex.write(output);
output.writeFieldEnd();
}
output.writeFieldStop();
output.writeStructEnd();
return;
};
var ThriftHive_getThriftSchema_args = function(args) {
};
ThriftHive_getThriftSchema_args.prototype = {};
ThriftHive_getThriftSchema_args.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
input.skip(ftype);
input.readFieldEnd();
}
input.readStructEnd();
return;
};
ThriftHive_getThriftSchema_args.prototype.write = function(output) {
output.writeStructBegin('ThriftHive_getThriftSchema_args');
output.writeFieldStop();
output.writeStructEnd();
return;
};
var ThriftHive_getThriftSchema_result = function(args) {
this.success = null;
this.ex = null;
if (args) {
if (args.success !== undefined) {
this.success = args.success;
}
if (args.ex !== undefined) {
this.ex = args.ex;
}
}
};
ThriftHive_getThriftSchema_result.prototype = {};
ThriftHive_getThriftSchema_result.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
switch (fid)
{
case 0:
if (ftype == Thrift.Type.STRUCT) {
this.success = new hive_metastore_ttypes.Schema();
this.success.read(input);
} else {
input.skip(ftype);
}
break;
case 1:
if (ftype == Thrift.Type.STRUCT) {
this.ex = new ttypes.HiveServerException();
this.ex.read(input);
} else {
input.skip(ftype);
}
break;
default:
input.skip(ftype);
}
input.readFieldEnd();
}
input.readStructEnd();
return;
};
ThriftHive_getThriftSchema_result.prototype.write = function(output) {
output.writeStructBegin('ThriftHive_getThriftSchema_result');
if (this.success) {
output.writeFieldBegin('success', Thrift.Type.STRUCT, 0);
this.success.write(output);
output.writeFieldEnd();
}
if (this.ex) {
output.writeFieldBegin('ex', Thrift.Type.STRUCT, 1);
this.ex.write(output);
output.writeFieldEnd();
}
output.writeFieldStop();
output.writeStructEnd();
return;
};
var ThriftHive_getClusterStatus_args = function(args) {
};
ThriftHive_getClusterStatus_args.prototype = {};
ThriftHive_getClusterStatus_args.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
input.skip(ftype);
input.readFieldEnd();
}
input.readStructEnd();
return;
};
ThriftHive_getClusterStatus_args.prototype.write = function(output) {
output.writeStructBegin('ThriftHive_getClusterStatus_args');
output.writeFieldStop();
output.writeStructEnd();
return;
};
var ThriftHive_getClusterStatus_result = function(args) {
this.success = null;
this.ex = null;
if (args) {
if (args.success !== undefined) {
this.success = args.success;
}
if (args.ex !== undefined) {
this.ex = args.ex;
}
}
};
ThriftHive_getClusterStatus_result.prototype = {};
ThriftHive_getClusterStatus_result.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
switch (fid)
{
case 0:
if (ftype == Thrift.Type.STRUCT) {
this.success = new ttypes.HiveClusterStatus();
this.success.read(input);
} else {
input.skip(ftype);
}
break;
case 1:
if (ftype == Thrift.Type.STRUCT) {
this.ex = new ttypes.HiveServerException();
this.ex.read(input);
} else {
input.skip(ftype);
}
break;
default:
input.skip(ftype);
}
input.readFieldEnd();
}
input.readStructEnd();
return;
};
ThriftHive_getClusterStatus_result.prototype.write = function(output) {
output.writeStructBegin('ThriftHive_getClusterStatus_result');
if (this.success) {
output.writeFieldBegin('success', Thrift.Type.STRUCT, 0);
this.success.write(output);
output.writeFieldEnd();
}
if (this.ex) {
output.writeFieldBegin('ex', Thrift.Type.STRUCT, 1);
this.ex.write(output);
output.writeFieldEnd();
}
output.writeFieldStop();
output.writeStructEnd();
return;
};
var ThriftHive_getQueryPlan_args = function(args) {
};
ThriftHive_getQueryPlan_args.prototype = {};
ThriftHive_getQueryPlan_args.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
input.skip(ftype);
input.readFieldEnd();
}
input.readStructEnd();
return;
};
ThriftHive_getQueryPlan_args.prototype.write = function(output) {
output.writeStructBegin('ThriftHive_getQueryPlan_args');
output.writeFieldStop();
output.writeStructEnd();
return;
};
var ThriftHive_getQueryPlan_result = function(args) {
this.success = null;
this.ex = null;
if (args) {
if (args.success !== undefined) {
this.success = args.success;
}
if (args.ex !== undefined) {
this.ex = args.ex;
}
}
};
ThriftHive_getQueryPlan_result.prototype = {};
ThriftHive_getQueryPlan_result.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
switch (fid)
{
case 0:
if (ftype == Thrift.Type.STRUCT) {
this.success = new queryplan_ttypes.QueryPlan();
this.success.read(input);
} else {
input.skip(ftype);
}
break;
case 1:
if (ftype == Thrift.Type.STRUCT) {
this.ex = new ttypes.HiveServerException();
this.ex.read(input);
} else {
input.skip(ftype);
}
break;
default:
input.skip(ftype);
}
input.readFieldEnd();
}
input.readStructEnd();
return;
};
ThriftHive_getQueryPlan_result.prototype.write = function(output) {
output.writeStructBegin('ThriftHive_getQueryPlan_result');
if (this.success) {
output.writeFieldBegin('success', Thrift.Type.STRUCT, 0);
this.success.write(output);
output.writeFieldEnd();
}
if (this.ex) {
output.writeFieldBegin('ex', Thrift.Type.STRUCT, 1);
this.ex.write(output);
output.writeFieldEnd();
}
output.writeFieldStop();
output.writeStructEnd();
return;
};
var ThriftHive_clean_args = function(args) {
};
ThriftHive_clean_args.prototype = {};
ThriftHive_clean_args.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
input.skip(ftype);
input.readFieldEnd();
}
input.readStructEnd();
return;
};
ThriftHive_clean_args.prototype.write = function(output) {
output.writeStructBegin('ThriftHive_clean_args');
output.writeFieldStop();
output.writeStructEnd();
return;
};
var ThriftHive_clean_result = function(args) {
};
ThriftHive_clean_result.prototype = {};
ThriftHive_clean_result.prototype.read = function(input) {
input.readStructBegin();
while (true)
{
var ret = input.readFieldBegin();
var fname = ret.fname;
var ftype = ret.ftype;
var fid = ret.fid;
if (ftype == Thrift.Type.STOP) {
break;
}
input.skip(ftype);
input.readFieldEnd();
}
input.readStructEnd();
return;
};
ThriftHive_clean_result.prototype.write = function(output) {
output.writeStructBegin('ThriftHive_clean_result');
output.writeFieldStop();
output.writeStructEnd();
return;
};
var ThriftHiveClient = exports.Client = function(output, pClass) {
this.output = output;
this.pClass = pClass;
this.seqid = 0;
this._reqs = {};
};
Thrift.inherits(ThriftHiveClient, ThriftHiveMetastoreClient)
ThriftHiveClient.prototype.execute = function(query, callback) {
this.seqid += 1;
this._reqs[this.seqid] = callback;
this.send_execute(query);
};
ThriftHiveClient.prototype.send_execute = function(query) {
var output = new this.pClass(this.output);
output.writeMessageBegin('execute', Thrift.MessageType.CALL, this.seqid);
var args = new ThriftHive_execute_args();
args.query = query;
args.write(output);
output.writeMessageEnd();
return this.output.flush();
};
ThriftHiveClient.prototype.recv_execute = function(input,mtype,rseqid) {
var callback = this._reqs[rseqid] || function() {};
delete this._reqs[rseqid];
if (mtype == Thrift.MessageType.EXCEPTION) {
var x = new Thrift.TApplicationException();
x.read(input);
input.readMessageEnd();
return callback(x);
}
var result = new ThriftHive_execute_result();
result.read(input);
input.readMessageEnd();
if (null !== result.ex) {
return callback(result.ex);
}
callback(null)
};
ThriftHiveClient.prototype.fetchOne = function(callback) {
this.seqid += 1;
this._reqs[this.seqid] = callback;
this.send_fetchOne();
};
ThriftHiveClient.prototype.send_fetchOne = function() {
var output = new this.pClass(this.output);
output.writeMessageBegin('fetchOne', Thrift.MessageType.CALL, this.seqid);
var args = new ThriftHive_fetchOne_args();
args.write(output);
output.writeMessageEnd();
return this.output.flush();
};
ThriftHiveClient.prototype.recv_fetchOne = function(input,mtype,rseqid) {
var callback = this._reqs[rseqid] || function() {};
delete this._reqs[rseqid];
if (mtype == Thrift.MessageType.EXCEPTION) {
var x = new Thrift.TApplicationException();
x.read(input);
input.readMessageEnd();
return callback(x);
}
var result = new ThriftHive_fetchOne_result();
result.read(input);
input.readMessageEnd();
if (null !== result.ex) {
return callback(result.ex);
}
if (null !== result.success) {
return callback(null, result.success);
}
return callback('fetchOne failed: unknown result');
};
ThriftHiveClient.prototype.fetchN = function(numRows, callback) {
this.seqid += 1;
this._reqs[this.seqid] = callback;
this.send_fetchN(numRows);
};
ThriftHiveClient.prototype.send_fetchN = function(numRows) {
var output = new this.pClass(this.output);
output.writeMessageBegin('fetchN', Thrift.MessageType.CALL, this.seqid);
var args = new ThriftHive_fetchN_args();
args.numRows = numRows;
args.write(output);
output.writeMessageEnd();
return this.output.flush();
};
ThriftHiveClient.prototype.recv_fetchN = function(input,mtype,rseqid) {
var callback = this._reqs[rseqid] || function() {};
delete this._reqs[rseqid];
if (mtype == Thrift.MessageType.EXCEPTION) {
var x = new Thrift.TApplicationException();
x.read(input);
input.readMessageEnd();
return callback(x);
}
var result = new ThriftHive_fetchN_result();
result.read(input);
input.readMessageEnd();
if (null !== result.ex) {
return callback(result.ex);
}
if (null !== result.success) {
return callback(null, result.success);
}
return callback('fetchN failed: unknown result');
};
ThriftHiveClient.prototype.fetchAll = function(callback) {
this.seqid += 1;
this._reqs[this.seqid] = callback;
this.send_fetchAll();
};
ThriftHiveClient.prototype.send_fetchAll = function() {
var output = new this.pClass(this.output);
output.writeMessageBegin('fetchAll', Thrift.MessageType.CALL, this.seqid);
var args = new ThriftHive_fetchAll_args();
args.write(output);
output.writeMessageEnd();
return this.output.flush();
};
ThriftHiveClient.prototype.recv_fetchAll = function(input,mtype,rseqid) {
var callback = this._reqs[rseqid] || function() {};
delete this._reqs[rseqid];
if (mtype == Thrift.MessageType.EXCEPTION) {
var x = new Thrift.TApplicationException();
x.read(input);
input.readMessageEnd();
return callback(x);
}
var result = new ThriftHive_fetchAll_result();
result.read(input);
input.readMessageEnd();
if (null !== result.ex) {
return callback(result.ex);
}
if (null !== result.success) {
return callback(null, result.success);
}
return callback('fetchAll failed: unknown result');
};
ThriftHiveClient.prototype.getSchema = function(callback) {
this.seqid += 1;
this._reqs[this.seqid] = callback;
this.send_getSchema();
};
ThriftHiveClient.prototype.send_getSchema = function() {
var output = new this.pClass(this.output);
output.writeMessageBegin('getSchema', Thrift.MessageType.CALL, this.seqid);
var args = new ThriftHive_getSchema_args();
args.write(output);
output.writeMessageEnd();
return this.output.flush();
};
ThriftHiveClient.prototype.recv_getSchema = function(input,mtype,rseqid) {
var callback = this._reqs[rseqid] || function() {};
delete this._reqs[rseqid];
if (mtype == Thrift.MessageType.EXCEPTION) {
var x = new Thrift.TApplicationException();
x.read(input);
input.readMessageEnd();
return callback(x);
}
var result = new ThriftHive_getSchema_result();
result.read(input);
input.readMessageEnd();
if (null !== result.ex) {
return callback(result.ex);
}
if (null !== result.success) {
return callback(null, result.success);
}
return callback('getSchema failed: unknown result');
};
ThriftHiveClient.prototype.getThriftSchema = function(callback) {
this.seqid += 1;
this._reqs[this.seqid] = callback;
this.send_getThriftSchema();
};
ThriftHiveClient.prototype.send_getThriftSchema = function() {
var output = new this.pClass(this.output);
output.writeMessageBegin('getThriftSchema', Thrift.MessageType.CALL, this.seqid);
var args = new ThriftHive_getThriftSchema_args();
args.write(output);
output.writeMessageEnd();
return this.output.flush();
};
ThriftHiveClient.prototype.recv_getThriftSchema = function(input,mtype,rseqid) {
var callback = this._reqs[rseqid] || function() {};
delete this._reqs[rseqid];
if (mtype == Thrift.MessageType.EXCEPTION) {
var x = new Thrift.TApplicationException();
x.read(input);
input.readMessageEnd();
return callback(x);
}
var result = new ThriftHive_getThriftSchema_result();
result.read(input);
input.readMessageEnd();
if (null !== result.ex) {
return callback(result.ex);
}
if (null !== result.success) {
return callback(null, result.success);
}
return callback('getThriftSchema failed: unknown result');
};
ThriftHiveClient.prototype.getClusterStatus = function(callback) {
this.seqid += 1;
this._reqs[this.seqid] = callback;
this.send_getClusterStatus();
};
ThriftHiveClient.prototype.send_getClusterStatus = function() {
var output = new this.pClass(this.output);
output.writeMessageBegin('getClusterStatus', Thrift.MessageType.CALL, this.seqid);
var args = new ThriftHive_getClusterStatus_args();
args.write(output);
output.writeMessageEnd();
return this.output.flush();
};
ThriftHiveClient.prototype.recv_getClusterStatus = function(input,mtype,rseqid) {
var callback = this._reqs[rseqid] || function() {};
delete this._reqs[rseqid];
if (mtype == Thrift.MessageType.EXCEPTION) {
var x = new Thrift.TApplicationException();
x.read(input);
input.readMessageEnd();
return callback(x);
}
var result = new ThriftHive_getClusterStatus_result();
result.read(input);
input.readMessageEnd();
if (null !== result.ex) {
return callback(result.ex);
}
if (null !== result.success) {
return callback(null, result.success);
}
return callback('getClusterStatus failed: unknown result');
};
ThriftHiveClient.prototype.getQueryPlan = function(callback) {
this.seqid += 1;
this._reqs[this.seqid] = callback;
this.send_getQueryPlan();
};
ThriftHiveClient.prototype.send_getQueryPlan = function() {
var output = new this.pClass(this.output);
output.writeMessageBegin('getQueryPlan', Thrift.MessageType.CALL, this.seqid);
var args = new ThriftHive_getQueryPlan_args();
args.write(output);
output.writeMessageEnd();
return this.output.flush();
};
ThriftHiveClient.prototype.recv_getQueryPlan = function(input,mtype,rseqid) {
var callback = this._reqs[rseqid] || function() {};
delete this._reqs[rseqid];
if (mtype == Thrift.MessageType.EXCEPTION) {
var x = new Thrift.TApplicationException();
x.read(input);
input.readMessageEnd();
return callback(x);
}
var result = new ThriftHive_getQueryPlan_result();
result.read(input);
input.readMessageEnd();
if (null !== result.ex) {
return callback(result.ex);
}
if (null !== result.success) {
return callback(null, result.success);
}
return callback('getQueryPlan failed: unknown result');
};
ThriftHiveClient.prototype.clean = function(callback) {
this.seqid += 1;
this._reqs[this.seqid] = callback;
this.send_clean();
};
ThriftHiveClient.prototype.send_clean = function() {
var output = new this.pClass(this.output);
output.writeMessageBegin('clean', Thrift.MessageType.CALL, this.seqid);
var args = new ThriftHive_clean_args();
args.write(output);
output.writeMessageEnd();
return this.output.flush();
};
ThriftHiveClient.prototype.recv_clean = function(input,mtype,rseqid) {
var callback = this._reqs[rseqid] || function() {};
delete this._reqs[rseqid];
if (mtype == Thrift.MessageType.EXCEPTION) {
var x = new Thrift.TApplicationException();
x.read(input);
input.readMessageEnd();
return callback(x);
}
var result = new ThriftHive_clean_result();
result.read(input);
input.readMessageEnd();
callback(null)
};
var ThriftHiveProcessor = exports.Processor = function(handler) {
this._handler = handler
}
Thrift.inherits(ThriftHiveProcessor, ThriftHiveMetastoreProcessor)
ThriftHiveProcessor.prototype.process = function(input, output) {
var r = input.readMessageBegin();
if (this['process_' + r.fname]) {
return this['process_' + r.fname].call(this, r.rseqid, input,
gitextract_i18lfx99/
├── .gitignore
├── .gitmodules
├── README.md
├── app.js
├── bin/
│ ├── dbmigrate.js
│ └── purge.js
├── config.js
├── examples/
│ ├── hive_beeline_client.js
│ ├── hive_client.js
│ ├── hive_methods.js
│ ├── hive_multi_client.js
│ ├── hive_server2_engine.example.js
│ └── hive_server2_results.js
├── lib/
│ └── shib/
│ ├── access_control.js
│ ├── auth/
│ │ ├── dumb.js
│ │ ├── http_basic_auth.js
│ │ └── http_custom_header.js
│ ├── auth.js
│ ├── client.js
│ ├── engine.js
│ ├── engines/
│ │ ├── bigquery/
│ │ │ └── index.js
│ │ ├── dummy/
│ │ │ └── index.js
│ │ ├── dummyengine.js
│ │ ├── hiveserver/
│ │ │ ├── FacebookService.js
│ │ │ ├── ThriftHive.js
│ │ │ ├── ThriftHiveMetastore.js
│ │ │ ├── fb303_types.js
│ │ │ ├── hive_metastore_types.js
│ │ │ ├── hive_service_types.js
│ │ │ ├── index.js
│ │ │ └── queryplan_types.js
│ │ ├── hiveserver2/
│ │ │ ├── TCLIService.js
│ │ │ ├── TCLIService_types.js
│ │ │ └── index.js
│ │ ├── huahin_mrv1/
│ │ │ ├── index.js
│ │ │ └── rest.js
│ │ ├── huahin_yarn/
│ │ │ ├── index.js
│ │ │ └── rest.js
│ │ ├── jobtracker/
│ │ │ ├── client.js
│ │ │ └── index.js
│ │ ├── presto/
│ │ │ └── index.js
│ │ └── yarn/
│ │ ├── client.js
│ │ └── index.js
│ ├── huahin_client.js
│ ├── index.js
│ ├── localdiskstore.js
│ ├── logger.js
│ ├── query.js
│ └── simple_csv_builder.js
├── package.json
├── public/
│ ├── css/
│ │ ├── redmond/
│ │ │ └── jquery-ui-1.8.13.custom.css
│ │ ├── shib.css
│ │ └── ui.dynatree.css
│ ├── index.graphtest.html
│ ├── index.jquery-ui-default.html
│ └── js/
│ └── shib.js
├── src/
│ └── hiveserver/
│ └── if/
│ ├── TCLIService.thrift
│ ├── hive_service.thrift
│ ├── metastore/
│ │ └── if/
│ │ └── hive_metastore.thrift
│ ├── ql/
│ │ └── if/
│ │ └── queryplan.thrift
│ └── share/
│ └── fb303/
│ └── if/
│ └── fb303.thrift
├── test/
│ ├── t_access_control.js
│ ├── t_query.js
│ ├── t_simple_csv_builder.js
│ └── t_thrifthivemock.js
├── testtools/
│ ├── ThriftHiveMock.js
│ ├── check_database_hs2_engine.js
│ ├── check_database_hs2_use.js
│ ├── check_engine.js
│ ├── check_engine_huahin_mrv1.js
│ └── hive_server_mock.js
├── var/
│ └── .datadir
└── views/
└── index.jade
SYMBOL INDEX (88 symbols across 11 files)
FILE: app.js
function shutdown (line 24) | function shutdown(signal){
function error_handle (line 35) | function error_handle(req, res, err){
function shibclient (line 42) | function shibclient(req){
function pseudo_query_data (line 272) | function pseudo_query_data(query){
function pseudo_status (line 428) | function pseudo_status(query_state){
function pseudo_result_data (line 469) | function pseudo_result_data(query){
FILE: lib/shib/client.js
function error_callback (line 71) | function error_callback(name, t, callback, err, data){
FILE: lib/shib/engines/bigquery/index.js
function convertStatus (line 339) | function convertStatus(jobname, status) {
FILE: lib/shib/engines/huahin_mrv1/index.js
function convertStatus (line 23) | function convertStatus(status) {
FILE: lib/shib/engines/huahin_yarn/index.js
function convertStatus (line 23) | function convertStatus(status, info) { // 'status' is one of members of...
FILE: lib/shib/engines/jobtracker/index.js
function convertStatus (line 31) | function convertStatus(status) {
FILE: lib/shib/engines/presto/index.js
function convertStatus (line 314) | function convertStatus(jobname, status) {
FILE: lib/shib/localdiskstore.js
function generatePlaceholders (line 138) | function generatePlaceholders(num) {
FILE: lib/shib/query.js
function pad2 (line 153) | function pad2(n){return n < 10 ? '0' + n : n ;}
FILE: public/js/shib.js
function authAjax (line 17) | function authAjax(req){
function authGet (line 23) | function authGet(url, callback){
function authGetJSON (line 27) | function authGetJSON(url, callback){
function authGetText (line 31) | function authGetText(url, callback){
function load_pairs (line 91) | function load_pairs(callback) {
function check_auth_initial (line 112) | function check_auth_initial() {
function set_execute_query_list (line 137) | function set_execute_query_list(list) {
function delete_execute_query_item (line 142) | function delete_execute_query_item(queryid) {
function execute_query_list (line 147) | function execute_query_list() {
function push_execute_query_list (line 158) | function push_execute_query_list(queryid, refresh) {
function set_bookmark_query_list (line 170) | function set_bookmark_query_list(list) {
function delete_bookmark_query_list (line 175) | function delete_bookmark_query_list(queryid) {
function bookmark_query_list (line 180) | function bookmark_query_list() {
function exists_in_bookmark_query_list (line 191) | function exists_in_bookmark_query_list(queryid) {
function push_bookmark_query_list (line 196) | function push_bookmark_query_list(queryid) {
function query_last_result (line 203) | function query_last_result(query) {
function query_second_last_result (line 210) | function query_second_last_result(query) {
function query_last_done_result (line 217) | function query_last_done_result(query) {
function query_result_schema_label (line 224) | function query_result_schema_label(result){
function query_current_state (line 228) | function query_current_state(query) {
function timelabel_elapsed (line 257) | function timelabel_elapsed(completed_at, executed_at){
function follow_current_uri (line 271) | function follow_current_uri() {
function follow_current_uri_query (line 283) | function follow_current_uri_query(queryid){
function follow_current_uri_tag (line 319) | function follow_current_uri_tag(tag){
function update_history_by_query (line 340) | function update_history_by_query(query) {
function show_notification (line 365) | function show_notification(event){ /* event object is not used */
function update_notification (line 390) | function update_notification(type, title, message){
function show_info (line 401) | function show_info(title, message, duration){
function show_error (line 405) | function show_error(title, message, duration, optional_object){
function show_tables_dialog (line 413) | function show_tables_dialog() {
function show_describe_dialog (line 447) | function show_describe_dialog() {
function show_taglist_dialog (line 483) | function show_taglist_dialog() {
function show_status_dialog (line 513) | function show_status_dialog(target) {
function show_edit_tag_dialog (line 568) | function show_edit_tag_dialog(){
function execute_add_tag (line 598) | function execute_add_tag(){
function execute_remove_tag (line 620) | function execute_remove_tag(){
function show_auth_dialog (line 642) | function show_auth_dialog(){
function update_tabs (line 648) | function update_tabs(reloading, taginfo, history_disabled) {
function load_tabs (line 693) | function load_tabs(opts) {
function create_queryitem_object (line 733) | function create_queryitem_object(queryid, id_prefix){
function update_yours_tab (line 753) | function update_yours_tab(){
function update_bookmark_tab (line 763) | function update_bookmark_tab(){
function update_history_tab (line 773) | function update_history_tab(){
function update_tag_tab (line 787) | function update_tag_tab(tag, queryids){
function deselect_and_new_query (line 796) | function deselect_and_new_query(quiet){
function set_selected_query (line 803) | function set_selected_query(query, dom){
function release_selected_query (line 810) | function release_selected_query(){
function select_queryitem (line 818) | function select_queryitem(event){
function initiate_mainview (line 840) | function initiate_mainview(eventNotUsed, quiet) {
function copy_selected_query (line 847) | function copy_selected_query(eventNotUsed) {
function clip_selected_query (line 866) | function clip_selected_query(eventNotUsed) {
function unclip_selected_query (line 878) | function unclip_selected_query(eventNotUsed) {
function update_mainview (line 890) | function update_mainview(query){
function update_queryeditor (line 896) | function update_queryeditor(editable, querystring) {
function update_editbox (line 905) | function update_editbox(query, optional_state) {
function show_editbox_buttons (line 965) | function show_editbox_buttons(buttons){
function show_query_exec_pairs (line 980) | function show_query_exec_pairs(query){
function change_editbox_querystatus_style (line 993) | function change_editbox_querystatus_style(query, state, result){
function show_editbox_querytags (line 1037) | function show_editbox_querytags(query){
function load_query_tree (line 1068) | function load_query_tree(queryids, callback){
function load_queries (line 1080) | function load_queries(queryids, callback){
function load_results (line 1099) | function load_results(resultids, callback){
function check_selected_running_query_state (line 1122) | function check_selected_running_query_state(event){ /* event object is n...
function update_query_display (line 1130) | function update_query_display(query) {
function update_query (line 1136) | function update_query(query){
function update_running_queries (line 1163) | function update_running_queries(event){
function check_auth (line 1179) | function check_auth(e) {
function execute_query (line 1205) | function execute_query() {
function giveup_query (line 1254) | function giveup_query() {
function show_status_query (line 1288) | function show_status_query(eventNotUsed) {
function delete_query (line 1296) | function delete_query(event) {
function show_result_query (line 1326) | function show_result_query(opts) { /* opts: {range:full/head} */
function download_result_query (line 1340) | function download_result_query(opts) { /* opts: {format:tsv/csv} */
FILE: testtools/ThriftHiveMock.js
function pad (line 226) | function pad(n){return n<10 ? '0'+n : n;}
Condensed preview — 73 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (1,285K chars).
[
{
"path": ".gitignore",
"chars": 104,
"preview": "var/database.sqlite3\nvar/database.sqlite3.v0\nvar/results\nnode_modules\nnpm-debug.log\nproduction.js\n*.swp\n"
},
{
"path": ".gitmodules",
"chars": 0,
"preview": ""
},
{
"path": "README.md",
"chars": 15637,
"preview": "# shib\n\n* http://github.com/tagomoris/shib\n* Blog entry [tagomoris.tumblr](http://tagomoris.tumblr.com/post/77890621867/"
},
{
"path": "app.js",
"chars": 19999,
"preview": "var express = require('express'),\n jade = require('jade'),\n async = require('async'),\n fs = require('fs'),\n "
},
{
"path": "bin/dbmigrate.js",
"chars": 5635,
"preview": "var async = require('async')\n , fs = require('fs')\n , sqlite3 = require('sqlite3');\n\nvar target_db_path = process.argv"
},
{
"path": "bin/purge.js",
"chars": 1947,
"preview": "var async = require('async')\n , fs = require('fs')\n , sqlite3 = require('sqlite3');\n\nif (process.argv.length < 4) {\n "
},
{
"path": "config.js",
"chars": 671,
"preview": "var servers = exports.servers = {\n listen: 3000,\n fetch_lines: 1000,\n query_timeout: null, // seconds. (null:shib wil"
},
{
"path": "examples/hive_beeline_client.js",
"chars": 6448,
"preview": "// NODE_PATH=lib node example/hive_beeline_client.js\n\nvar thrift = require('node-thrift')\n , ttransport = require('node"
},
{
"path": "examples/hive_client.js",
"chars": 862,
"preview": "var thrift = require('thrift'),\n ttransport = require('thrift/transport'),\n ThriftHive = require('./gen-nodejs/Thr"
},
{
"path": "examples/hive_methods.js",
"chars": 1905,
"preview": "var thrift = require('thrift'),\n ttransport = require('thrift/transport'),\n ThriftHive = require('gen-nodejs/Thrif"
},
{
"path": "examples/hive_multi_client.js",
"chars": 1760,
"preview": "var thrift = require('thrift'),\n ttransport = require('thrift/transport'),\n ThriftHive = require('gen-nodejs/Thrif"
},
{
"path": "examples/hive_server2_engine.example.js",
"chars": 2133,
"preview": "var engine = require('shib/engines/hiveserver2');\nvar executer = new engine.Executer({\"name\":\"hiveserver2\", \"host\":\"hive"
},
{
"path": "examples/hive_server2_results.js",
"chars": 5659,
"preview": "/*\n * GetTables\n * rows: [ table_items ]\n * table_items: [ ???(catalog?), database(schema?), tablename, tabletype, ??? ]"
},
{
"path": "lib/shib/access_control.js",
"chars": 1509,
"preview": "var AccessControl = exports.AccessControl = function(config){\n this.default_rule = config['default'] || \"allow\";\n this"
},
{
"path": "lib/shib/auth/dumb.js",
"chars": 371,
"preview": "var Auth = exports.Auth = function(args){\n};\n\nAuth.prototype.check = function(req, callback) {\n callback(null, {usernam"
},
{
"path": "lib/shib/auth/http_basic_auth.js",
"chars": 2925,
"preview": "var url = require('url')\n , http = require('http')\n , https = require('https');\n\nvar AccessControl = require('shib/acc"
},
{
"path": "lib/shib/auth/http_custom_header.js",
"chars": 2849,
"preview": "var AccessControl = require('shib/access_control').AccessControl;\n\n/*\nvar auth = {\n require_always: true,\n type: 'http"
},
{
"path": "lib/shib/auth.js",
"chars": 2732,
"preview": "var crypto = require('crypto');\n\nvar dumb = require('./auth/dumb')\n , http_basic_auth = require('./auth/http_basic_auth"
},
{
"path": "lib/shib/client.js",
"chars": 18260,
"preview": "var Query = require('./query').Query;\n\nvar async = require('async');\n\nvar localdiskstore = require('./localdiskstore'),\n"
},
{
"path": "lib/shib/engine.js",
"chars": 11454,
"preview": "var dummyengine = require('./engines/dummyengine')\n , dummy = require('./engines/dummy')\n , hiveserver = require('./en"
},
{
"path": "lib/shib/engines/bigquery/index.js",
"chars": 9794,
"preview": "var gcloud = require('gcloud')\n , util = require('util');\n\n// check interval for real queries, not system queries\nvar B"
},
{
"path": "lib/shib/engines/dummy/index.js",
"chars": 3163,
"preview": "/*\n * This engine is for development on local environment\n */\n\nvar Executer = exports.Executer = function(conf, logger){"
},
{
"path": "lib/shib/engines/dummyengine.js",
"chars": 1763,
"preview": "var Executer = exports.Executer = function(){\n this.supports = function(operation){ return true; };\n};\n\nExecuter.protot"
},
{
"path": "lib/shib/engines/hiveserver/FacebookService.js",
"chars": 47285,
"preview": "//\n// Autogenerated by Thrift Compiler (0.8.0)\n//\n// DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n//"
},
{
"path": "lib/shib/engines/hiveserver/ThriftHive.js",
"chars": 36438,
"preview": "//\n// Autogenerated by Thrift Compiler (0.8.0)\n//\n// DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n//"
},
{
"path": "lib/shib/engines/hiveserver/ThriftHiveMetastore.js",
"chars": 343168,
"preview": "//\n// Autogenerated by Thrift Compiler (0.8.0)\n//\n// DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n//"
},
{
"path": "lib/shib/engines/hiveserver/fb303_types.js",
"chars": 309,
"preview": "//\n// Autogenerated by Thrift Compiler (0.8.0)\n//\n// DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n//"
},
{
"path": "lib/shib/engines/hiveserver/hive_metastore_types.js",
"chars": 79424,
"preview": "//\n// Autogenerated by Thrift Compiler (0.8.0)\n//\n// DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n//"
},
{
"path": "lib/shib/engines/hiveserver/hive_service_types.js",
"chars": 5715,
"preview": "//\n// Autogenerated by Thrift Compiler (0.8.0)\n//\n// DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n//"
},
{
"path": "lib/shib/engines/hiveserver/index.js",
"chars": 4345,
"preview": "var thrift = require('node-thrift'),\n ttransport = require('node-thrift/lib/thrift/transport'),\n ThriftHive = requ"
},
{
"path": "lib/shib/engines/hiveserver/queryplan_types.js",
"chars": 34407,
"preview": "//\n// Autogenerated by Thrift Compiler (0.8.0)\n//\n// DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n//"
},
{
"path": "lib/shib/engines/hiveserver2/TCLIService.js",
"chars": 67416,
"preview": "//\n// Autogenerated by Thrift Compiler (0.9.1)\n//\n// DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n//"
},
{
"path": "lib/shib/engines/hiveserver2/TCLIService_types.js",
"chars": 123713,
"preview": "//\n// Autogenerated by Thrift Compiler (0.9.1)\n//\n// DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n//"
},
{
"path": "lib/shib/engines/hiveserver2/index.js",
"chars": 21878,
"preview": "var thrift = require('node-thrift')\n , ttransport = require('node-thrift/lib/thrift/transport')\n , TCLIService = requi"
},
{
"path": "lib/shib/engines/huahin_mrv1/index.js",
"chars": 39472,
"preview": "var RESTClient = require('./rest').Client;\n\nvar Monitor = exports.Monitor = function(conf, logger) {\n if (conf.name !=="
},
{
"path": "lib/shib/engines/huahin_mrv1/rest.js",
"chars": 2058,
"preview": "var http = require('http');\n\nvar Client = exports.Client = function(host, port){\n this.host = host;\n this.port = port;"
},
{
"path": "lib/shib/engines/huahin_yarn/index.js",
"chars": 4526,
"preview": "var RESTClient = require('./rest').Client;\n\nvar Monitor = exports.Monitor = function(conf, logger) {\n if (conf.name !=="
},
{
"path": "lib/shib/engines/huahin_yarn/rest.js",
"chars": 1845,
"preview": "var http = require('http');\n\nvar Client = exports.Client = function(host, port){\n this._host = host;\n this._port = por"
},
{
"path": "lib/shib/engines/jobtracker/client.js",
"chars": 2896,
"preview": "var http = require('http')\n , child_process = require('child_process');\n\nvar cheerio = require('cheerio');\n\nvar MRv1Cli"
},
{
"path": "lib/shib/engines/jobtracker/index.js",
"chars": 2495,
"preview": "var MRv1Client = require('./client').MRv1Client;\n\nvar Monitor = exports.Monitor = function(conf, logger) {\n if (conf.na"
},
{
"path": "lib/shib/engines/presto/index.js",
"chars": 12350,
"preview": "var Client = require('presto-client').Client\n , JSONbig = require('json-bigint');\n\n// check interval for real queries, "
},
{
"path": "lib/shib/engines/yarn/client.js",
"chars": 4335,
"preview": "var http = require('http')\n , child_process = require('child_process');\n\nvar YARNClient = exports.YARNClient = function"
},
{
"path": "lib/shib/engines/yarn/index.js",
"chars": 882,
"preview": "var YARNClient = require('./client').YARNClient;\n\nvar Monitor = exports.Monitor = function(conf, logger) {\n if (conf.na"
},
{
"path": "lib/shib/huahin_client.js",
"chars": 2167,
"preview": "var http = require('http');\n\nvar HuahinClient = exports.HuahinClient = function(host, port){\n this.host = host;\n this."
},
{
"path": "lib/shib/index.js",
"chars": 1370,
"preview": "var client = require('./client')\n , auth = require('./auth')\n , logger = require('./logger');\n\nvar default_configs = {"
},
{
"path": "lib/shib/localdiskstore.js",
"chars": 11987,
"preview": "var fs = require('fs'),\n async = require('async'),\n sqlite3 = require('sqlite3');\n\nvar Query = require('./query')."
},
{
"path": "lib/shib/logger.js",
"chars": 1290,
"preview": "var util = require('util');\n\nvar TRACE = 0\n , DEBUG = 1\n , INFO = 2\n , WARN = 3\n , ERROR = 4\n , FATAL = 5;\n\nvar LOG"
},
{
"path": "lib/shib/query.js",
"chars": 6565,
"preview": "var crypto = require('crypto');\n\nvar STATE_RUNNING = \"running\",\n STATE_DONE = \"done\",\n STATE_ERROR = \"error\";\n\nvar"
},
{
"path": "lib/shib/simple_csv_builder.js",
"chars": 472,
"preview": "var SimpleCSVBuilder = exports.SimpleCSVBuilder = function(args){\n};\n\nSimpleCSVBuilder.escape = function(value){\n retur"
},
{
"path": "package.json",
"chars": 1387,
"preview": "{\n \"name\": \"shib\",\n \"version\": \"1.0.0\",\n \"engines\": {\n \"node\": \">= 0.8\"\n },\n \"description\": \"Hive Client Web App"
},
{
"path": "public/css/redmond/jquery-ui-1.8.13.custom.css",
"chars": 34297,
"preview": "/*\n * jQuery UI CSS Framework 1.8.13\n *\n * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)\n * Dual licensed unde"
},
{
"path": "public/css/shib.css",
"chars": 4830,
"preview": "/* overwrite jquery-ui style */\n.ui-tabs .ui-tabs-panel {padding: 1px;}\n\nbody {font: 62.5% \"Trebuchet MS\", sans-serif; m"
},
{
"path": "public/css/ui.dynatree.css",
"chars": 10339,
"preview": "/*******************************************************************************\r\n * Tree container\r\n */\r\nul.dynatree-co"
},
{
"path": "public/index.graphtest.html",
"chars": 31827,
"preview": "<!DOCTYPE html>\n<html>\n<head>\n<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\" />\n<title>shib - Query "
},
{
"path": "public/index.jquery-ui-default.html",
"chars": 31127,
"preview": "<!DOCTYPE html>\n<html>\n<head>\n<meta http-equiv=\"Content-Type\" content=\"text/html; charset=iso-8859-1\" />\n<title>jQuery U"
},
{
"path": "public/js/shib.js",
"chars": 42001,
"preview": "var shibnotifications = [];\nvar shibdata = {};\nvar shibselectedquery = null;\nvar shibselectedquery_dom = null;\n\nvar shib"
},
{
"path": "src/hiveserver/if/TCLIService.thrift",
"chars": 26980,
"preview": "// Licensed to the Apache Software Foundation (ASF) under one\n// or more contributor license agreements. See the NOTICE"
},
{
"path": "src/hiveserver/if/hive_service.thrift",
"chars": 3070,
"preview": "#!/usr/local/bin/thrift -java\n\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor licens"
},
{
"path": "src/hiveserver/if/metastore/if/hive_metastore.thrift",
"chars": 18810,
"preview": "#!/usr/local/bin/thrift -java\n#\n# Thrift Service that the MetaStore is built on\n#\n\ninclude \"share/fb303/if/fb303.thrift\""
},
{
"path": "src/hiveserver/if/ql/if/queryplan.thrift",
"chars": 2232,
"preview": "namespace java org.apache.hadoop.hive.ql.plan.api\nnamespace cpp Apache.Hadoop.Hive\n\nenum AdjacencyType { CONJUNCTIVE, DI"
},
{
"path": "src/hiveserver/if/share/fb303/if/fb303.thrift",
"chars": 2439,
"preview": "/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements. See the NOTI"
},
{
"path": "test/t_access_control.js",
"chars": 2352,
"preview": "var testCase = require('nodeunit').testCase;\nvar access_control = require('shib/access_control'),\n AccessControl = ac"
},
{
"path": "test/t_query.js",
"chars": 12086,
"preview": "var testCase = require('nodeunit').testCase;\nvar query = require('shib/query'),\n Query = query.Query;\n\nvar convertSer"
},
{
"path": "test/t_simple_csv_builder.js",
"chars": 1680,
"preview": "var testCase = require('nodeunit').testCase;\nvar builder = require('shib/simple_csv_builder').SimpleCSVBuilder;\n\nmodule."
},
{
"path": "test/t_thrifthivemock.js",
"chars": 2798,
"preview": "var testCase = require('nodeunit').testCase;\nvar mock = require('./../testtools/ThriftHiveMock');\n\nmodule.exports = test"
},
{
"path": "testtools/ThriftHiveMock.js",
"chars": 10664,
"preview": "var hive_service_types = require('shib/engines/hiveserver/hive_service_types'),\n hive_metastore_types = require('shib"
},
{
"path": "testtools/check_database_hs2_engine.js",
"chars": 1165,
"preview": "var async = require('async');\nvar Executer = require('shib/engines/hiveserver2/test').Executer;\n\nvar host = ''\n , port "
},
{
"path": "testtools/check_database_hs2_use.js",
"chars": 1783,
"preview": "var async = require('async');\nvar Executer = require('shib/engines/hiveserver2').Executer;\n\nvar host = ''\n , port = 100"
},
{
"path": "testtools/check_engine.js",
"chars": 1432,
"preview": "var engine = require('shib/engine');\n\n// node check_engine.js CONFIG_JS_PATH OPERATION args ...\n//\n// ex: node check_eng"
},
{
"path": "testtools/check_engine_huahin_mrv1.js",
"chars": 972,
"preview": "var huahin_mrv1 = require('shib/engines/huahin_mrv1');\n\n// node check_engine_huahin_mrv1.js HUAHIN_HOST HUAHIN_PORT OPER"
},
{
"path": "testtools/hive_server_mock.js",
"chars": 3286,
"preview": "var thrift = require('node-thrift'),\n ttransport = require('node-thrift/lib/thrift/transport');\nvar ThriftHive = requ"
},
{
"path": "var/.datadir",
"chars": 0,
"preview": ""
},
{
"path": "views/index.jade",
"chars": 8532,
"preview": "doctype html\nhtml\n head\n meta(http-equiv=\"Content-Type\", content=\"text/html\", charset=\"utf-8\")\n title shib - Quer"
}
]
About this extraction
This page contains the full source code of the tagomoris/shib GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 73 files (1.2 MB), approximately 324.3k tokens, and a symbol index with 88 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.