[ { "path": ".gitignore", "content": ".idea\n.DS_Store\nnode_modules\n*.sock\n*.rdb\ntest/incomplete\n*.swp\n" }, { "path": ".npmignore", "content": "support\ntest\nexamples\n*.sock\n" }, { "path": ".travis.yml", "content": "language: node_js\nnode_js:\n - \"4\"\n - \"6\"\n - \"8\"\n - \"node\"\nservices:\n - redis-server\n" }, { "path": "History.md", "content": "0.11.5 / 2016-11-05\n===================\n\n* Fix even more redis command callbacks\n* Fix redis commands SLC integration #978\n\n\n0.11.4 / 2016-10-21\n===================\n\n* adding reds module to optional dependencies\n\n\n0.11.3 / 2016-10-21\n===================\n\n* Fix making reds module optional, #969\n\n\n0.11.2 / 2016-10-14\n===================\n\n* Update packages to remove CVEs, #932\n* Make reds an optional dependency, #922\n* Remove unnecessary dependency to lodash-deep, #921\n* Expose shutdown in process worker ctx, #912\n* Add ioredis support to watchStuckJobs, #884\n\n\n0.11.1 / 2016-06-15\n===================\n\n* Upgrade redis to 2.6\n* Add switch for each job event\n\n\n0.11.0 / 2016-05-13\n===================\n\n* force node_redis version to 2.4.x, Closes #857\n* Converting Job ids back into integers, #855\n* Fix LPUSH crash during shutdown, #854\n* Install kue-dashboard script, #853\n* Add start event to documentation, #841\n* Add parameter for testMode.enter to continue processing jobs, #821\n* Modern Node.js versions support, #812\n* Don't start the next job until the current one is totally finished, Closes #806\n* Store multiple instances of jobs in jobs id map to emit events for all, #750 \n\n\n0.10.6 / 2016-04-27\n===================\n\n* Redis Cluster fix, Closes #861\n\n\n0.10.5 / 2016-01-14\n===================\n\n* Attempts surpassing max attempts on delay jobs upon failure, resulting in infinite retries, Fixes #797\n* Add yargs dependency for kue-dashboard, #796\n\n\n0.10.4 / 2016-01-14\n===================\n\n* fix zpop callback on shutdown\n* fix connection_options in test.js\n* Unit tests for redis.js #779\n* Tests for kue.js #778\n\n\n0.10.3 / 2015-11-20\n===================\n\n* Fixing Job processing order without munging the job id, Closes #708, Closes #678\n\n\n0.10.2 / 2015-11-20\n===================\n\n* Add support for ioredis, Closes #652\n* Add support for Redis Cluster, Closes #642\n* Fix `this.state` on refreshTTL \n\n\n0.10.0 / 2015-11-20\n===================\n\n* Update TTL on job progress, Closes #694\n* Upgrade to node_redis 2.3, #717\n* Fix LPUSH vs connection quit race when shutting down\n* Restart task btn, #754\n* Fix uncaught exception in job.js, #751\n* Added kue-dashboard script for conveniently running the dashboard #611\n* Fixed invalid CSS on production, #755\n* Connection string not supporting DB number #725\n* Fix attempts remaining logic, #742\n* Update jade, #741\n* Properly set job IDs in test mode, #727\n* Enhanced Job.log formatting, #630\n* Use node's util#format() in Job.log, #724\n\n\n0.9.6 / 2015-10-06\n===================\n\n* Fix redirection issue\n\n\n0.9.5 / 2015-09-16\n===================\n\n* When no ttl is set for jobs, don't let high priorities to conflict, fixes #697\n* Fix redirection issue, closes #685\n* Get progress_data along with other redis fields, PR #642\n* Grab only password from Redis URL, fixes #681\n* Add remove job event, PR #665\n\n\n0.9.4 / 2015-07-17\n===================\n\n* Job that doesn't call done() retries twice, fixes #669\n\n\n0.9.3 / 2015-05-07\n===================\n\n* Fix unlocking promotion lock, Closes #608\n\n\n0.9.2 / 2015-05-07\n===================\n\n* Fix duplicate job promotion/ttl race, Closes #601\n\n\n0.9.1 / 2015-05-05\n===================\n\n* Filter only jobs that have ttl set, Fixes #590\n\n\n0.9.0 / 2015-05-02\n===================\n\n* Upgrade to express 4.x, Closes #537\n* Move `job.reprocess` done callback to the last, Closes #387, Closes #385\n* Standardize signature of `.shutdown()` callback, Closes #454\n* Turn off search indexes by default, Closes #412\n* Improve delayed job promotion feature, Closes #533, fixes #312, closes #352\n* Use a distributed redis lock to hide job promotion from user, Closes #556\n* Deprecate `.promote` and update documentation\n* Document Javascript API to query queue state, Closes #455\n* Add jobEvents flag to switch off job events for memory optimization, Closes #401\n* Add idle event to capture unsuccessful zpop's in between of worker get Job, should fix #538\n* Add TTL for active jobs, Closes #544\n* Document `jobEvents` queue config, Closes #557\n* Bulk job create API now processes all jobs in case of intermediate errors, Closes #552\n* Merge `red job remove buttons and tooltips` PR, Closes #566\n* Add a in-memory test Kue mode, Closes #561\n* Update reds package to `0.2.5`\n* Merge PR #594, bad redirect URL in old express versions, fixes #592\n* update dependency to forked warlock repo to fix redis connection cleanup on shutdown, fixes #578\n* Update job hash with the worker ID, Closes #580\n\n\n0.8.12 / 2015-03-22\n===================\n\n* Bulk job create JSON API, Closes #334, Closes #500, Closes #527\n* Add feature to specify redis connection string/url, Closes #540\n* Mention kue-ui in readme, Closes #502\n* Add an extra parameter to the progress method to notify extra contextual data, Closes #466, Closes #427, Closes #313\n* Document job event callback arguments, Closes #542\n* Fix typo in documentation, Closes #506\n* Document importance of using Kue `error` listeners, Closes #409\n* Document Queue maintenance and job.removeOnComplete( true ), Closes #439\n* Document how to query all the active jobs programmatically, Closes #418\n* Document to explain how \"stuck queued jobs\" happens, Closes #451\n* Document on proper error handling to prevent stuck jobs, Closes #391\n\n\n0.8.11 / 2014-12-15\n===================\n\n* Fix shutdown on re-attemptable jobs, Closes #469\n* Fix race condition in delaying jobs when re-attempts, Closes #483\n* Make `watchStuckJobs` aware of queue prefix, Closes #452\n* Send along error message when emitting a failed event, Closes #461\n\n\n0.8.10 / 2014-12-13\n===================\n\n* Add more tests, Closes #280\n* More atomic job state changes, Closes #411\n* Documentation: error passed to done should be string or standard JS error object, Closes #394\n* Documentation: backoff documentation, Closes #435\n* Documentation: correct `promote` usage, Closes #413\n* Add job enqueue event, Closes #458\n* Watch for errors with non-string err.stack, Closes #426\n* Fix web app redirect path for express 4.0, Closes #393\n* `removeBadJob` should do pessimistic job removal from all state ZSETs, Closes #438\n* Add stats json api by type and state, Closes #477\n* Don't let concurrent graceful shutdowns on subsequent`Queue#shutdown`calls, Closes #479\n* Fix `cleanup` global leak, Closes #475\n\n\n0.8.9 / 2014-10-01\n==================\n\n* Properly update status flags on resume, Closes #423\n\n0.8.8 / 2014-09-12\n==================\n\n* Fix tests to limited shutdown timeouts\n* Add a redis lua watchdog to fix stuck inactive jobs, fixes #130\n* Stuck inactive jobs watchdog, Closes #130\n\n0.8.7 / 2014-09-12\n==================\n\n* Shutdown timeout problems and races, fixes #406\n\n0.8.6 / 2014-08-30\n==================\n\n* Quit redis connections on shutdown & let the process exit, closes #398\n\n0.8.5 / 2014-08-10\n==================\n\n * Fix typo in removeOnComplete\n \n0.8.4 / 2014-08-08\n==================\n\n * Emit event 'job failed attempt' after job successfully updated, closes #377\n * Fix delaying jobs when failed, closes #384\n * Implement `job.removeOnComplete`, closes #383\n * Make searchKeys chainable, closes #379\n * Add extra job options to JSON API, closes #378\n \n0.8.3 / 2014-07-13\n==================\n\n * Inject other Redis clients compatible with node_redis #344\n * Add support to connect to Redis using Linux sockets #362\n * Add .save callback sample code in documentation #367\n\n0.8.2 / 2014-07-08\n==================\n\n * Fix broken failure backoff #360\n * Merge web console redirection fix #357\n * Add db selection option to redis configuration #354\n * Get number of jobs with given state and type #349\n * Add Queue.prototype.delayed function #351\n\n0.8.1 / 2014-06-13\n==================\n\n * Fix wrong parameter orders in complete event #343s\n * Graceful shutdown bug fix #328\n\n0.8.0 / 2014-06-11\n==================\n\n * Implement backoff on failure retries #300\n * Allow passing back worker results via done to event handlers #170\n * Allow job producer to specify which keys of `job.data` to be indexed for search #284\n * Waffle.io Badge #332\n * Dropping monkey-patch style redis client connections\n * Update docs: Worker Pause/Resume-ability\n * Update docs: Reliability of Queue event handlers over Job event handlers\n\n0.7.9 / 2014-06-01\n==================\n\n * Graceful shutdown bug fix #336\n * More robust graceful shutdown under heavy load #328\n\n0.7.6 / 2014-05-02\n==================\n\n * Fixed broken monkey-patch style redis connections #323\n\n0.7.0 / 2014-01-24\n==================\n\n * Suppress \"undefined\" messages on String errors. Closes #230\n * Fix cannot read property id of undefined errors. Closes #252\n * Parameterize limit of jobs checked in promotion cycles. Closes #244\n * Graceful shutdown\n * Worker pause/resume ability, Closes #163\n * Ensure event subscription before job save. Closes #179\n * Fix Queue singleton\n * Fix failed event being called in first attempt. Closes #142\n * Disable search (Search index memory leaks). See #58 & #218\n * Emit error events on both kue and job\n * JS/Coffeescript tests added (Mocha+Should)\n * Travis support added\n\n\n0.6.2 / 2013-04-03\n==================\n\n * Fix redirection to active for mounted apps\n\n\n0.6.1 / 2013-03-25\n==================\n\n * Fixed issue preventing polling for new jobs. Closes #192\n\n\n0.6.0 / 2013-03-20\n==================\n\n * Make pollForJobs actually use ms argument. Closes #158\n * Support delay over HTTP POST. Closes #165\n * Fix natural sorting. Closes #174\n * Update `updated_at` timestamp during `log`, `progress`, `attempt`, or `state` changes. Closes #188\n * Fix redirection to /active. Closes #190\n\n0.5.0 / 2012-11-16\n==================\n\n * add POST /job to create a job\n * fix /job/search hang\n\n0.4.2 / 2012-11-08\n==================\n\n * Revert \"Fix delay() not really delaying\"\n * Revert \"If a job with a delay has more attempts, honor the original delay\"\n\n0.4.1 / 2012-09-25\n==================\n\n * fix: if a job with a delay has more attempts, honor the original delay [mathrawka]\n\n0.4.0 / 2012-06-28\n==================\n\n * Added 0.8.0 support\n\n0.3.4 / 2012-02-23\n==================\n\n * Changed: reduce polling by using BLPOP to notify workers of activity [Davide Bertola]\n\n0.3.3 / 2011-11-28\n==================\n\n * Fixed: use relative stats route to support mounting [alexkwolfe]\n * Fixed 0.6.x support\n * Removed empty Makefile\n\n0.3.2 / 2011-10-04\n==================\n\n * Removed unnecessary \"pooling\"\n * Fixed multiple event emitting. Closes #73\n * Fixed menu styling\n\n0.3.1 / 2011-08-25\n==================\n\n * Fixed auto event subscription. Closes #68\n * Changed: one redis connection for all workers\n * Removed user-select: none from everything. Closes #50\n\n0.3.0 / 2011-08-11\n==================\n\n * Added search capabilities\n * Added `workTime` stat\n * Added removal of stale jobs example\n * Added Queue-level job events, useful for removing stale jobs etc. Closes * Changed: lazy load reds search [David Wood]\n * Fixed `Job#error` for modules that throw strings or emit `error` events with strings [guillermo] #51\n * Fixed `Job#remove(fn)`\n * Fixed proxy issue with paths, use relative paths [booo]\n\n0.2.0 / 2011-07-25\n==================\n\n * Added infinite scroll\n * Added delayed job support\n * Added configurable redis support [davidwood]\n * Added job windowing. Closes #28\n * Added `Job#delay(ms)`\n * Removed job scrollIntoView\n * Removed fancy scrollbar (for infinite scroll / windowing :( )\n * Removed \"More\" button\n * Fixed z-index for actions\n * Fixed job mapping. Closes #43\n\n0.1.0 / 2011-07-19\n==================\n\n * Added exposing of progress via redis pubsub\n * Added pubsub job events \"complete\" and \"failed\"\n * Fixed: capping of progress > 100 == 100\n * UI: scroll details into view\n\n0.0.3 / 2011-07-07\n==================\n\n * Added caustic to aid in template management\n * Added job attempt support. Closes #31\n * Added `Job.attempts(n)`\n * Added minified jQuery\n * Added cluster integration docs. Closes #13\n * Added GET _/jobs/:from..:to_ to JSON API\n * Fixed: hide \"More\" on sort\n * Fixed: hide \"More\" on filter\n * Fixed: removed \"error\" emission, blows up when no one is listening\n\n0.0.2 / 2011-07-05\n==================\n\n * Added support to update state from UI. Closes #26\n * Added support to alter priority in UI. Closes #25\n * Added filtering by type. Closes #20\n\n0.0.1 / 2011-07-04\n==================\n\n * Initial release\n" }, { "path": "LICENSE", "content": "The MIT License\n\nCopyright (c) 2011 LearnBoost \n\nPermission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the 'Software'), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" }, { "path": "Makefile", "content": "REPORTER = spec\n\nall: build\n\nbuild:\n\t@./node_modules/coffee-script/bin/coffee \\\n -c \\\n -o lib src\n\ntest-tdd:\n\t@./node_modules/.bin/mocha \\\n --reporter $(REPORTER) \\\n\t\t--require should \\\n\t\t--require sinon \\\n --ui tdd \\\n test/tdd/*.js\n\ntest-bdd:\n\t@./node_modules/.bin/mocha \\\n --reporter $(REPORTER) \\\n --require should \\\n --ui bdd \\\n test/*.js\n\ntest-bdd-coffee:\n\t@./node_modules/.bin/mocha \\\n --compilers coffee:coffee-script \\\n --reporter $(REPORTER) \\\n --require should \\\n --require coffee-script/register \\\n --ui bdd \\\n test/*.coffee\n\n\ntest-all: test-tdd test-bdd test-bdd-coffee\n\n.PHONY: test-all\n" }, { "path": "Readme.md", "content": "# Kue\n\n## Kue is no longer maintained\n\nPlease see e.g. [Bull](https://github.com/OptimalBits/bull) as an alternative. Thank you!\n\n[![Build Status](https://travis-ci.org/Automattic/kue.svg?branch=master&style=flat)](https://travis-ci.org/Automattic/kue)\n[![npm version](https://badge.fury.io/js/kue.svg?style=flat)](http://badge.fury.io/js/kue)\n[![Dependency Status](https://img.shields.io/david/Automattic/kue.svg?style=flat)](https://david-dm.org/Automattic/kue)\n[![Join the chat at https://gitter.im/Automattic/kue](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/Automattic/kue?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)\n\n\nKue is a priority job queue backed by [redis](http://redis.io), built for [node.js](http://nodejs.org).\n\n**PROTIP** This is the latest Kue documentation, make sure to also read the [changelist](History.md).\n\n\n\n## Upgrade Notes (Please Read)\n - [0.9 -> 0.10](https://github.com/Automattic/kue/wiki/Upgrading-to-0.10.x)\n - [0.8 -> 0.9](https://github.com/Automattic/kue/wiki/Upgrading-to-0.9.x)\n\n\n\n## Installation\n\n - Latest release:\n\n $ npm install kue\n\n - Master branch:\n\n $ npm install http://github.com/Automattic/kue/tarball/master\n\n[![NPM](https://nodei.co/npm/kue.png?downloads=true&stars=true)](https://nodei.co/npm/kue/)\n\n## Features\n\n - Delayed jobs\n - Distribution of parallel work load\n - Job event and progress pubsub\n - Job TTL\n - Optional retries with backoff\n - Graceful workers shutdown\n - Full-text search capabilities\n - RESTful JSON API\n - Rich integrated UI\n - Infinite scrolling\n - UI progress indication\n - Job specific logging\n - Powered by Redis\n\n## Overview\n\n - [Creating Jobs](#creating-jobs)\n - [Jobs Priority](#job-priority)\n - [Failure Attempts](#failure-attempts)\n - [Failure Backoff](#failure-backoff)\n - [Job TTL](#job-ttl)\n - [Job Logs](#job-logs)\n - [Job Progress](#job-progress)\n - [Job Events](#job-events)\n - [Queue Events](#queue-events)\n - [Delayed Jobs](#delayed-jobs)\n - [Processing Jobs](#processing-jobs)\n - [Processing Concurrency](#processing-concurrency)\n - [Pause Processing](#pause-processing)\n - [Updating Progress](#updating-progress)\n - [Graceful Shutdown](#graceful-shutdown)\n - [Error Handling](#error-handling)\n - [Queue Maintenance](#queue-maintenance)\n - [Redis Connection Settings](#redis-connection-settings)\n - [User-Interface](#user-interface)\n - [JSON API](#json-api)\n - [Parallel Processing With Cluster](#parallel-processing-with-cluster)\n - [Securing Kue](#securing-kue)\n - [Testing](#testing)\n - [Screencasts](#screencasts)\n - [License](#license)\n\n\n\n## Creating Jobs\n\nFirst create a job `Queue` with `kue.createQueue()`:\n\n```js\nvar kue = require('kue')\n , queue = kue.createQueue();\n```\n\nCalling `queue.create()` with the type of job (\"email\"), and arbitrary job data will return a `Job`, which can then be `save()`ed, adding it to redis, with a default priority level of \"normal\". The `save()` method optionally accepts a callback, responding with an `error` if something goes wrong. The `title` key is special-cased, and will display in the job listings within the UI, making it easier to find a specific job.\n\n```js\nvar job = queue.create('email', {\n title: 'welcome email for tj'\n , to: 'tj@learnboost.com'\n , template: 'welcome-email'\n}).save( function(err){\n if( !err ) console.log( job.id );\n});\n```\n\n### Job Priority\n\nTo specify the priority of a job, simply invoke the `priority()` method with a number, or priority name, which is mapped to a number.\n\n```js\nqueue.create('email', {\n title: 'welcome email for tj'\n , to: 'tj@learnboost.com'\n , template: 'welcome-email'\n}).priority('high').save();\n```\n\nThe default priority map is as follows:\n\n```js\n{\n low: 10\n , normal: 0\n , medium: -5\n , high: -10\n , critical: -15\n};\n```\n\n### Failure Attempts\n\nBy default jobs only have _one_ attempt, that is when they fail, they are marked as a failure, and remain that way until you intervene. However, Kue allows you to specify this, which is important for jobs such as transferring an email, which upon failure, may usually retry without issue. To do this invoke the `.attempts()` method with a number.\n\n```js\n queue.create('email', {\n title: 'welcome email for tj'\n , to: 'tj@learnboost.com'\n , template: 'welcome-email'\n }).priority('high').attempts(5).save();\n```\n\n### Failure Backoff\nJob retry attempts are done as soon as they fail, with no delay, even if your job had a delay set via `Job#delay`. If you want to delay job re-attempts upon failures (known as backoff) you can use `Job#backoff` method in different ways:\n\n```js\n // Honor job's original delay (if set) at each attempt, defaults to fixed backoff\n job.attempts(3).backoff( true )\n\n // Override delay value, fixed backoff\n job.attempts(3).backoff( {delay: 60*1000, type:'fixed'} )\n\n // Enable exponential backoff using original delay (if set)\n job.attempts(3).backoff( {type:'exponential'} )\n\n // Use a function to get a customized next attempt delay value\n job.attempts(3).backoff( function( attempts, delay ){\n //attempts will correspond to the nth attempt failure so it will start with 0\n //delay will be the amount of the last delay, not the initial delay unless attempts === 0\n return my_customized_calculated_delay;\n })\n```\n\nIn the last scenario, provided function will be executed (via eval) on each re-attempt to get next attempt delay value, meaning that you can't reference external/context variables within it.\n\n### Job TTL\n\nJob producers can set an expiry value for the time their job can live in active state, so that if workers didn't reply in timely fashion, Kue will fail it with `TTL exceeded` error message preventing that job from being stuck in active state and spoiling concurrency.\n\n```js\nqueue.create('email', {title: 'email job with TTL'}).ttl(milliseconds).save();\n```\n\n### Job Logs\n\nJob-specific logs enable you to expose information to the UI at any point in the job's life-time. To do so simply invoke `job.log()`, which accepts a message string as well as variable-arguments for sprintf-like support:\n\n```js\njob.log('$%d sent to %s', amount, user.name);\n```\n\nor anything else (uses [util.inspect()](https://nodejs.org/api/util.html#util_util_inspect_object_options) internally):\n\n```js\njob.log({key: 'some key', value: 10});\njob.log([1,2,3,5,8]);\njob.log(10.1);\n```\n\n### Job Progress\n\nJob progress is extremely useful for long-running jobs such as video conversion. To update the job's progress simply invoke `job.progress(completed, total [, data])`:\n\n```js\njob.progress(frames, totalFrames);\n```\n\ndata can be used to pass extra information about the job. For example a message or an object with some extra contextual data to the current status.\n\n### Job Events\n\nJob-specific events are fired on the `Job` instances via Redis pubsub. The following events are currently supported:\n\n- `enqueue` the job is now queued\n- `start` the job is now running\n- `promotion` the job is promoted from delayed state to queued\n- `progress` the job's progress ranging from 0-100\n- `failed attempt` the job has failed, but has remaining attempts yet\n- `failed` the job has failed and has no remaining attempts\n- `complete` the job has completed\n- `remove` the job has been removed\n\n\nFor example this may look something like the following:\n\n```js\nvar job = queue.create('video conversion', {\n title: 'converting loki\\'s to avi'\n , user: 1\n , frames: 200\n});\n\njob.on('complete', function(result){\n console.log('Job completed with data ', result);\n\n}).on('failed attempt', function(errorMessage, doneAttempts){\n console.log('Job failed');\n\n}).on('failed', function(errorMessage){\n console.log('Job failed');\n\n}).on('progress', function(progress, data){\n console.log('\\r job #' + job.id + ' ' + progress + '% complete with data ', data );\n\n});\n```\n\n**Note** that Job level events are not guaranteed to be received upon process restarts, since restarted node.js process will lose the reference to the specific Job object. If you want a more reliable event handler look for [Queue Events](#queue-events).\n\n**Note** Kue stores job objects in memory until they are complete/failed to be able to emit events on them. If you have a huge concurrency in uncompleted jobs, turn this feature off and use queue level events for better memory scaling.\n\n ```js\n kue.createQueue({jobEvents: false})\n ```\n\n Alternatively, you can use the job level function `events` to control whether events are fired for a job at the job level.\n\n ```js\nvar job = queue.create('test').events(false).save();\n ```\n\n### Queue Events\n\nQueue-level events provide access to the job-level events previously mentioned, however scoped to the `Queue` instance to apply logic at a \"global\" level. An example of this is removing completed jobs:\n\n```js\nqueue.on('job enqueue', function(id, type){\n console.log( 'Job %s got queued of type %s', id, type );\n\n}).on('job complete', function(id, result){\n kue.Job.get(id, function(err, job){\n if (err) return;\n job.remove(function(err){\n if (err) throw err;\n console.log('removed completed job #%d', job.id);\n });\n });\n});\n```\n\nThe events available are the same as mentioned in \"Job Events\", however prefixed with \"job \".\n\n### Delayed Jobs\n\nDelayed jobs may be scheduled to be queued for an arbitrary distance in time by invoking the `.delay(ms)` method, passing the number of milliseconds relative to _now_. Alternatively, you can pass a JavaScript `Date` object with a specific time in the future.\nThis automatically flags the `Job` as \"delayed\".\n\n```js\nvar email = queue.create('email', {\n title: 'Account renewal required'\n , to: 'tj@learnboost.com'\n , template: 'renewal-email'\n}).delay(milliseconds)\n .priority('high')\n .save();\n```\n\nKue will check the delayed jobs with a timer, promoting them if the scheduled delay has been exceeded, defaulting to a check of top 1000 jobs every second.\n\n## Processing Jobs\n\nProcessing jobs is simple with Kue. First create a `Queue` instance much like we do for creating jobs, providing us access to redis etc, then invoke `queue.process()` with the associated type.\nNote that unlike what the name `createQueue` suggests, it currently returns a singleton `Queue` instance. So you can configure and use only a single `Queue` object within your node.js process.\n\nIn the following example we pass the callback `done` to `email`, When an error occurs we invoke `done(err)` to tell Kue something happened, otherwise we invoke `done()` only when the job is complete. If this function responds with an error it will be displayed in the UI and the job will be marked as a failure. The error object passed to done, should be of standard type `Error`.\n\n```js\nvar kue = require('kue')\n , queue = kue.createQueue();\n\nqueue.process('email', function(job, done){\n email(job.data.to, done);\n});\n\nfunction email(address, done) {\n if(!isValidEmail(address)) {\n //done('invalid to address') is possible but discouraged\n return done(new Error('invalid to address'));\n }\n // email send stuff...\n done();\n}\n```\n\nWorkers can also pass job result as the second parameter to done `done(null,result)` to store that in `Job.result` key. `result` is also passed through `complete` event handlers so that job producers can receive it if they like to.\n\n### Processing Concurrency\n\nBy default a call to `queue.process()` will only accept one job at a time for processing. For small tasks like sending emails this is not ideal, so we may specify the maximum active jobs for this type by passing a number:\n\n```js\nqueue.process('email', 20, function(job, done){\n // ...\n});\n```\n\n### Pause Processing\n\nWorkers can temporarily pause and resume their activity. That is, after calling `pause` they will receive no jobs in their process callback until `resume` is called. The `pause` function gracefully shutdowns this worker, and uses the same internal functionality as the `shutdown` method in [Graceful Shutdown](#graceful-shutdown).\n\n```js\nqueue.process('email', function(job, ctx, done){\n ctx.pause( 5000, function(err){\n console.log(\"Worker is paused... \");\n setTimeout( function(){ ctx.resume(); }, 10000 );\n });\n});\n```\n\n**Note** *The `ctx` parameter from Kue `>=0.9.0` is the second argument of the process callback function and `done` is idiomatically always the last*\n\n**Note** *The `pause` method signature is changed from Kue `>=0.9.0` to move the callback function to the last.*\n\n### Updating Progress\n\nFor a \"real\" example, let's say we need to compile a PDF from numerous slides with [node-canvas](https://github.com/Automattic/node-canvas). Our job may consist of the following data, note that in general you should _not_ store large data in the job it-self, it's better to store references like ids, pulling them in while processing.\n\n```js\nqueue.create('slideshow pdf', {\n title: user.name + \"'s slideshow\"\n , slides: [...] // keys to data stored in redis, mongodb, or some other store\n});\n```\n\nWe can access this same arbitrary data within a separate process while processing, via the `job.data` property. In the example we render each slide one-by-one, updating the job's log and progress.\n\n```js\nqueue.process('slideshow pdf', 5, function(job, done){\n var slides = job.data.slides\n , len = slides.length;\n\n function next(i) {\n var slide = slides[i]; // pretend we did a query on this slide id ;)\n job.log('rendering %dx%d slide', slide.width, slide.height);\n renderSlide(slide, function(err){\n if (err) return done(err);\n job.progress(i, len, {nextSlide : i == len ? 'itsdone' : i + 1});\n if (i == len) done()\n else next(i + 1);\n });\n }\n\n next(0);\n});\n```\n\n### Graceful Shutdown\n\n`Queue#shutdown([timeout,] fn)` signals all workers to stop processing after their current active job is done. Workers will wait `timeout` milliseconds for their active job's done to be called or mark the active job `failed` with shutdown error reason. When all workers tell Kue they are stopped `fn` is called.\n\n```javascript\nvar queue = require('kue').createQueue();\n\nprocess.once( 'SIGTERM', function ( sig ) {\n queue.shutdown( 5000, function(err) {\n console.log( 'Kue shutdown: ', err||'' );\n process.exit( 0 );\n });\n});\n```\n\n**Note** *that `shutdown` method signature is changed from Kue `>=0.9.0` to move the callback function to the last.*\n\n## Error Handling\n\nAll errors either in Redis client library or Queue are emitted to the `Queue` object. You should bind to `error` events to prevent uncaught exceptions or debug kue errors.\n\n```javascript\nvar queue = require('kue').createQueue();\n\nqueue.on( 'error', function( err ) {\n console.log( 'Oops... ', err );\n});\n```\n\n### Prevent from Stuck Active Jobs\n\nKue marks a job complete/failed when `done` is called by your worker, so you should use proper error handling to prevent uncaught exceptions in your worker's code and node.js process exiting before in handle jobs get done.\nThis can be achieved in two ways:\n\n1. Wrapping your worker's process function in [Domains](https://nodejs.org/api/domain.html)\n\n ```js\n queue.process('my-error-prone-task', function(job, done){\n var domain = require('domain').create();\n domain.on('error', function(err){\n done(err);\n });\n domain.run(function(){ // your process function\n throw new Error( 'bad things happen' );\n done();\n });\n });\n ```\n **Notice -** Domains are [deprecated](https://nodejs.org/api/documentation.html#documentation_stability_index) from Nodejs with **stability 0** and it's not recommended to use.\n\n This is the softest and best solution, however is not built-in with Kue. Please refer to [this discussion](https://github.com/kriskowal/q/issues/120). You can comment on this feature in the related open Kue [issue](https://github.com/Automattic/kue/pull/403).\n\n You can also use promises to do something like\n\n ```js\n queue.process('my-error-prone-task', function(job, done){\n Promise.method( function(){ // your process function\n throw new Error( 'bad things happen' );\n })().nodeify(done)\n });\n ```\n\n but this won't catch exceptions in your async call stack as domains do.\n\n\n\n2. Binding to `uncaughtException` and gracefully shutting down the Kue, however this is not a recommended error handling idiom in javascript since you are losing the error context.\n\n ```js\n process.once( 'uncaughtException', function(err){\n console.error( 'Something bad happened: ', err );\n queue.shutdown( 1000, function(err2){\n console.error( 'Kue shutdown result: ', err2 || 'OK' );\n process.exit( 0 );\n });\n });\n ```\n\n### Unstable Redis connections\n\nKue currently uses client side job state management and when redis crashes in the middle of that operations, some stuck jobs or index inconsistencies will happen. The consequence is that certain number of jobs will be stuck, and be pulled out by worker only when new jobs are created, if no more new jobs are created, they stuck forever. So we **strongly** suggest that you run watchdog to fix this issue by calling:\n\n```js\nqueue.watchStuckJobs(interval)\n```\n\n`interval` is in milliseconds and defaults to 1000ms\n\nKue will be refactored to fully atomic job state management from version 1.0 and this will happen by lua scripts and/or BRPOPLPUSH combination. You can read more [here](https://github.com/Automattic/kue/issues/130) and [here](https://github.com/Automattic/kue/issues/38).\n\n## Queue Maintenance\n\nQueue object has two type of methods to tell you about the number of jobs in each state\n\n```js\nqueue.inactiveCount( function( err, total ) { // others are activeCount, completeCount, failedCount, delayedCount\n if( total > 100000 ) {\n console.log( 'We need some back pressure here' );\n }\n});\n```\n\nyou can also query on an specific job type:\n\n```js\nqueue.failedCount( 'my-critical-job', function( err, total ) {\n if( total > 10000 ) {\n console.log( 'This is tOoOo bad' );\n }\n});\n```\n\nand iterating over job ids\n\n```js\nqueue.inactive( function( err, ids ) { // others are active, complete, failed, delayed\n // you may want to fetch each id to get the Job object out of it...\n});\n```\n\nhowever the second one doesn't scale to large deployments, there you can use more specific `Job` static methods:\n\n```js\nkue.Job.rangeByState( 'failed', 0, n, 'asc', function( err, jobs ) {\n // you have an array of maximum n Job objects here\n});\n```\nor\n\n```js\nkue.Job.rangeByType( 'my-job-type', 'failed', 0, n, 'asc', function( err, jobs ) {\n // you have an array of maximum n Job objects here\n});\n```\n\n**Note** *that the last two methods are subject to change in later Kue versions.*\n\n\n### Programmatic Job Management\n\nIf you did none of above in [Error Handling](#error-handling) section or your process lost active jobs in any way, you can recover from them when your process is restarted. A blind logic would be to re-queue all stuck jobs:\n\n```js\nqueue.active( function( err, ids ) {\n ids.forEach( function( id ) {\n kue.Job.get( id, function( err, job ) {\n // Your application should check if job is a stuck one\n job.inactive();\n });\n });\n});\n```\n\n**Note** *in a clustered deployment your application should be aware not to involve a job that is valid, currently inprocess by other workers.*\n\n### Job Cleanup\n\nJobs data and search indexes eat up redis memory space, so you will need some job-keeping process in real world deployments. Your first chance is using automatic job removal on completion.\n\n```javascript\nqueue.create( ... ).removeOnComplete( true ).save()\n```\n\nBut if you eventually/temporally need completed job data, you can setup an on-demand job removal script like below to remove top `n` completed jobs:\n\n```js\nkue.Job.rangeByState( 'complete', 0, n, 'asc', function( err, jobs ) {\n jobs.forEach( function( job ) {\n job.remove( function(){\n console.log( 'removed ', job.id );\n });\n });\n});\n```\n\n**Note** *that you should provide enough time for `.remove` calls on each job object to complete before your process exits, or job indexes will leak*\n\n\n## Redis Connection Settings\n\nBy default, Kue will connect to Redis using the client default settings (port defaults to `6379`, host defaults to `127.0.0.1`, prefix defaults to `q`). `Queue#createQueue(options)` accepts redis connection options in `options.redis` key.\n\n```javascript\nvar kue = require('kue');\nvar q = kue.createQueue({\n prefix: 'q',\n redis: {\n port: 1234,\n host: '10.0.50.20',\n auth: 'password',\n db: 3, // if provided select a non-default redis db\n options: {\n // see https://github.com/mranney/node_redis#rediscreateclient\n }\n }\n});\n```\n\n`prefix` controls the key names used in Redis. By default, this is simply `q`. Prefix generally shouldn't be changed unless you need to use one Redis instance for multiple apps. It can also be useful for providing an isolated testbed across your main application.\n\nYou can also specify the connection information as a URL string.\n\n```js\nvar q = kue.createQueue({\n redis: 'redis://example.com:1234?redis_option=value&redis_option=value'\n});\n```\n\n#### Connecting using Unix Domain Sockets\n\nSince [node_redis](https://github.com/mranney/node_redis) supports Unix Domain Sockets, you can also tell Kue to do so. See [unix-domain-socket](https://github.com/mranney/node_redis#unix-domain-socket) for your redis server configuration.\n\n```javascript\nvar kue = require('kue');\nvar q = kue.createQueue({\n prefix: 'q',\n redis: {\n socket: '/data/sockets/redis.sock',\n auth: 'password',\n options: {\n // see https://github.com/mranney/node_redis#rediscreateclient\n }\n }\n});\n```\n\n#### Replacing Redis Client Module\n\nAny node.js redis client library that conforms (or when adapted) to [node_redis](https://github.com/mranney/node_redis) API can be injected into Kue. You should only provide a `createClientFactory` function as a redis connection factory instead of providing node_redis connection options.\n\nBelow is a sample code to enable [redis-sentinel](https://github.com/ortoo/node-redis-sentinel) to connect to [Redis Sentinel](http://redis.io/topics/sentinel) for automatic master/slave failover.\n\n```javascript\nvar kue = require('kue');\nvar Sentinel = require('redis-sentinel');\nvar endpoints = [\n {host: '192.168.1.10', port: 6379},\n {host: '192.168.1.11', port: 6379}\n];\nvar opts = options || {}; // Standard node_redis client options\nvar masterName = 'mymaster';\nvar sentinel = Sentinel.Sentinel(endpoints);\n\nvar q = kue.createQueue({\n redis: {\n createClientFactory: function(){\n return sentinel.createClient(masterName, opts);\n }\n }\n});\n```\n\n**Note** *that all `<0.8.x` client codes should be refactored to pass redis options to `Queue#createQueue` instead of monkey patched style overriding of `redis#createClient` or they will be broken from Kue `0.8.x`.*\n\n#### Using ioredis client with cluster support\n\n```javascript\n\nvar Redis = require('ioredis');\nvar kue = require('kue');\n\n// using https://github.com/72squared/vagrant-redis-cluster\n\nvar queue = kue.createQueue({\n redis: {\n createClientFactory: function () {\n return new Redis.Cluster([{\n port: 7000\n }, {\n port: 7001\n }]);\n }\n }\n });\n```\n\n## User-Interface\n\nThe UI is a small [Express](https://github.com/strongloop/express) application.\nA script is provided in `bin/` for running the interface as a standalone application\nwith default settings. You may pass in options for the port, redis-url, and prefix. For example:\n\n```\nnode_modules/kue/bin/kue-dashboard -p 3050 -r redis://127.0.0.1:3000 -q prefix\n```\n\nYou can fire it up from within another application too:\n\n\n```js\nvar kue = require('kue');\nkue.createQueue(...);\nkue.app.listen(3000);\n```\n\nThe title defaults to \"Kue\", to alter this invoke:\n\n```js\nkue.app.set('title', 'My Application');\n```\n\n**Note** *that if you are using non-default Kue options, `kue.createQueue(...)` must be called before accessing `kue.app`.*\n\n### Third-party interfaces\n\nYou can also use [Kue-UI](https://github.com/StreetHub/kue-ui) web interface contributed by [Arnaud Bénard](https://github.com/arnaudbenard)\n\n\n## JSON API\n\nAlong with the UI Kue also exposes a JSON API, which is utilized by the UI.\n\n### GET /job/search?q=\n\nQuery jobs, for example \"GET /job/search?q=avi video\":\n\n```js\n[\"5\", \"7\", \"10\"]\n```\n\nBy default kue indexes the whole Job data object for searching, but this can be customized via calling `Job#searchKeys` to tell kue which keys on Job data to create index for:\n\n```javascript\nvar kue = require('kue');\nqueue = kue.createQueue();\nqueue.create('email', {\n title: 'welcome email for tj'\n , to: 'tj@learnboost.com'\n , template: 'welcome-email'\n}).searchKeys( ['to', 'title'] ).save();\n```\n\nSearch feature is turned off by default from Kue `>=0.9.0`. Read more about this [here](https://github.com/Automattic/kue/issues/412). You should enable search indexes and add [reds](https://www.npmjs.com/package/reds) in your dependencies if you need to:\n\n```javascript\nvar kue = require('kue');\nq = kue.createQueue({\n disableSearch: false\n});\n```\n\n```\nnpm install reds --save\n```\n\n### GET /stats\n\nCurrently responds with state counts, and worker activity time in milliseconds:\n\n```js\n{\"inactiveCount\":4,\"completeCount\":69,\"activeCount\":2,\"failedCount\":0,\"workTime\":20892}\n```\n\n### GET /job/:id\n\nGet a job by `:id`:\n\n```js\n{\"id\":\"3\",\"type\":\"email\",\"data\":{\"title\":\"welcome email for tj\",\"to\":\"tj@learnboost.com\",\"template\":\"welcome-email\"},\"priority\":-10,\"progress\":\"100\",\"state\":\"complete\",\"attempts\":null,\"created_at\":\"1309973155248\",\"updated_at\":\"1309973155248\",\"duration\":\"15002\"}\n```\n\n### GET /job/:id/log\n\nGet job `:id`'s log:\n\n```js\n['foo', 'bar', 'baz']\n```\n\n### GET /jobs/:from..:to/:order?\n\nGet jobs with the specified range `:from` to `:to`, for example \"/jobs/0..2\", where `:order` may be \"asc\" or \"desc\":\n\n```js\n[{\"id\":\"12\",\"type\":\"email\",\"data\":{\"title\":\"welcome email for tj\",\"to\":\"tj@learnboost.com\",\"template\":\"welcome-email\"},\"priority\":-10,\"progress\":0,\"state\":\"active\",\"attempts\":null,\"created_at\":\"1309973299293\",\"updated_at\":\"1309973299293\"},{\"id\":\"130\",\"type\":\"email\",\"data\":{\"title\":\"welcome email for tj\",\"to\":\"tj@learnboost.com\",\"template\":\"welcome-email\"},\"priority\":-10,\"progress\":0,\"state\":\"active\",\"attempts\":null,\"created_at\":\"1309975157291\",\"updated_at\":\"1309975157291\"}]\n```\n\n### GET /jobs/:state/:from..:to/:order?\n\nSame as above, restricting by `:state` which is one of:\n\n - active\n - inactive\n - failed\n - complete\n\n### GET /jobs/:type/:state/:from..:to/:order?\n\nSame as above, however restricted to `:type` and `:state`.\n\n### DELETE /job/:id\n\nDelete job `:id`:\n\n $ curl -X DELETE http://local:3000/job/2\n {\"message\":\"job 2 removed\"}\n\n### POST /job\n\nCreate a job:\n\n $ curl -H \"Content-Type: application/json\" -X POST -d \\\n '{\n \"type\": \"email\",\n \"data\": {\n \"title\": \"welcome email for tj\",\n \"to\": \"tj@learnboost.com\",\n \"template\": \"welcome-email\"\n },\n \"options\" : {\n \"attempts\": 5,\n \"priority\": \"high\"\n }\n }' http://localhost:3000/job\n {\"message\": \"job created\", \"id\": 3}\n\nYou can create multiple jobs at once by passing an array. In this case, the response will be an array too, preserving the order:\n\n $ curl -H \"Content-Type: application/json\" -X POST -d \\\n '[{\n \"type\": \"email\",\n \"data\": {\n \"title\": \"welcome email for tj\",\n \"to\": \"tj@learnboost.com\",\n \"template\": \"welcome-email\"\n },\n \"options\" : {\n \"attempts\": 5,\n \"priority\": \"high\"\n }\n },\n {\n \"type\": \"email\",\n \"data\": {\n \"title\": \"followup email for tj\",\n \"to\": \"tj@learnboost.com\",\n \"template\": \"followup-email\"\n },\n \"options\" : {\n \"delay\": 86400,\n \"attempts\": 5,\n \"priority\": \"high\"\n }\n }]' http://localhost:3000/job\n [\n {\"message\": \"job created\", \"id\": 4},\n {\"message\": \"job created\", \"id\": 5}\n ]\n\nNote: when inserting multiple jobs in bulk, if one insertion fails Kue will keep processing the remaining jobs in order. The response array will contain the ids of the jobs added successfully, and any failed element will be an object describing the error: `{\"error\": \"error reason\"}`.\n\n\n## Parallel Processing With Cluster\n\nThe example below shows how you may use [Cluster](http://nodejs.org/api/cluster.html) to spread the job processing load across CPUs. Please see [Cluster module's documentation](http://nodejs.org/api/cluster.html) for more detailed examples on using it.\n\nWhen cluster `.isMaster` the file is being executed in context of the master process, in which case you may perform tasks that you only want once, such as starting the web app bundled with Kue. The logic in the `else` block is executed _per worker_.\n\n```js\nvar kue = require('kue')\n , cluster = require('cluster')\n , queue = kue.createQueue();\n\nvar clusterWorkerSize = require('os').cpus().length;\n\nif (cluster.isMaster) {\n kue.app.listen(3000);\n for (var i = 0; i < clusterWorkerSize; i++) {\n cluster.fork();\n }\n} else {\n queue.process('email', 10, function(job, done){\n var pending = 5\n , total = pending;\n\n var interval = setInterval(function(){\n job.log('sending!');\n job.progress(total - pending, total);\n --pending || done();\n pending || clearInterval(interval);\n }, 1000);\n });\n}\n```\n\nThis will create an `email` job processor (worker) per each of your machine CPU cores, with each you can handle 10 concurrent email jobs, leading to total `10 * N` concurrent email jobs processed in your `N` core machine.\n\nNow when you visit Kue's UI in the browser you'll see that jobs are being processed roughly `N` times faster! (if you have `N` cores).\n\n## Securing Kue\n\nThrough the use of app mounting you may customize the web application, enabling TLS, or adding additional middleware like `basic-auth-connect`.\n\n```bash\n$ npm install --save basic-auth-connect\n```\n\n```js\nvar basicAuth = require('basic-auth-connect');\nvar app = express.createServer({ ... tls options ... });\napp.use(basicAuth('foo', 'bar'));\napp.use(kue.app);\napp.listen(3000);\n```\n\n## Testing\n\nEnable test mode to push all jobs into a `jobs` array. Make assertions against\nthe jobs in that array to ensure code under test is correctly enqueuing jobs.\n\n```js\nqueue = require('kue').createQueue();\n\nbefore(function() {\n queue.testMode.enter();\n});\n\nafterEach(function() {\n queue.testMode.clear();\n});\n\nafter(function() {\n queue.testMode.exit()\n});\n\nit('does something cool', function() {\n queue.createJob('myJob', { foo: 'bar' }).save();\n queue.createJob('anotherJob', { baz: 'bip' }).save();\n expect(queue.testMode.jobs.length).to.equal(2);\n expect(queue.testMode.jobs[0].type).to.equal('myJob');\n expect(queue.testMode.jobs[0].data).to.eql({ foo: 'bar' });\n});\n```\n\n**IMPORTANT:** By default jobs aren't processed when created during test mode. You can enable job processing by passing true to testMode.enter\n\n```js\nbefore(function() {\n queue.testMode.enter(true);\n});\n```\n\n\n## Screencasts\n\n - [Introduction](http://www.screenr.com/oyNs) to Kue\n - API [walkthrough](https://vimeo.com/26963384) to Kue\n\n## Contributing\n\n**We love contributions!**\n\nWhen contributing, follow the simple rules:\n\n* Don't violate [DRY](http://programmer.97things.oreilly.com/wiki/index.php/Don%27t_Repeat_Yourself) principles.\n* [Boy Scout Rule](http://programmer.97things.oreilly.com/wiki/index.php/The_Boy_Scout_Rule) needs to have been applied.\n* Your code should look like all the other code – this project should look like it was written by one person, always.\n* If you want to propose something – just create an issue and describe your question with as much description as you can.\n* If you think you have some general improvement, consider creating a pull request with it.\n* If you add new code, it should be covered by tests. No tests – no code.\n* If you add a new feature, don't forget to update the documentation for it.\n* If you find a bug (or at least you think it is a bug), create an issue with the library version and test case that we can run and see what are you talking about, or at least full steps by which we can reproduce it.\n\n## License\n\n(The MIT License)\n\nCopyright (c) 2011 LearnBoost <tj@learnboost.com>\n\nPermission is hereby granted, free of charge, to any person obtaining\na copy of this software and associated documentation files (the\n'Software'), to deal in the Software without restriction, including\nwithout limitation the rights to use, copy, modify, merge, publish,\ndistribute, sublicense, and/or sell copies of the Software, and to\npermit persons to whom the Software is furnished to do so, subject to\nthe following conditions:\n\nThe above copyright notice and this permission notice shall be\nincluded in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,\nEXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\nMERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\nIN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY\nCLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,\nTORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\nSOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" }, { "path": "bin/kue-dashboard", "content": "#!/usr/bin/env node\nvar kue = require('kue');\nvar argv = require('yargs')\n\t.usage('Usage: $0 [options]')\n\t.example('$0 -p 3050 -r redis://10.0.0.4:6379 -q q')\n\t.describe('r', 'Redis url')\n\t.describe('p', 'Dashboard port')\n\t.describe('q', 'Prefix to use')\n\t.default('p', 3000)\n\t.default('r', 'redis://127.0.0.1:6379')\n\t.default('q', 'q')\n\t.help('h')\n .alias('h', 'help')\n .argv\n;\n\nkue.createQueue({\n redis: argv.r,\n prefix: argv.q\n});\n\n\nkue.app.listen(argv.p);\nconsole.log(\"Running on http://127.0.0.1:\" + argv.p);\n" }, { "path": "examples/delayed.js", "content": "var kue = require( '../' );\n\n// create our job queue\n\nvar jobs = kue.createQueue();\n\n// one minute\n\nvar minute = 60000;\n\nvar email = jobs.create( 'email', {\n title: 'Account renewal required', to: 'tj@learnboost.com', template: 'renewal-email'\n} ).delay( minute )\n .priority( 'high' )\n .save();\n\n\nemail.on( 'promotion', function () {\n console.log( 'renewal job promoted' );\n} );\n\nemail.on( 'complete', function () {\n console.log( 'renewal job completed' );\n} );\n\njobs.create( 'email', {\n title: 'Account expired', to: 'tj@learnboost.com', template: 'expired-email'\n} ).delay( minute * 10 )\n .priority( 'high' )\n .save();\n\njobs.promote();\n\njobs.process( 'email', 10, function ( job, done ) {\n setTimeout( function () {\n done();\n }, Math.random() * 5000 );\n} );\n\n// start the UI\nkue.app.listen( 3000 );\nconsole.log( 'UI started on port 3000' );" }, { "path": "examples/events.js", "content": "var kue = require( '../' );\n\n// create our job queue\n\nvar jobs = kue.createQueue();\n\n// start redis with $ redis-server\n\n// create some jobs at random,\n// usually you would create these\n// in your http processes upon\n// user input etc.\n\nfunction create() {\n var name = [ 'tobi', 'loki', 'jane', 'manny' ][ Math.random() * 4 | 0 ];\n var job = jobs.create( 'video conversion', {\n title: 'converting ' + name + '\\'s to avi', user: 1, frames: 200\n } );\n\n job.on( 'complete', function () {\n console.log( \" Job complete\" );\n } ).on( 'failed', function () {\n console.log( \" Job failed\" );\n } ).on( 'progress', function ( progress ) {\n process.stdout.write( '\\r job #' + job.id + ' ' + progress + '% complete' );\n } );\n\n job.save();\n\n setTimeout( create, Math.random() * 2000 | 0 );\n}\n\ncreate();\n\n// process video conversion jobs, 1 at a time.\n\njobs.process( 'video conversion', 1, function ( job, done ) {\n var frames = job.data.frames;\n\n function next( i ) {\n // pretend we are doing some work\n convertFrame( i, function ( err ) {\n if ( err ) return done( err );\n // report progress, i/frames complete\n job.progress( i, frames );\n if ( i >= frames ) done()\n else next( i + Math.random() * 10 );\n } );\n }\n\n next( 0 );\n} );\n\nfunction convertFrame( i, fn ) {\n setTimeout( fn, Math.random() * 50 );\n}\n\n// start the UI\nkue.app.listen( 3000 );\nconsole.log( 'UI started on port 3000' );\n" }, { "path": "examples/many.js", "content": "var kue = require( '../' )\n , express = require( 'express' );\n\n// create our job queue\n\nvar jobs = kue.createQueue();\n\nfunction create() {\n var name = [ 'tobi', 'loki', 'jane', 'manny' ][ Math.random() * 4 | 0 ];\n jobs.create( 'video conversion', {\n title: 'converting ' + name + '\\'s to avi', user: 1, frames: 200\n } ).save();\n setTimeout( create, Math.random() * 3000 | 0 );\n}\n\ncreate();\n\nfunction create2() {\n var name = [ 'tobi', 'loki', 'jane', 'manny' ][ Math.random() * 4 | 0 ];\n jobs.create( 'email', {\n title: 'emailing ' + name + '', body: 'hello'\n } ).save();\n setTimeout( create2, Math.random() * 1000 | 0 );\n}\n\ncreate2();\n\n// process video conversion jobs, 2 at a time.\n\njobs.process( 'video conversion', 2, function ( job, done ) {\n console.log( 'video' );\n setTimeout( done, Math.random() * 5000 );\n} );\n\n// process 10 emails at a time\n\njobs.process( 'email', 10, function ( job, done ) {\n console.log( 'email' );\n setTimeout( done, Math.random() * 2000 );\n} );\n\n// start the UI\nkue.app.listen( 3000 );\nconsole.log( 'UI started on port 3000' );\n" }, { "path": "examples/shutdown.js", "content": "var kue = require( '../' )\n\nvar jobs = kue.createQueue()\n\n\nfunction generateJobs() {\n for ( var i = 0; i < 12; i++ ) {\n console.log( 'Creating Job #' + i );\n jobs.create( 'long render', {\n title: 'rendering frame #' + i\n } ).save();\n }\n}\n\n\njobs.process( 'long render', 4, function ( job, done ) {\n console.log( 'Starting ' + job.data.title );\n setTimeout( function () {\n console.log( 'Finished ' + job.data.title );\n done();\n }, 3000 );\n} )\n\n\ngenerateJobs();\n\nsetTimeout( function () {\n console.log( '[ Shutting down when all jobs finish... ]' );\n jobs.shutdown( function ( err ) {\n console.log( '[ All jobs finished. Kue is shut down. ]' );\n process.exit( 0 );\n } )\n}, 4200 )\n\n" }, { "path": "examples/stale.js", "content": "var kue = require( '../' )\n , express = require( 'express' );\n\n// create our job queue\n\nvar jobs = kue.createQueue()\n , Job = kue.Job;\n\n// start redis with $ redis-server\n\n// create some jobs at random,\n// usually you would create these\n// in your http processes upon\n// user input etc.\n\nfunction create() {\n var name = [ 'tobi', 'loki', 'jane', 'manny' ][ Math.random() * 4 | 0 ];\n console.log( '- creating job for %s', name );\n jobs.create( 'video conversion', {\n title: 'converting ' + name + '\\'s to avi', user: 1, frames: 200\n } ).save();\n setTimeout( create, Math.random() * 3000 | 0 );\n}\n\ncreate();\n\n// process video conversion jobs, 3 at a time.\n\njobs.process( 'video conversion', 3, function ( job, done ) {\n var frames = job.data.frames;\n console.log( \"job process %d\", job.id );\n function next( i ) {\n // pretend we are doing some work\n convertFrame( i, function ( err ) {\n if ( err ) return done( err );\n // report progress, i/frames complete\n job.progress( i, frames );\n if ( i == frames ) done()\n else next( i + 5 );\n } );\n }\n\n next( 0 );\n} );\n\nfunction convertFrame( i, fn ) {\n setTimeout( fn, Math.random() * 100 );\n}\n\n// remove stale jobs\njobs.on( 'job complete', function ( id ) {\n Job.get( id, function ( err, job ) {\n if ( err ) return;\n job.remove( function ( err ) {\n if ( err ) throw err;\n console.log( 'removed completed job #%d', job.id );\n } );\n } );\n} );\n\n// start the UI\nvar app = express.createServer();\napp.use( kue.app );\napp.listen( 3000 );\nconsole.log( 'UI started on port 3000' );" }, { "path": "examples/video.js", "content": "var kue = require( '../' )\n , express = require( 'express' );\n\n// create our job queue\n\nvar jobs = kue.createQueue();\n\n// start redis with $ redis-server\n\n// create some jobs at random,\n// usually you would create these\n// in your http processes upon\n// user input etc.\n\nfunction create() {\n var name = [ 'tobi', 'loki', 'jane', 'manny' ][ Math.random() * 4 | 0 ];\n console.log( '- creating job for %s', name );\n jobs.create( 'video conversion', {\n title: 'converting ' + name + '\\'s to avi', user: 1, frames: 200\n } ).save();\n setTimeout( create, Math.random() * 3000 | 0 );\n}\n\ncreate();\n\n// process video conversion jobs, 3 at a time.\n\njobs.process( 'video conversion', 3, function ( job, done ) {\n var frames = job.data.frames;\n console.log( \"job process %d\", job.id );\n function next( i ) {\n // pretend we are doing some work\n convertFrame( i, function ( err ) {\n if ( err ) return done( err );\n // report progress, i/frames complete\n job.progress( i, frames );\n if ( i == frames ) done()\n else next( i + 1 );\n } );\n }\n\n next( 0 );\n} );\n\nfunction convertFrame( i, fn ) {\n setTimeout( fn, Math.random() * 100 );\n}\n\n// start the UI\nvar app = express.createServer();\napp.use( express.basicAuth( 'foo', 'bar' ) );\napp.use( kue.app );\napp.listen( 3000 );\nconsole.log( 'UI started on port 3000' );" }, { "path": "index.js", "content": "module.exports = require('./lib/kue');" }, { "path": "lib/http/index.js", "content": "/*!\n * q - http\n * Copyright (c) 2011 LearnBoost \n * MIT Licensed\n */\n\n/**\n * Module dependencies.\n */\n\nvar express = require('express');\n\n// setup\n\nvar app = express()\n , bodyParser = require('body-parser')\n , provides = require('./middleware/provides')\n , stylus = require('stylus')\n , routes = require('./routes')\n , pug = require('pug')\n , json = require('./routes/json')\n , util = require('util')\n , nib = require('nib');\n\n// expose the app\n\nmodule.exports = app;\n\n// stylus config\n\nfunction compile( str, path ) {\n return stylus(str)\n .set('filename', path)\n .use(nib());\n}\n\n// config\n\napp.set('view options', { doctype: 'html' });\napp.set('view engine', 'pug');\napp.engine('pug', pug.renderFile);\napp.set('views', __dirname + '/views');\napp.set('title', 'Kue');\napp.locals = { inspect: util.inspect };\n\n// middlewares\n\napp.use(stylus.middleware({ src: __dirname + '/public', compile: compile }));\napp.use(express.static(__dirname + '/public'));\n\n// JSON api\n\napp.get('/stats', provides('json'), json.stats);\napp.get('/job/search', provides('json'), json.search);\napp.get('/jobs/:from..:to/:order?', provides('json'), json.jobRange);\napp.get('/jobs/:type/:state/:from..:to/:order?', provides('json'), json.jobTypeRange);\napp.get('/jobs/:type/:state/stats', provides('json'), json.jobTypeStateStats);\napp.get('/jobs/:state/:from..:to/:order?', provides('json'), json.jobStateRange);\napp.get('/job/types', provides('json'), json.types);\napp.get('/job/:id', provides('json'), json.job);\napp.get('/job/:id/log', provides('json'), json.log);\napp.put('/job/:id/state/:state', provides('json'), json.updateState);\napp.put('/job/:id/priority/:priority', provides('json'), json.updatePriority);\napp.delete('/job/:id', provides('json'), json.remove);\napp.post('/job', provides('json'), bodyParser.json(), json.createJob);\napp.get('/inactive/:id', provides('json'), json.inactive);\n\n// routes\n\napp.get('/', routes.jobs('active'));\n\napp.get('/active', routes.jobs('active'));\napp.get('/inactive', routes.jobs('inactive'));\napp.get('/failed', routes.jobs('failed'));\napp.get('/complete', routes.jobs('complete'));\napp.get('/delayed', routes.jobs('delayed'));\n" }, { "path": "lib/http/middleware/provides.js", "content": "/**\n * Specify that the route provides `type`.\n *\n * @param {String} type\n * @return {Function}\n * @api private\n */\n\nmodule.exports = function( type ) {\n return function( req, res, next ) {\n if( req.accepts(type) ) return next();\n next('route');\n }\n};" }, { "path": "lib/http/public/javascripts/caustic.js", "content": "/*!\n * EventEmitter\n * Copyright (c) 2011 TJ Holowaychuk \n * MIT Licensed\n */\n\n/**\n * EventEmitter.\n */\n\nfunction EventEmitter() {\n this.callbacks = {};\n}\n\n/**\n * Listen on the given `event` with `fn`.\n *\n * @param {String} event\n * @param {Function} fn\n */\n\nEventEmitter.prototype.on = function (event, fn) {\n (this.callbacks[event] = this.callbacks[event] || [])\n .push(fn);\n return this;\n};\n\n/**\n * Emit `event` with the given args.\n *\n * @param {String} event\n * @param {Mixed} ...\n */\n\nEventEmitter.prototype.emit = function (event) {\n var args = Array.prototype.slice.call(arguments, 1)\n , callbacks = this.callbacks[event];\n\n if (callbacks) {\n for (var i = 0, len = callbacks.length; i < len; ++i) {\n callbacks[i].apply(this, args)\n }\n }\n\n return this;\n};\n\n/*!\n * caustic\n * Copyright(c) 2011 TJ Holowaychuk \n * MIT Licensed\n */\n\n// TODO: `make caustic.js` should wrap in an anonymous function\n// TODO: `make caustic.min.js`\n\n// TODO: compile sub-views such as User etc based on the given\n// html, as there's no need to keep traversing each time.\n\n/**\n * Convert callback `fn` to a function when a string is given.\n *\n * @param {Type} name\n * @return {Type}\n * @api private\n */\n\nfunction callback(fn) {\n return 'string' == typeof fn\n ? function (obj) {\n return obj[fn]();\n }\n : fn;\n}\n\n/**\n * Initialize a new view with the given `name`\n * or string of html. When a `name` is given an element\n * with the id `name + \"-template\"` will be used.\n *\n * Examples:\n *\n * var user = new View('user');\n * var list = new View('
');\n *\n * @param {String} name\n * @api public\n */\n\nfunction View(name) {\n if (!(this instanceof View)) return new View(name);\n EventEmitter.call(this);\n var html;\n if (~name.indexOf('<')) html = name;\n else html = $('#' + name + '-template').html();\n this.el = $(html);\n this.visit(this.el);\n}\n\n/**\n * Inherit from `EventEmitter.prototype`.\n */\n\nView.prototype.__proto__ = EventEmitter.prototype;\n\n/**\n * Visit `el`.\n *\n * @param {jQuery} el\n * @param {Boolean} ignore\n * @api private\n */\n\nView.prototype.visit = function (el, ignore) {\n var self = this\n , type = el.get(0).nodeName\n , classes = el.attr('class').split(/ +/)\n , method = 'visit' + type;\n\n if (this[method] && !ignore) this[method](el, classes[0]);\n\n el.children().each(function (i, el) {\n self.visit($(el));\n });\n};\n\n/**\n * Visit INPUT tag.\n *\n * @param {jQuery} el\n * @api public\n */\n\nView.prototype.visitINPUT = function (el) {\n var self = this\n , name = el.attr('name')\n , type = el.attr('type');\n\n switch (type) {\n case 'text':\n this[name] = function (val) {\n if (0 == arguments.length) return el.val();\n el.val(val);\n return this;\n }\n\n this[name].isEmpty = function () {\n return '' == el.val();\n };\n\n this[name].clear = function () {\n el.val('');\n return self;\n };\n break;\n case 'checkbox':\n this[name] = function (val) {\n if (0 == arguments.length) return el.attr('checked');\n switch (typeof val) {\n case 'function':\n el.change(function (e) {\n val.call(self, el.attr('checked'), e);\n });\n break;\n default:\n el.attr('checked', val\n ? 'checked'\n : val);\n }\n return this;\n }\n break;\n }\n};\n\n/**\n * Visit FORM.\n *\n * @param {jQuery} el\n * @api private\n */\n\nView.prototype.visitFORM = function (el, name) {\n var self = this;\n this.submit = function (val) {\n switch (typeof val) {\n case 'function':\n el.submit(function (e) {\n val.call(self, e, el);\n return false;\n });\n break;\n }\n }\n};\n\n/**\n * Visit A tag.\n *\n * @param {jQuery} el\n * @api private\n */\n\nView.prototype.visitA = function (el, name) {\n var self = this;\n\n el.click(function (e) {\n self.emit(name, e, el);\n });\n\n this[name] = function (fn) {\n el.click(function (e) {\n fn.call(self, e, el);\n return false;\n });\n return this;\n }\n};\n\n/**\n * Visit P, TD, SPAN, or DIV tag.\n *\n * @param {jQuery} el\n * @api private\n */\n\nView.prototype.visitP =\n View.prototype.visitTD =\n View.prototype.visitSPAN =\n View.prototype.visitDIV = function (el, name) {\n var self = this;\n this[name] = function (val) {\n if (0 == arguments.length) return el;\n el.empty().append(val.el || val);\n return this;\n };\n };\n\n/**\n * Visit UL tag.\n *\n * @param {jQuery} el\n * @api private\n */\n\nView.prototype.visitUL = function (el, name) {\n var self = this;\n this.children = [];\n\n this[name] = el;\n\n // TODO: move these out\n\n /**\n * Add `val` to this list.\n *\n * @param {String|jQuery|View} val\n * @return {View} for chaining\n * @api public\n */\n\n el.add = function (val) {\n var li = $('
  • ');\n self.children.push(val);\n el.append(li.append(val.el || val));\n return this;\n };\n\n /**\n * Return the list item `View`s as an array.\n *\n * @return {Array}\n * @api public\n */\n\n el.items = function () {\n return self.children;\n };\n\n /**\n * Iterate the list `View`s, calling `fn(item, i)`.\n *\n * @param {Function} fn\n * @return {View} for chaining\n * @api public\n */\n\n el.each = function (fn) {\n for (var i = 0, len = self.children.length; i < len; ++i) {\n fn(self.children[i], i);\n }\n return this;\n };\n\n /**\n * Map the list `View`s, calling `fn(item, i)`.\n *\n * @param {String|function} fn\n * @return {Array}\n * @api public\n */\n\n el.map = function (fn) {\n var ret = []\n , fn = callback(fn);\n\n for (var i = 0, len = self.children.length; i < len; ++i) {\n ret.push(fn(self.children[i], i));\n }\n\n return ret;\n };\n};\n\n/**\n * Visit TABLE.\n *\n * @param {jQuery} el\n * @api private\n */\n\nView.prototype.visitTABLE = function (el, name) {\n this[name] = el;\n\n this[name].add = function (val) {\n this.append(val.el || val);\n };\n};\n\n/**\n * Visit CANVAS.\n *\n * @param {jQuery} el\n * @api private\n */\n\nView.prototype.visitCANVAS = function (el, name) {\n this[name] = el.get(0);\n};\n\n/**\n * Visit H1-H5 tags.\n *\n * @param {jQuery} el\n * @api private\n */\n\nView.prototype.visitH1 =\n View.prototype.visitH2 =\n View.prototype.visitH3 =\n View.prototype.visitH4 =\n View.prototype.visitH5 = function (el, name) {\n var self = this;\n this[name] = function (val) {\n if (0 == arguments.length) return el.text();\n el.text(val.el || val);\n return this;\n };\n };\n\n/**\n * Remove the view from the DOM.\n *\n * @return {View}\n * @api public\n */\n\nView.prototype.remove = function () {\n var parent = this.el.parent()\n , type = parent.get(0).nodeName;\n if ('LI' == type) parent.remove();\n else this.el.remove();\n return this;\n};\n\n/**\n * Append this view's element to `val`.\n *\n * @param {String|jQuery} val\n * @return {View}\n * @api public\n */\n\nView.prototype.appendTo = function (val) {\n this.el.appendTo(val.el || val);\n return this;\n};\n" }, { "path": "lib/http/public/javascripts/job.js", "content": "/*!\n * kue - Job\n * Copyright (c) 2011 LearnBoost \n * MIT Licensed\n */\n\n/**\n * Initialize a new `Job` with the given `data`.\n *\n * @param {Object} obj\n */\n\nfunction Job(data) {\n this.update(data);\n}\n\n/**\n * Show progress indicator.\n *\n * @param {Boolean} val\n * @return {Job} for chaining\n */\n\nJob.prototype.showProgress = function (val) {\n this._showProgress = val;\n return this;\n};\n\n/**\n * Show error message when `val` is true.\n *\n * @param {Boolean} val\n * @return {Job} for chaining\n */\n\nJob.prototype.showErrorMessage = function (val) {\n this._showError = val;\n return this;\n};\n\n/**\n * Remove the job and callback `fn()`.\n *\n * @param {Function} fn\n */\n\nJob.prototype.remove = function (fn) {\n request('DELETE', './job/' + this.id, fn);\n return this;\n};\n\n/**\n * Restart the job and callback `fn()`.\n *\n * @param {Function} fn\n */\n\nJob.prototype.restart = function (fn) {\n request('GET', './inactive/' + this.id, fn);\n return this;\n};\n\n/**\n * Update the job with the given `data`.\n *\n * @param {Object} data\n * @return {Job} for chaining\n */\n\nJob.prototype.update = function (data) {\n for (var key in data) this[key] = data[key];\n if (!this.data) this.data = {};\n return this;\n};\n\n/**\n * Render the job, returning an oQuery object.\n *\n * @param {Boolean} isNew\n * @return {oQuery}\n */\n\nJob.prototype.render = function (isNew) {\n var self = this\n , id = this.id\n , view = this.view\n , keys = Object.keys(this.data).sort()\n , data;\n\n if (isNew) {\n view = this.view = View('job');\n\n view.remove(function () {\n this.remove();\n self.remove();\n });\n\n view.restart(function () {\n this.restart();\n self.restart();\n });\n\n var canvas = view.progress\n , ctx = this.ctx = canvas.getContext('2d')\n , progress = new Progress;\n\n progress.size(canvas.width);\n this._progress = progress;\n\n // initially hide the logs\n view.log.hide();\n\n // populate title and id\n view.el.attr('id', 'job-' + id);\n view.id(id);\n\n // show job data\n for (var i = 0, len = keys.length; i < len; ++i) {\n data = this.data[keys[i]];\n if ('object' == typeof data) data = JSON.stringify(data);\n var row = View('row');\n// row.title(keys[i] + ':').value(data);\n row.title(keys[i] + ':').value($('

    ').text(data).html());\n view.data.add(row);\n }\n\n // alter state\n view.state(this.state);\n view.state().click(function () {\n var select = o('', options(states, self.state));\n o(this).replaceWith(select);\n select.change(function () {\n self.updateState(select.val());\n });\n return false;\n });\n\n // alter priority\n view.priority(priority(this));\n view.priority().click(function () {\n var select = o('', options(priorities, self.priority));\n o(this).replaceWith(select);\n select.change(function () {\n self.updatePriority(select.val());\n })\n return false;\n });\n\n // show details\n view.el.find('.contents').toggle(function () {\n view.details().addClass('show');\n self.showDetails = true;\n }, function () {\n view.details().removeClass('show');\n self.showDetails = false;\n });\n }\n\n this.renderUpdate();\n\n return view.el;\n};\n\n/**\n * Update this jobs state to `state`.\n *\n * @param {String} state\n */\n\nJob.prototype.updateState = function (state) {\n request('PUT', './job/' + this.id + '/state/' + state);\n};\n\n/**\n * Update this jobs priority to `n`.\n *\n * @param {Number} n\n */\n\nJob.prototype.updatePriority = function (n) {\n request('PUT', './job/' + this.id + '/priority/' + n);\n};\n\n/**\n * Update the job view.\n */\n\nJob.prototype.renderUpdate = function () {\n // TODO: templates\n var view = this.view\n , showError = this._showError\n , showProgress = this._showProgress;\n\n // type\n view.type(this.type);\n\n // errors\n if (showError && this.error) {\n view.errorMessage(this.error.split('\\n')[0]);\n } else {\n view.errorMessage().remove();\n }\n\n // attempts\n if (this.attempts.made) {\n view.attempts(this.attempts.made + '/' + this.attempts.max);\n } else {\n view.attempts().parent().remove();\n }\n\n // title\n view.title(this.data.title\n ? this.data.title\n : 'untitled');\n\n // details\n this.renderTimestamp('created_at');\n this.renderTimestamp('updated_at');\n this.renderTimestamp('failed_at');\n\n // delayed\n if ('delayed' == this.state) {\n var delay = parseInt(this.delay, 10)\n , creation = parseInt(this.created_at, 10)\n , remaining = relative(creation + delay - Date.now());\n view.title((this.data.title || '') + ' ( ' + remaining + ' )');\n }\n\n // inactive\n if ('inactive' == this.state) view.log.remove();\n\n // completion\n if ('complete' == this.state) {\n view.duration(relative(this.duration));\n view.updated_at().prev().text('Completed: ');\n view.priority().parent().hide();\n } else {\n view.duration().parent().remove();\n }\n\n // error\n if ('failed' == this.state) {\n view.error().show().find('pre').text(this.error);\n } else {\n view.error().hide();\n }\n\n // progress indicator\n if (showProgress) this._progress.update(this.progress).draw(this.ctx);\n\n // logs\n if (this.showDetails) {\n request('GET', './job/' + this.id + '/log', function (log) {\n var ul = view.log.show();\n \n // return early if log hasnt changed\n if (ul.text() === log) return;\n \n ul.find('li').remove();\n log.forEach(function (line) {\n ul.append(o('
  • %s
    • ', line));\n });\n });\n }\n};\n\n/**\n * Render timestamp for the given `prop`.\n *\n * @param {String} prop\n */\n\nJob.prototype.renderTimestamp = function (prop) {\n var val = this[prop]\n , view = this.view;\n\n if (val) {\n view[prop]().text(relative(Date.now() - val) + ' ago');\n } else {\n view[prop]().parent().remove();\n }\n};\n" }, { "path": "lib/http/public/javascripts/jquery.ext.js", "content": "// proxy to allow formatting\n// and because $ is ugly\n\nvar o = function (val) {\n var args = arguments\n , options = args[1]\n , i = 0;\n\n if ('string' != typeof val) return $(val);\n if (!~val.indexOf('<')) return $(val);\n\n val = val.replace(/%([sd])/g, function (_, specifier) {\n var arg = args[++i];\n switch (specifier) {\n case 's':\n return String(arg)\n case 'd':\n return arg | 0;\n }\n });\n\n val = val.replace(/\\{(\\w+)\\}/g, function (_, name) {\n return options[name];\n });\n\n return $(val);\n};\n\nfor (var key in $) o[key] = $[key];" }, { "path": "lib/http/public/javascripts/loading.js", "content": "/*!\n * kue - LoadingIndicator\n * Copyright (c) 2011 LearnBoost \n * MIT Licensed\n */\n\n/**\n * Initialize a new `LoadingIndicator`.\n */\n\nfunction LoadingIndicator() {\n this.size(0);\n this.fontSize(9);\n this.font('helvetica, arial, sans-serif');\n}\n\n/**\n * Set size to `n`.\n *\n * @param {Number} n\n * @return {LoadingIndicator} for chaining\n * @api public\n */\n\nLoadingIndicator.prototype.size = function (n) {\n this._size = n;\n return this;\n};\n\n/**\n * Set font size to `n`.\n *\n * @param {Number} n\n * @return {LoadingIndicator} for chaining\n * @api public\n */\n\nLoadingIndicator.prototype.fontSize = function (n) {\n this._fontSize = n;\n return this;\n};\n\n/**\n * Set font `family`.\n *\n * @param {String} family\n * @return {LoadingIndicator} for chaining\n */\n\nLoadingIndicator.prototype.font = function (family) {\n this._font = family;\n return this;\n};\n\n/**\n * Update pos to `n`.\n *\n * @param {Number} n\n * @return {LoadingIndicator} for chaining\n */\n\nLoadingIndicator.prototype.update = function (n) {\n this.pos = n;\n return this;\n};\n\n/**\n * Draw on `ctx`.\n *\n * @param {CanvasRenderingContext2d} ctx\n * @return {LoadingIndicator} for chaining\n */\n\nLoadingIndicator.prototype.draw = function (ctx) {\n var pos = this.pos % 360\n , size = this._size\n , half = size / 2\n , x = half\n , y = half\n , rad = half - 1\n , fontSize = this._fontSize;\n\n ctx.font = fontSize + 'px ' + this._font;\n\n ctx.clearRect(0, 0, size, size);\n\n // outer circle\n ctx.strokeStyle = '#9f9f9f';\n ctx.beginPath();\n ctx.arc(x, y, rad, pos, Math.PI / 2 + pos, false);\n ctx.stroke();\n\n // inner circle\n ctx.strokeStyle = '#eee';\n ctx.beginPath();\n ctx.arc(x, y, rad - 3, -pos, Math.PI / 2 - pos, false);\n ctx.stroke();\n\n // text\n var text = 'Loading'\n , w = ctx.measureText(text).width;\n\n ctx.fillText(\n text\n , x - w / 2 + 1\n , y + fontSize / 2 - 1);\n\n return this;\n};\n" }, { "path": "lib/http/public/javascripts/main.js", "content": "/*!\n * kue - http - main\n * Copyright (c) 2011 LearnBoost \n * MIT Licensed\n */\n\n// TODO: clean up\n// TODO: server-side config for this stuff\n// TODO: optimize! many of these jQuery objects can be cached\n\n/**\n * Active state.\n */\n\nvar active;\n\n/**\n * Active type filter.\n */\n\nvar filter;\n\n/**\n * Number of jobs fetched when \"more\" is clicked.\n */\n\nvar more = 10;\n\n/**\n * Number of jobs shown.\n */\n\nvar to = more;\n\n/**\n * Sort order.\n */\n\nvar sort = 'asc';\n\n/**\n * Loading indicator.\n */\n\nvar loading;\n\n/**\n * Initialize UI.\n */\n\nfunction init(state) {\n var canvas = o('#loading canvas').get(0)\n , ctx = canvas.getContext('2d');\n\n loading = new LoadingIndicator;\n loading.ctx = ctx;\n loading.size(canvas.width);\n\n pollStats(1000);\n show(state)();\n o('li.inactive a').click(show('inactive'));\n o('li.complete a').click(show('complete'));\n o('li.active a').click(show('active'));\n o('li.failed a').click(show('failed'));\n o('li.delayed a').click(show('delayed'));\n\n o('#filter').change(function () {\n filter = $(this).val();\n });\n\n o('#sort').change(function () {\n sort = $(this).val();\n o('#jobs .job').remove();\n });\n\n onpopstate = function (e) {\n if (e.state) show(e.state.state)();\n };\n}\n\n/**\n * Show loading indicator.\n */\n\nfunction showLoading() {\n var n = 0;\n o('#loading').show();\n showLoading.timer = setInterval(function () {\n loading.update(++n).draw(loading.ctx);\n }, 50);\n}\n\n/**\n * Hide loading indicator.\n */\n\nfunction hideLoading() {\n o('#loading').hide();\n clearInterval(showLoading.timer);\n}\n\n/**\n * Infinite scroll.\n */\n\nfunction infiniteScroll() {\n if (infiniteScroll.bound) return;\n var body = o('body');\n hideLoading();\n infiniteScroll.bound = true;\n\n o(window).scroll(function (e) {\n var top = body.scrollTop()\n , height = body.innerHeight()\n , windowHeight = window.innerHeight\n , pad = 30;\n\n if (top + windowHeight + pad >= height) {\n to += more;\n infiniteScroll.bound = false;\n showLoading();\n o(window).unbind('scroll');\n }\n });\n}\n\n/**\n * Show jobs with `state`.\n *\n * @param {String} state\n * @param {Boolean} init\n * @return {Function}\n */\n\nfunction show(state) {\n return function () {\n active = state;\n if (pollForJobs.timer) {\n clearTimeout(pollForJobs.timer);\n delete pollForJobs.timer;\n }\n history.pushState({ state: state }, state, state);\n o('#jobs .job').remove();\n o('#menu li a').removeClass('active');\n o('#menu li.' + state + ' a').addClass('active');\n pollForJobs(state, 1000);\n return false;\n }\n}\n\n/**\n * Poll for jobs with `state` every `ms`.\n *\n * @param {String} state\n * @param {Number} ms\n */\n\nfunction pollForJobs(state, ms) {\n o('h1').text(state);\n refreshJobs(state, function () {\n infiniteScroll();\n if (!pollForJobs.timer) pollForJobs.timer = setTimeout(function () {\n delete pollForJobs.timer;\n pollForJobs(state, ms);\n }, ms);\n });\n};\n\n/**\n * Re-request and refresh job elements.\n *\n * @param {String} state\n * @param {Function} fn\n */\n\nfunction refreshJobs(state, fn) {\n // TODO: clean this crap up\n var jobHeight = o('#jobs .job .block').outerHeight(true)\n , top = o(window).scrollTop()\n , height = window.innerHeight\n , visibleFrom = Math.max(0, Math.floor(top / jobHeight))\n , visibleTo = Math.floor((top + height) / jobHeight)\n , url = './jobs/'\n + (filter ? filter + '/' : '')\n + state + '/0..' + to\n + '/' + sort;\n\n // var color = ['blue', 'red', 'yellow', 'green', 'purple'][Math.random() * 5 | 0];\n\n request(url, function (jobs) {\n var len = jobs.length\n , job\n , el;\n\n // remove jobs which have changed their state\n o('#jobs .job').each(function (i, el) {\n var el = $(el)\n , id = (el.attr('id') || '').replace('job-', '')\n , found = jobs.some(function (job) {\n return job && id == job.id;\n });\n if (!found) el.remove();\n });\n\n for (var i = 0; i < len; ++i) {\n if (!jobs[i]) continue;\n\n // exists\n if (o('#job-' + jobs[i].id).length) {\n if (i < visibleFrom || i > visibleTo) continue;\n el = o('#job-' + jobs[i].id);\n // el.css('background-color', color);\n job = el.get(0).job;\n job.update(jobs[i])\n .showProgress('active' == active)\n .showErrorMessage('failed' == active)\n .render();\n // new\n } else {\n job = new Job(jobs[i]);\n el = job.showProgress('active' == active)\n .showErrorMessage('failed' == active)\n .render(true);\n\n el.get(0).job = job;\n el.appendTo('#jobs');\n }\n }\n\n fn();\n });\n}\n\n/**\n * Poll for stats every `ms`.\n *\n * @param {Number} ms\n */\n\nfunction pollStats(ms) {\n request('./stats', function (data) {\n o('li.inactive .count').text(data.inactiveCount);\n o('li.active .count').text(data.activeCount);\n o('li.complete .count').text(data.completeCount);\n o('li.failed .count').text(data.failedCount);\n o('li.delayed .count').text(data.delayedCount);\n setTimeout(function () {\n pollStats(ms);\n }, ms);\n });\n}\n\n/**\n * Request `url` and invoke `fn(res)`.\n *\n * @param {String} url\n * @param {Function} fn\n */\n\nfunction request(url, fn) {\n var method = 'GET';\n\n if ('string' == typeof fn) {\n method = url;\n url = fn;\n fn = arguments[2];\n }\n\n fn = fn || function () {\n };\n\n o.ajax({ type: method, url: url })\n .success(function (res) {\n res.error\n ? error(res.error)\n : fn(res);\n });\n}\n\n/**\n * Display error `msg`.\n *\n * @param {String} msg\n */\n\nfunction error(msg) {\n o('#error').text(msg).addClass('show');\n setTimeout(function () {\n o('#error').removeClass('show');\n }, 4000);\n}\n" }, { "path": "lib/http/public/javascripts/progress.js", "content": "/*!\n * kue - Progress\n * Copyright (c) 2011 LearnBoost \n * MIT Licensed\n */\n\n/**\n * Initialize a new `Progress` indicator.\n */\n\nfunction Progress() {\n this.percent = 0;\n this.size(0);\n this.fontSize(12);\n this.font('helvetica, arial, sans-serif');\n}\n\n/**\n * Set progress size to `n`.\n *\n * @param {Number} n\n * @return {Progress} for chaining\n * @api public\n */\n\nProgress.prototype.size = function (n) {\n this._size = n;\n return this;\n};\n\n/**\n * Set text to `str`.\n *\n * @param {String} str\n * @return {Progress} for chaining\n * @api public\n */\n\nProgress.prototype.text = function (str) {\n this._text = str;\n return this;\n};\n\n/**\n * Set font size to `n`.\n *\n * @param {Number} n\n * @return {Progress} for chaining\n * @api public\n */\n\nProgress.prototype.fontSize = function (n) {\n this._fontSize = n;\n return this;\n};\n\n/**\n * Set font `family`.\n *\n * @param {String} family\n * @return {Progress} for chaining\n */\n\nProgress.prototype.font = function (family) {\n this._font = family;\n return this;\n};\n\n/**\n * Update percentage to `n`.\n *\n * @param {Number} n\n * @return {Progress} for chaining\n */\n\nProgress.prototype.update = function (n) {\n this.percent = n;\n return this;\n};\n\n/**\n * Draw on `ctx`.\n *\n * @param {CanvasRenderingContext2d} ctx\n * @return {Progress} for chaining\n */\n\nProgress.prototype.draw = function (ctx) {\n var percent = Math.min(this.percent, 100)\n , size = this._size\n , half = size / 2\n , x = half\n , y = half\n , rad = half - 1\n , fontSize = this._fontSize;\n\n ctx.font = fontSize + 'px ' + this._font;\n\n var angle = Math.PI * 2 * (percent / 100);\n ctx.clearRect(0, 0, size, size);\n\n // outer circle\n ctx.strokeStyle = '#9f9f9f';\n ctx.beginPath();\n ctx.arc(x, y, rad, 0, angle, false);\n ctx.stroke();\n\n // inner circle\n ctx.strokeStyle = '#eee';\n ctx.beginPath();\n ctx.arc(x, y, rad - 1, 0, angle, true);\n ctx.stroke();\n\n // text\n var text = this._text || (percent | 0) + '%'\n , w = ctx.measureText(text).width;\n\n ctx.fillText(\n text\n , x - w / 2 + 1\n , y + fontSize / 2 - 1);\n\n return this;\n};\n" }, { "path": "lib/http/public/javascripts/search.js", "content": "o(function () {\n var search = o('#search');\n search.keyup(function () {\n var val = search.val().trim()\n , jobs = o('#jobs .job');\n\n // show all\n if (val.length < 2) return jobs.show();\n\n // query\n o.get('./job/search?q=' + encodeURIComponent(val), function (ids) {\n jobs.each(function (i, el) {\n var id = el.id.replace('job-', '');\n if (~ids.indexOf(id)) {\n o(el).show();\n } else {\n o(el).hide();\n }\n });\n });\n });\n});\n" }, { "path": "lib/http/public/javascripts/utils.js", "content": "/*!\n * kue - utils\n * Copyright (c) 2010 LearnBoost \n * MIT Licensed\n */\n\n/**\n * Format `ms` in words.\n *\n * @param {Number} ms\n * @return {String}\n */\n\nfunction relative(ms) {\n var sec = 1000\n , min = 60 * sec\n , hour = 60 * min;\n\n function n(n, name) {\n n = Math.round(n);\n return n + ' ' + name + (n > 1 ? 's' : '');\n }\n\n if (isNaN(ms)) return '';\n if (ms < sec) return 'less than one second';\n if (ms < min) return n(ms / sec, 'second');\n if (ms < hour) return n(ms / min, 'minute');\n return n(ms / hour, 'hour');\n // TODO: larger than an hour or so, we should\n // have some nice date formatting\n}\n\n/**\n * Default job states.\n */\n\nvar states = {\n active: 'active', inactive: 'inactive', failed: 'failed', complete: 'complete', delayed: 'delayed'\n};\n\n/**\n * Default job priority map.\n */\n\nvar priorities = {\n '10': 'low', '0': 'normal', '-5': 'medium', '-10': 'high', '-15': 'critical'\n};\n\n/**\n * Return priority string for `job`.\n *\n * @param {Job} job\n * @return {String}\n */\n\nfunction priority(job) {\n return priorities[job.priority] || job.priority;\n}\n\n/**\n * Generate options from `obj`.\n *\n * @param {Object} obj\n * @param {String} selected\n * @return {String}\n */\n\nfunction options(obj, selected) {\n var html = '';\n for (var key in obj) {\n html += '\\n';\n }\n return html;\n}\n" }, { "path": "lib/http/public/stylesheets/actions.styl", "content": "\n#actions\n fixed: top -2px right -2px\n z-index: 20\n\n#sort\n#filter\n#search\n float: left\n margin: 0\n padding: 5px 10px\n border: 1px solid #eee\n border-radius: 0 0 0 5px\n -webkit-appearance: none\n color: dark\n outline: none\n &:hover\n border-color: #eee - 10%\n\n#sort\n#filter\n cursor: pointer\n\n#sort\n#filter\n border-radius: 0\n border-left: none" }, { "path": "lib/http/public/stylesheets/config.styl", "content": "// general colors\n\ndark = #3b3b3b\nlight = #666\nlighter = #777\n\nbg = #fff\n\n// status colors\n\ninactive-color = #00CCCC\ncomplete-color = #00CC7A\nactive-color = #CCC500\nfailed-color = #c00\n\n// menu config\n\nmenu-bg = dark\nmenu-fg = lighter\n\nmenu-intensity = 13%\nmenu-colored = false\n\n// job config\n\njob-bg = white\n\n// scrollbar\n\nscroll-bg = transparent\nscroll-thumb = menu-bg\nscroll-width = 6px\n" }, { "path": "lib/http/public/stylesheets/context-menu.styl", "content": "@import 'mixins'\n\nhighlight-color = #00B3E9\n\n.context-menu\n display: none\n reset-list()\n decorated-box()\n li\n &:last-child a\n border-bottom: none\n a\n display: block\n background: white\n padding: 5px 10px\n border: 1px solid transparent\n border-bottom: 1px solid #eee\n font-size: 12px\n &:hover\n background: linear-gradient(bottom, highlight-color, highlight-color + 50%)\n color: white\n border: 1px solid white\n &:active\n background: linear-gradient(bottom, highlight-color + 10%, highlight-color + 10% + 50%)\n" }, { "path": "lib/http/public/stylesheets/error.styl", "content": "\n#error\n fixed: top -50px right 15px\n padding: 20px\n transition: top 500ms, opacity 500ms\n opacity: 0\n background: rgba(dark, .2)\n border: 1px solid rgba(dark, .3)\n border-radius: 5px\n color: dark\n &.show\n top: 15px\n opacity: 1\n" }, { "path": "lib/http/public/stylesheets/job.styl", "content": "@import 'mixins'\n\n#job-template\n display: none\n\nbar(color)\n background: linear-gradient(top, color + 20%, color)\n border: 1px solid rgba(white, .2)\n color: white\n\n// generic blocks\n\n.block\n decorated-box()\n width: 90%\n margin: 10px 25px\n padding: 20px 25px\n h2\n margin: 0\n absolute: top 5px left -15px\n padding: 5px\n font-size: 10px\n border-radius: left 5px right 2px\n background: linear-gradient(left, menu-fg - 10%, 50% menu-fg + 5%)\n box-shadow: -1px 0 1px 1px rgba(black, .1)\n color: white\n text-shadow: 1px 1px 1px #444\n .type\n color: lighter + 20%\n\n// job delay\n.job td.title em\n color: lighter + 20%\n\n// job blocks\n\n.job .block\n position: relative\n background: job-bg\n cursor: pointer\n table td:first-child\n display: none\n .progress\n absolute: top 15px right 20px\n .attempts\n display: none\n absolute: top right\n padding: 5px 8px\n border-radius: 2px\n font-size: 10px\n .remove\n absolute: top 30px right -6px\n /*background: white*/\n background: #F05151\n color: white\n display: block\n width: size = 20px\n height: size\n line-height: size\n text-align: center\n font-size: 12px\n font-weight: bold\n outline: none\n border: 1px solid #eee\n border-radius: size\n transition: opacity 200ms, top 300ms\n opacity: 0\n &:hover\n border: 1px solid #eee - 10%\n &:active\n border: 1px solid #eee - 20%\n .restart\n absolute: top 30px right -6px\n /*background: white*/\n background: #00e600\n color: white\n display: block\n width: size = 20px\n height: size\n line-height: size\n text-align: center\n font-size: 12px\n font-weight: bold\n outline: none\n border: 1px solid #eee\n border-radius: size\n transition: opacity 200ms, top 300ms\n opacity: 0\n &:hover\n border: 1px solid #eee - 10%\n &:active\n border: 1px solid #eee - 20%\n &:hover\n .remove\n opacity: 1\n top: -6px\n .restart\n opacity: 1\n top: 16px\n\n// details\n\n.job .details\n background: dark\n width: 89%\n margin-top: -10px\n margin-left: 35px\n border-radius: bottom 5px\n box-shadow: inset 0 1px 10px 0 rgba(black, .8)\n transition: padding 200ms, height 200ms\n height: 0\n overflow: hidden\n table\n width: 100%\n td:first-child\n width: 60px\n color: light + 30%\n &.show\n padding: 15px 20px\n height: auto\n\n// job log\n\n.job ul.log\n reset-list()\n margin: 5px\n padding: 10px\n max-height: 100px\n overflow-y: auto\n border-radius: 5px\n width: 95%\n li\n padding: 5px 0\n border-bottom: 1px dotted light - 35%\n color: light\n &:last-child\n border-bottom: none\n\n// scrollbar\n\n.job .details\n ::-webkit-scrollbar\n width: 2px\n ::-webkit-scrollbar-thumb:vertical\n background: light + 20%\n ::-webkit-scrollbar-track\n border: 1px solid rgba(white, .1)\n\n// sections\n\n.job .details > div\n padding: 10px 0\n border-bottom: 1px solid light - 35%\n &:last-child\n border-bottom: none\n" }, { "path": "lib/http/public/stylesheets/main.css", "content": "body {\n padding: 50px 120px;\n}\n#menu {\n margin: 0;\n padding: 0;\n position: fixed;\n top: 0;\n left: 0;\n height: 100%;\n width: 80px;\n background: #3b3b3b;\n border-right: 1px solid #232323;\n -webkit-box-shadow: 0 0 0 1px rgba(255,255,255,0.5);\n box-shadow: 0 0 0 1px rgba(255,255,255,0.5);\n}\n#menu li {\n margin: 0;\n list-style: none;\n}\n#menu li {\n position: relative;\n text-align: center;\n}\n#menu li .count {\n position: absolute;\n top: 15px;\n left: 0;\n text-shadow: 1px 1px 1px #333;\n width: 100%;\n color: #808080;\n}\n#menu li a {\n display: block;\n padding: 40px 0 10px 0;\n color: #777;\n border-top: 1px solid #545454;\n border-bottom: 1px solid #333;\n font-size: 12px;\n background: #393939;\n}\n#menu li a:hover {\n background: #434343;\n}\n#menu li a:active,\n#menu li a.active {\n background: #343434;\n -webkit-box-shadow: inset 0 0 3px 2px #282828, inset 0 -5px 10px 2px #303030;\n box-shadow: inset 0 0 3px 2px #282828, inset 0 -5px 10px 2px #303030;\n border-bottom: 1px solid #222;\n}\n.context-menu {\n display: none;\n margin: 0;\n padding: 0;\n border: 1px solid #eee;\n border-bottom-color: rgba(0,0,0,0.25);\n border-left-color: rgba(0,0,0,0.2);\n border-right-color: rgba(0,0,0,0.2);\n -webkit-box-shadow: 0 2px 2px 0 rgba(0,0,0,0.1);\n box-shadow: 0 2px 2px 0 rgba(0,0,0,0.1);\n -webkit-border-radius: 4px;\n border-radius: 4px;\n}\n.context-menu li {\n margin: 0;\n list-style: none;\n}\n.context-menu li:last-child a {\n border-bottom: none;\n}\n.context-menu li a {\n display: block;\n background: #fff;\n padding: 5px 10px;\n border: 1px solid transparent;\n border-bottom: 1px solid #eee;\n font-size: 12px;\n}\n.context-menu li a:hover {\n background: -webkit-linear-gradient(bottom, #00b3e9, #74dfff);\n background: -moz-linear-gradient(bottom, #00b3e9, #74dfff);\n background: -o-linear-gradient(bottom, #00b3e9, #74dfff);\n background: -ms-linear-gradient(bottom, #00b3e9, #74dfff);\n background: linear-gradient(to top, #00b3e9, #74dfff);\n color: #fff;\n border: 1px solid #fff;\n}\n.context-menu li a:active {\n background: -webkit-linear-gradient(bottom, #06c5ff, #83e2ff);\n background: -moz-linear-gradient(bottom, #06c5ff, #83e2ff);\n background: -o-linear-gradient(bottom, #06c5ff, #83e2ff);\n background: -ms-linear-gradient(bottom, #06c5ff, #83e2ff);\n background: linear-gradient(to top, #06c5ff, #83e2ff);\n}\n#job-template {\n display: none;\n}\n.block {\n border: 1px solid #eee;\n border-bottom-color: rgba(0,0,0,0.25);\n border-left-color: rgba(0,0,0,0.2);\n border-right-color: rgba(0,0,0,0.2);\n -webkit-box-shadow: 0 2px 2px 0 rgba(0,0,0,0.1);\n box-shadow: 0 2px 2px 0 rgba(0,0,0,0.1);\n -webkit-border-radius: 4px;\n border-radius: 4px;\n width: 90%;\n margin: 10px 25px;\n padding: 20px 25px;\n}\n.block h2 {\n margin: 0;\n position: absolute;\n top: 5px;\n left: -15px;\n padding: 5px;\n font-size: 10px;\n -webkit-border-top-left-radius: 5px;\n border-top-left-radius: 5px;\n -webkit-border-bottom-left-radius: 5px;\n border-bottom-left-radius: 5px;\n -webkit-border-top-right-radius: 2px;\n border-top-right-radius: 2px;\n -webkit-border-bottom-right-radius: 2px;\n border-bottom-right-radius: 2px;\n background: -webkit-linear-gradient(left, #6b6b6b, #7e7e7e 50%);\n background: -moz-linear-gradient(left, #6b6b6b, #7e7e7e 50%);\n background: -o-linear-gradient(left, #6b6b6b, #7e7e7e 50%);\n background: -ms-linear-gradient(left, #6b6b6b, #7e7e7e 50%);\n background: linear-gradient(to right, #6b6b6b, #7e7e7e 50%);\n -webkit-box-shadow: -1px 0 1px 1px rgba(0,0,0,0.1);\n box-shadow: -1px 0 1px 1px rgba(0,0,0,0.1);\n color: #fff;\n text-shadow: 1px 1px 1px #444;\n}\n.block .type {\n color: #929292;\n}\n.job td.title em {\n color: #929292;\n}\n.job .block {\n position: relative;\n background: #fff;\n cursor: pointer;\n}\n.job .block table td:first-child {\n display: none;\n}\n.job .block .progress {\n position: absolute;\n top: 15px;\n right: 20px;\n}\n.job .block .attempts {\n display: none;\n position: absolute;\n top: 0;\n right: 0;\n padding: 5px 8px;\n -webkit-border-radius: 2px;\n border-radius: 2px;\n font-size: 10px;\n}\n.job .block .remove {\n position: absolute;\n top: 30px;\n right: -6px;\n/*background: white*/\n background: #f05151;\n color: #fff;\n display: block;\n width: 20px;\n height: 20px;\n line-height: 20px;\n text-align: center;\n font-size: 12px;\n font-weight: bold;\n outline: none;\n border: 1px solid #eee;\n -webkit-border-radius: 20px;\n border-radius: 20px;\n -webkit-transition: opacity 200ms, top 300ms;\n -moz-transition: opacity 200ms, top 300ms;\n -o-transition: opacity 200ms, top 300ms;\n -ms-transition: opacity 200ms, top 300ms;\n transition: opacity 200ms, top 300ms;\n opacity: 0;\n -ms-filter: \"progid:DXImageTransform.Microsoft.Alpha(Opacity=0)\";\n filter: alpha(opacity=0);\n}\n.job .block .remove:hover {\n border: 1px solid #d6d6d6;\n}\n.job .block .remove:active {\n border: 1px solid #bebebe;\n}\n.job .block .restart {\n position: absolute;\n top: 30px;\n right: -6px;\n/*background: white*/\n background: #00e600;\n color: #fff;\n display: block;\n width: 20px;\n height: 20px;\n line-height: 20px;\n text-align: center;\n font-size: 12px;\n font-weight: bold;\n outline: none;\n border: 1px solid #eee;\n -webkit-border-radius: 20px;\n border-radius: 20px;\n -webkit-transition: opacity 200ms, top 300ms;\n -moz-transition: opacity 200ms, top 300ms;\n -o-transition: opacity 200ms, top 300ms;\n -ms-transition: opacity 200ms, top 300ms;\n transition: opacity 200ms, top 300ms;\n opacity: 0;\n -ms-filter: \"progid:DXImageTransform.Microsoft.Alpha(Opacity=0)\";\n filter: alpha(opacity=0);\n}\n.job .block .restart:hover {\n border: 1px solid #d6d6d6;\n}\n.job .block .restart:active {\n border: 1px solid #bebebe;\n}\n.job .block:hover .remove {\n opacity: 1;\n -ms-filter: none;\n filter: none;\n top: -6px;\n}\n.job .block:hover .restart {\n opacity: 1;\n -ms-filter: none;\n filter: none;\n top: 16px;\n}\n.job .details {\n background: #3b3b3b;\n width: 89%;\n margin-top: -10px;\n margin-left: 35px;\n -webkit-border-bottom-left-radius: 5px;\n border-bottom-left-radius: 5px;\n -webkit-border-bottom-right-radius: 5px;\n border-bottom-right-radius: 5px;\n -webkit-box-shadow: inset 0 1px 10px 0 rgba(0,0,0,0.8);\n box-shadow: inset 0 1px 10px 0 rgba(0,0,0,0.8);\n -webkit-transition: padding 200ms, height 200ms;\n -moz-transition: padding 200ms, height 200ms;\n -o-transition: padding 200ms, height 200ms;\n -ms-transition: padding 200ms, height 200ms;\n transition: padding 200ms, height 200ms;\n height: 0;\n overflow: hidden;\n}\n.job .details table {\n width: 100%;\n}\n.job .details table td:first-child {\n width: 60px;\n color: #949494;\n}\n.job .details.show {\n padding: 15px 20px;\n height: auto;\n}\n.job ul.log {\n margin: 0;\n padding: 0;\n margin: 5px;\n padding: 10px;\n max-height: 100px;\n overflow-y: auto;\n -webkit-border-radius: 5px;\n border-radius: 5px;\n width: 95%;\n}\n.job ul.log li {\n margin: 0;\n list-style: none;\n}\n.job ul.log li {\n padding: 5px 0;\n border-bottom: 1px dotted #424242;\n color: #666;\n}\n.job ul.log li:last-child {\n border-bottom: none;\n}\n.job .details ::-webkit-scrollbar {\n width: 2px;\n}\n.job .details ::-webkit-scrollbar-thumb:vertical {\n background: #858585;\n}\n.job .details ::-webkit-scrollbar-track {\n border: 1px solid rgba(255,255,255,0.1);\n}\n.job .details > div {\n padding: 10px 0;\n border-bottom: 1px solid #424242;\n}\n.job .details > div:last-child {\n border-bottom: none;\n}\n#actions {\n position: fixed;\n top: -2px;\n right: -2px;\n z-index: 20;\n}\n#sort,\n#filter,\n#search {\n float: left;\n margin: 0;\n padding: 5px 10px;\n border: 1px solid #eee;\n -webkit-border-radius: 0 0 0 5px;\n border-radius: 0 0 0 5px;\n -webkit-appearance: none;\n color: #3b3b3b;\n outline: none;\n}\n#sort:hover,\n#filter:hover,\n#search:hover {\n border-color: #d6d6d6;\n}\n#sort,\n#filter {\n cursor: pointer;\n}\n#sort,\n#filter {\n -webkit-border-radius: 0;\n border-radius: 0;\n border-left: none;\n}\n#error {\n position: fixed;\n top: -50px;\n right: 15px;\n padding: 20px;\n -webkit-transition: top 500ms, opacity 500ms;\n -moz-transition: top 500ms, opacity 500ms;\n -o-transition: top 500ms, opacity 500ms;\n -ms-transition: top 500ms, opacity 500ms;\n transition: top 500ms, opacity 500ms;\n opacity: 0;\n -ms-filter: \"progid:DXImageTransform.Microsoft.Alpha(Opacity=0)\";\n filter: alpha(opacity=0);\n background: rgba(59,59,59,0.2);\n border: 1px solid rgba(59,59,59,0.3);\n -webkit-border-radius: 5px;\n border-radius: 5px;\n color: #3b3b3b;\n}\n#error.show {\n top: 15px;\n opacity: 1;\n -ms-filter: none;\n filter: none;\n}\nbody {\n font: 13px \"helvetica neue\", helvetica, arial, sans-serif;\n -webkit-font-smoothing: antialiased;\n background: #fff;\n color: #666;\n}\nh1,\nh2,\nh3 {\n margin: 0 0 25px 0;\n padding: 0;\n font-weight: normal;\n text-transform: capitalize;\n color: #666;\n}\nh2 {\n font-size: 16px;\n margin-top: 20px;\n}\npre {\n margin-top: 20px;\n}\na {\n text-decoration: none;\n cursor: pointer;\n}\ntable {\n border-collapse: separate;\n border-spacing: 0;\n vertical-align: middle;\n}\ntable tr td {\n padding: 2px 5px;\n}\n#loading {\n width: 100%;\n text-align: center;\n margin-top: 40px;\n margin-left: 20px;\n}\n#loading canvas {\n margin: 0 auto;\n}\n" }, { "path": "lib/http/public/stylesheets/main.styl", "content": "font-smoothing()\n -webkit-font-smoothing: arguments\n\n@import 'nib'\n@import 'config'\n@import 'scrollbar'\n@import 'menu'\n@import 'context-menu'\n@import 'job'\n@import 'actions'\n@import 'error'\n\nbody\n font: 13px \"helvetica neue\", helvetica, arial, sans-serif\n font-smoothing: antialiased\n background: bg\n color: light\n\nh1, h2, h3\n margin: 0 0 25px 0\n padding: 0\n font-weight: normal\n text-transform: capitalize\n color: light\n\nh2\n font-size: 16px\n margin-top: 20px\n\nbutton\na.button\ninput[type='submit']\n bold-button(glow:#00ABFA)\n\npre\n margin-top: 20px\n\na\n text-decoration: none\n cursor: pointer\n\ntable\n reset-table()\n tr td\n padding: 2px 5px\n\n#loading\n width: 100%\n text-align: center\n margin-top: 40px\n margin-left: 20px\n canvas\n margin: 0 auto\n" }, { "path": "lib/http/public/stylesheets/menu.styl", "content": "@import 'mixins'\n\n#menu\n reset-list()\n fixed: top left\n height: 100%\n width: 80px\n background: menu-bg\n border-right: 1px solid menu-bg - 40%\n box-shadow: 0 0 0 1px rgba(white, .5)\n li\n position: relative\n text-align: center\n if menu-colored\n &.inactive\n border-right: 1px solid inactive-color\n &.active\n border-right: 1px solid active-color\n &.complete\n border-right: 1px solid complete-color\n &.failed\n border-right: 1px solid failed-color\n .count\n absolute: top 15px left\n text-shadow: 1px 1px 1px menu-bg - menu-intensity\n width: 100%\n color: menu-fg + (menu-intensity / 2)\n a\n display: block\n padding: 40px 0 10px 0\n color: menu-fg\n border-top: 1px solid menu-bg + menu-intensity\n border-bottom: 1px solid menu-bg - menu-intensity\n font-size: 12px\n background: menu-bg -= 2%\n &:hover\n background: menu-bg + 5%\n &:active\n &.active\n background: #343434\n box-shadow: inset 0 0 3px 2px menu-bg - 30%, inset 0 -5px 10px 2px menu-bg - 15%\n border-bottom: 1px solid menu-bg - 40%\n" }, { "path": "lib/http/public/stylesheets/mixins.styl", "content": "reset-list()\n margin: 0\n padding: 0\n li\n margin: 0\n list-style: none\n\ndecorated-box()\n border: 1px solid #eee\n border-bottom-color: rgba(black, .25)\n border-left-color: rgba(black, .2)\n border-right-color: rgba(black, .2)\n box-shadow: 0 2px 2px 0 rgba(black, .1)\n border-radius: 4px\n" }, { "path": "lib/http/public/stylesheets/scrollbar.styl", "content": "width = scroll-width\npad-x = 60px\npad-y = 40px\n\nbody\n padding: 50px 120px\n\n/*\nhtml\n overflow: auto\n\nbody\n position: absolute\n top: pad-y\n left: pad-x\n bottom: pad-y\n right: pad-x\n padding: 0 pad-x\n overflow-y: scroll\n overflow-x: hidden\n\n::-webkit-scrollbar\n background: scroll-bg\n width: width\n\n::-webkit-scrollbar-button:start:decrement\n::-webkit-scrollbar-button:start:increment\n display: none\n\n::-webkit-scrollbar-track\n border-radius: (width / 2)\n box-shadow: inset 0 0 1px rgba(black, .2), inset 0 4px 10px rgba(black, .2)\n border: 1px solid rgba(white, .5)\n\n::-webkit-scrollbar-track-piece\n background: transparent\n\n::-webkit-scrollbar-thumb:vertical\n height: 30px\n transition: background-color 300ms ease-out\n background: rgba(scroll-thumb, .5)\n border-radius: (width / 2)\n &:window-inactive\n background: rgba(scroll-thumb, .2)\n*/" }, { "path": "lib/http/routes/index.js", "content": "/*!\n * kue - http - routes\n * Copyright (c) 2011 LearnBoost \n * MIT Licensed\n */\n\n/**\n * Module dependencies.\n */\n\nvar Queue = require('../../kue')\n , Job = require('../../queue/job')\n , queue = Queue.createQueue();\n\n/**\n * Serve the index page.\n */\n\nexports.jobs = function( state ) {\n return function( req, res ) {\n queue.types(function( err, types ) {\n res.render('job/list', {\n state: state, types: types, title: req.app.get('title')\n });\n });\n };\n};\n" }, { "path": "lib/http/routes/json.js", "content": "/*!\n * kue - http - routes - json\n * Copyright (c) 2011 LearnBoost \n * MIT Licensed\n */\n\n/**\n * Module dependencies.\n */\n\nvar Queue = require('../../kue')\n , Job = require('../../queue/job')\n , lodash = require('lodash')\n , queue = Queue.createQueue();\n\n/**\n * Search instance.\n */\n\nvar search;\nfunction getSearch() {\n if( search ) return search;\n var reds = require('reds');\n reds.createClient = require('../../redis').createClient;\n return search = reds.createSearch(queue.client.getKey('search'));\n}\n\n/**\n * Get statistics including:\n *\n * - inactive count\n * - active count\n * - complete count\n * - failed count\n * - delayed count\n *\n */\n\nexports.stats = function( req, res ) {\n get(queue)\n ('inactiveCount')\n ('completeCount')\n ('activeCount')\n ('failedCount')\n ('delayedCount')\n ('workTime')\n (function( err, obj ) {\n if( err ) return res.json({ error: err.message });\n res.json(obj);\n });\n};\n\n/**\n * Get job types.\n */\n\nexports.types = function( req, res ) {\n queue.types(function( err, types ) {\n if( err ) return res.json({ error: err.message });\n res.json(types);\n });\n};\n\n/**\n * Get jobs by range :from..:to.\n */\n\nexports.jobRange = function( req, res ) {\n var from = parseInt(req.params.from, 10)\n , to = parseInt(req.params.to, 10)\n , order = req.params.order;\n\n Job.range(from, to, order, function( err, jobs ) {\n if( err ) return res.json({ error: err.message });\n res.json(jobs);\n });\n};\n\n/**\n * Get jobs by :state, and range :from..:to.\n */\n\nexports.jobStateRange = function( req, res ) {\n var state = req.params.state\n , from = parseInt(req.params.from, 10)\n , to = parseInt(req.params.to, 10)\n , order = req.params.order;\n\n Job.rangeByState(state, from, to, order, function( err, jobs ) {\n if( err ) return res.json({ error: err.message });\n res.json(jobs);\n });\n};\n\n/**\n * Get jobs by :type, :state, and range :from..:to.\n */\n\nexports.jobTypeRange = function( req, res ) {\n var type = req.params.type\n , state = req.params.state\n , from = parseInt(req.params.from, 10)\n , to = parseInt(req.params.to, 10)\n , order = req.params.order;\n\n Job.rangeByType(type, state, from, to, order, function( err, jobs ) {\n if( err ) return res.json({ error: err.message });\n res.json(jobs);\n });\n};\n\n/**\n * Get jobs stats by :type and :state\n */\n\nexports.jobTypeStateStats = function( req, res ) {\n var type = req.params.type\n , state = req.params.state;\n\n queue.cardByType(type, state, function( err, count ) {\n if( err ) return res.json({ error: err.message });\n res.json({ count: count });\n });\n};\n\n/**\n * Get job by :id.\n */\n\nexports.job = function( req, res ) {\n var id = req.params.id;\n Job.get(id, function( err, job ) {\n if( err ) return res.json({ error: err.message });\n res.json(job);\n });\n};\n\n/**\n * Restart job by :id.\n */\n\nexports.inactive = function( req, res ) {\n var id = req.params.id;\n Job.get(id, function( err, job ) {\n if( err ) return res.json({ error: err.message });\n job.inactive();\n res.json({ message: 'job ' + id + ' inactive' });\n });\n};\n\n/**\n * Create a job.\n */\n\nexports.createJob = function( req, res ) {\n var body = req.body;\n\n function _create( args, next ) {\n if( !args.type ) return next({ error: 'Must provide job type' }, null, 400);\n\n var job = new Job(args.type, args.data || {});\n var options = args.options || {};\n if( options.attempts ) job.attempts(parseInt(options.attempts));\n if( options.priority ) job.priority(options.priority);\n if( options.delay ) job.delay(options.delay);\n if( options.searchKeys ) job.searchKeys(options.searchKeys);\n if( options.backoff ) job.backoff(options.backoff);\n if( options.removeOnComplete ) job.removeOnComplete(options.removeOnComplete);\n if( options.ttl ) job.ttl(options.ttl);\n \n job.save(function( err ) {\n if( err ) {\n return next({ error: err.message }, null, 500);\n }\n else {\n return next(null, { message: 'job created', id: job.id });\n }\n });\n }\n\n if( !lodash.isEmpty(body) ) {\n if( lodash.isArray(body) ) {\n var returnErrorCode = 0; // Default: we don't have any error\n var i = 0, len = body.length;\n var result = [];\n -function _iterate() {\n _create(body[ i ], function( err, status, errCode ) {\n result.push(err || status);\n if( err ) {\n // Set an error code for the response\n if( !returnErrorCode ) {\n returnErrorCode = errCode || 500;\n }\n }\n\n // Keep processing even after an error\n i++;\n if( i < len ) {\n _iterate();\n }\n else {\n // If we had an error code, return it\n if( returnErrorCode ) {\n res.status(returnErrorCode);\n }\n\n res.json(result);\n }\n })\n }()\n }\n else {\n _create(body, function( err, status, errCode ) {\n if( err ) {\n res.status(errCode || 500).json(err);\n }\n else {\n res.json(status);\n }\n })\n }\n }\n else {\n res.status(204); // \"No content\" status code\n res.end();\n }\n};\n\n/**\n * Remove job :id.\n */\n\nexports.remove = function( req, res ) {\n var id = req.params.id;\n Job.remove(id, function( err ) {\n if( err ) return res.json({ error: err.message });\n res.json({ message: 'job ' + id + ' removed' });\n });\n};\n\n/**\n * Update job :id :priority.\n */\n\nexports.updatePriority = function( req, res ) {\n var id = req.params.id\n , priority = parseInt(req.params.priority, 10);\n\n if( isNaN(priority) ) return res.json({ error: 'invalid priority' });\n Job.get(id, function( err, job ) {\n if( err ) return res.json({ error: err.message });\n job.priority(priority);\n job.save(function( err ) {\n if( err ) return res.json({ error: err.message });\n res.json({ message: 'updated priority' });\n });\n });\n};\n\n/**\n * Update job :id :state.\n */\n\nexports.updateState = function( req, res ) {\n var id = req.params.id\n , state = req.params.state;\n\n Job.get(id, function( err, job ) {\n if( err ) return res.json({ error: err.message });\n job.state(state);\n job.save(function( err ) {\n if( err ) return res.json({ error: err.message });\n res.json({ message: 'updated state' });\n });\n });\n};\n\n/**\n * Search and respond with ids.\n */\n\nexports.search = function( req, res ) {\n getSearch().query(req.query.q).end(function( err, ids ) {\n if( err ) return res.json({ error: err.message });\n res.json(ids);\n });\n};\n\n/**\n * Get log for job :id.\n */\n\nexports.log = function( req, res ) {\n var id = req.params.id;\n Job.log(id, function( err, log ) {\n if( err ) return res.json({ error: err.message });\n res.json(log);\n });\n};\n\n/**\n * Data fetching helper.\n */\n\nfunction get( obj ) {\n var pending = 0\n , res = {}\n , callback\n , done;\n\n return function _( arg ) {\n switch(typeof arg) {\n case 'function':\n callback = arg;\n break;\n case 'string':\n ++pending;\n obj[ arg ](function( err, val ) {\n if( done ) return;\n if( err ) return done = true, callback(err);\n res[ arg ] = val;\n --pending || callback(null, res);\n });\n break;\n }\n return _;\n };\n}\n" }, { "path": "lib/http/views/_filter.pug", "content": "select#filter\n option(value='') filter by\n each type in types\n option(value=type)= type\n" }, { "path": "lib/http/views/_job.pug", "content": ".job\n .block.contents\n h2.id\n a.remove(title='Delete Job') x\n a.restart(title='Restart Job') ↻\n canvas.progress(width=50, height=50)\n table.meta\n tbody\n tr\n td Type:\n td.type\n tr\n td Title:\n td.title\n tr\n td Error:\n td.errorMessage\n .details\n .data\n table.data\n tbody\n tr\n td State:\n td.state\n tr\n td Priority:\n td.priority\n tr\n td Attempts:\n td.attempts\n tr.time\n td Duration:\n td.duration\n tr.time\n td Created:\n td.created_at\n tr.time\n td Updated:\n td.updated_at\n tr.time\n td Failed:\n td.failed_at\n .error\n pre\n ul.log\n " }, { "path": "lib/http/views/_menu.pug", "content": "ul#menu\n li.inactive\n a(href='./inactive')\n .count 0\n | Queued\n li.active\n a.active(href='./active')\n .count 0\n | Active\n li.failed\n a(href='./failed')\n .count 0\n | Failed\n li.complete\n a(href='./complete')\n .count 0\n | Complete\n li.delayed\n a(href='./delayed')\n .count 0\n | Delayed\n" }, { "path": "lib/http/views/_row.pug", "content": "tr\n td.title\n td.value" }, { "path": "lib/http/views/_search.pug", "content": "input#search(type='text', placeholder='Search')" }, { "path": "lib/http/views/_sort.pug", "content": "select#sort\n option(value='asc') sort\n option(value='asc') asc\n option(value='desc') desc" }, { "path": "lib/http/views/job/list.pug", "content": "extends ../layout\n\nblock body\n h1 #{state}\n\n script.\n o(function(){\n init('#{state}');\n });\n\n\n #jobs\n #loading: canvas(width=50, height=50)\n" }, { "path": "lib/http/views/layout.pug", "content": "html\n head\n title= title\n link(rel='stylesheet', href='./stylesheets/main.css')\n script(src='./javascripts/utils.js')\n script(src='./javascripts/jquery.min.js')\n script(src='./javascripts/jquery.ext.js')\n script(src='./javascripts/caustic.js')\n script(src='./javascripts/progress.js')\n script(src='./javascripts/loading.js')\n script(src='./javascripts/job.js')\n script(src='./javascripts/search.js')\n script(src='./javascripts/main.js')\n body\n include _menu\n #actions\n include _search\n include _filter\n include _sort\n #content\n block body\n script(type='text/template')#job-template\n include _job\n script(type='text/template')#row-template\n include _row\n #error\n" }, { "path": "lib/kue.js", "content": "/*!\n * kue\n * Copyright (c) 2013 Automattic \n * Copyright (c) 2011 LearnBoost \n * MIT Licensed\n */\n\n/**\n * Module dependencies.\n */\n\nvar EventEmitter = require('events').EventEmitter\n , Worker = require('./queue/worker')\n , events = require('./queue/events')\n , Job = require('./queue/job')\n , Warlock = require('node-redis-warlock')\n , _ = require('lodash')\n , redis = require('./redis')\n , noop = function(){};\n\n/**\n * Expose `Queue`.\n */\n\nexports = module.exports = Queue;\n\n/**\n * Library version.\n */\n\nexports.version = require('../package.json').version;\n\n/**\n * Expose `Job`.\n */\n\nexports.Job = Job;\n\n/**\n * Server instance (that is lazily required)\n */\n\nvar app;\n\n/**\n * Expose the server.\n */\n\nObject.defineProperty(exports, 'app', {\n get: function() {\n return app || (app = require('./http'));\n }\n});\n\n/**\n * Expose the RedisClient factory.\n */\n\nexports.redis = redis;\n\n/**\n * Create a new `Queue`.\n *\n * @return {Queue}\n * @api public\n */\n\nexports.createQueue = function( options ) {\n if( !Queue.singleton ) {\n Queue.singleton = new Queue(options);\n }\n events.subscribe();\n return Queue.singleton;\n\n};\n\n/**\n * Store workers\n */\nexports.workers = [];\n\n/**\n * Initialize a new job `Queue`.\n *\n * @api public\n */\n\nfunction Queue( options ) {\n options = options || {};\n this.name = options.name || 'kue';\n this.id = [ 'kue', require('os').hostname(), process.pid ].join(':');\n this._options = options;\n this.promoter = null;\n this.workers = exports.workers;\n this.shuttingDown = false;\n Job.disableSearch = options.disableSearch !== false;\n options.jobEvents !== undefined ? Job.jobEvents = options.jobEvents : '';\n redis.configureFactory(options, this);\n this.client = Worker.client = Job.client = redis.createClient();\n}\n\n/**\n * Inherit from `EventEmitter.prototype`.\n */\n\nQueue.prototype.__proto__ = EventEmitter.prototype;\n\n/**\n * Create a `Job` with the given `type` and `data`.\n *\n * @param {String} type\n * @param {Object} data\n * @return {Job}\n * @api public\n */\n\nQueue.prototype.create =\n Queue.prototype.createJob = function( type, data ) {\n return new Job(type, data);\n };\n\n/**\n * Proxy to auto-subscribe to events.\n *\n * @api public\n */\n\nvar on = EventEmitter.prototype.on;\nQueue.prototype.on = function( event ) {\n if( 0 == event.indexOf('job') ) events.subscribe();\n return on.apply(this, arguments);\n};\n\n/**\n * Promote delayed jobs, checking every `ms`,\n * defaulting to 1 second.\n *\n * @params {Number} ms\n * @deprecated\n */\n\nQueue.prototype.promote = function( ms, l ) {\n console.warn('promote method is deprecated, you don\\'t need to call this anymore. You can safely remove it from your code now.');\n};\n\n/**\n * sets up promotion & ttl timers\n */\n\nQueue.prototype.setupTimers = function() {\n if( this.warlock === undefined ) {\n this.lockClient = redis.createClient();\n this.warlock = new Warlock(this.lockClient);\n }\n this.checkJobPromotion(this._options.promotion);\n this.checkActiveJobTtl(this._options.promotion);\n};\n\n/**\n * This new method is called by Kue when created\n *\n * Promote delayed jobs, checking every `ms`,\n * defaulting to 1 second.\n *\n * @params {Number} ms\n */\n\nQueue.prototype.checkJobPromotion = function( promotionOptions ) {\n promotionOptions = promotionOptions || {};\n var client = this.client\n , self = this\n , timeout = promotionOptions.interval || 1000\n , lockTtl = promotionOptions.lockTtl || 2000\n //, lockTtl = timeout\n , limit = promotionOptions.limit || 1000;\n clearInterval(this.promoter);\n this.promoter = setInterval(function() {\n self.warlock.lock('promotion', lockTtl, function( err, unlock ) {\n if( err ) {\n // Something went wrong and we weren't able to set a lock\n self.emit('error', err);\n return;\n }\n if( typeof unlock === 'function' ) {\n // If the lock is set successfully by this process, an unlock function is passed to our callback.\n client.zrangebyscore(client.getKey('jobs:delayed'), 0, Date.now(), 'LIMIT', 0, limit, function( err, ids ) {\n if( err || !ids.length ) return unlock();\n //TODO do a ZREMRANGEBYRANK jobs:delayed 0 ids.length-1\n var doUnlock = _.after(ids.length, unlock);\n ids.forEach(function( id ) {\n id = client.stripFIFO(id);\n Job.get(id, function( err, job ) {\n if( err ) return doUnlock();\n events.emit(id, 'promotion');\n job.inactive(doUnlock);\n });\n });\n });\n } else {\n // The lock was not established by us, be silent\n }\n });\n }, timeout);\n};\n\n\nQueue.prototype.checkActiveJobTtl = function( ttlOptions ) {\n ttlOptions = ttlOptions || {};\n var client = this.client\n , self = this\n , timeout = ttlOptions.interval || 1000\n , lockTtl = 2000\n , limit = ttlOptions.limit || 1000;\n clearInterval(this.activeJobsTtlTimer);\n this.activeJobsTtlTimer = setInterval(function() {\n self.warlock.lock('activeJobsTTL', lockTtl, function( err, unlock ) {\n if( err ) {\n // Something went wrong and we weren't able to set a lock\n self.emit('error', err);\n return;\n }\n if( typeof unlock === 'function' ) {\n // If the lock is set successfully by this process, an unlock function is passed to our callback.\n // filter only jobs set with a ttl (timestamped) between a large number and current time\n client.zrangebyscore(client.getKey('jobs:active'), 100000, Date.now(), 'LIMIT', 0, limit, function( err, ids ) {\n if( err || !ids.length ) return unlock();\n\n var idsRemaining = ids.slice();\n var doUnlock = _.after(ids.length, function(){\n self.removeAllListeners( 'job ttl exceeded ack' );\n waitForAcks && clearTimeout( waitForAcks );\n unlock && unlock();\n });\n\n self.on( 'job ttl exceeded ack', function( id ) {\n idsRemaining.splice( idsRemaining.indexOf( id ), 1 );\n doUnlock();\n });\n\n var waitForAcks = setTimeout( function(){\n idsRemaining.forEach( function( id ){\n id = client.stripFIFO(id);\n Job.get(id, function( err, job ) {\n if( err ) return doUnlock();\n job.failedAttempt( { error: true, message: 'TTL exceeded' }, doUnlock );\n });\n });\n }, 1000 );\n\n ids.forEach(function( id ) {\n id = client.stripFIFO(id);\n events.emit(id, 'ttl exceeded');\n });\n });\n } else {\n // The lock was not established by us, be silent\n }\n });\n }, timeout);\n};\n\n/**\n * Runs a LUA script to diff inactive jobs ZSET cardinality\n * and helper pop LIST length each `ms` milliseconds and syncs helper LIST.\n *\n * @param {Number} ms interval for periodical script runs\n * @api public\n */\n\nQueue.prototype.watchStuckJobs = function( ms ) {\n var client = this.client\n , self = this\n , ms = ms || 1000;\n var prefix = this.client.prefix;\n\n if( this.client.constructor.name == 'Redis' || this.client.constructor.name == 'Cluster') {\n // {prefix}:jobs format is needed in using ioredis cluster to keep they keys in same node\n prefix = '{' + prefix + '}';\n }\n var script =\n 'local msg = redis.call( \"keys\", \"' + prefix + ':jobs:*:inactive\" )\\n\\\n local need_fix = 0\\n\\\n for i,v in ipairs(msg) do\\n\\\n local queue = redis.call( \"zcard\", v )\\n\\\n local jt = string.match(v, \"' + prefix + ':jobs:(.*):inactive\")\\n\\\n local pending = redis.call( \"LLEN\", \"' + prefix + ':\" .. jt .. \":jobs\" )\\n\\\n if queue > pending then\\n\\\n need_fix = need_fix + 1\\n\\\n for j=1,(queue-pending) do\\n\\\n redis.call( \"lpush\", \"' + prefix + ':\"..jt..\":jobs\", 1 )\\n\\\n end\\n\\\n end\\n\\\n end\\n\\\n return need_fix';\n clearInterval(this.stuck_job_watch);\n client.script('LOAD', script, function( err, sha ) {\n if( err ) {\n return self.emit('error', err);\n }\n this.stuck_job_watch = setInterval(function() {\n client.evalsha(sha, 0, function( err, fixes ) {\n if( err ) return clearInterval(this.stuck_job_watch);\n }.bind(this));\n }.bind(this), ms);\n\n }.bind(this));\n};\n\n/**\n * Get setting `name` and invoke `fn(err, res)`.\n *\n * @param {String} name\n * @param {Function} fn\n * @return {Queue} for chaining\n * @api public\n */\n\nQueue.prototype.setting = function( name, fn ) {\n fn = fn || noop;\n this.client.hget(this.client.getKey('settings'), name, fn);\n return this;\n};\n\n/**\n * Process jobs with the given `type`, invoking `fn(job)`.\n *\n * @param {String} type\n * @param {Number|Function} n\n * @param {Function} fn\n * @api public\n */\n\nQueue.prototype.process = function( type, n, fn ) {\n var self = this;\n\n if( 'function' == typeof n ) fn = n, n = 1;\n\n while( n-- ) {\n var worker = new Worker(this, type).start(fn);\n worker.id = [ self.id, type, self.workers.length + 1 ].join(':');\n worker.on('error', function( err ) {\n self.emit('error', err);\n });\n worker.on('job complete', function( job ) {\n // guard against emit after shutdown\n if( self.client ) {\n self.client.incrby(self.client.getKey('stats:work-time'), job.duration, noop);\n }\n });\n // Save worker so we can access it later\n self.workers.push(worker);\n }\n this.setupTimers();\n};\n\n/**\n * Graceful shutdown\n *\n * @param {Number} timeout in milliseconds to wait for workers to finish\n * @param {String} type specific worker type to shutdown\n * @param {Function} fn callback\n * @return {Queue} for chaining\n * @api public\n */\n\nQueue.prototype.shutdown = function( timeout, type, fn ) {\n var self = this\n , n = self.workers.length;\n if( arguments.length === 1 ) {\n fn = timeout;\n type = '';\n timeout = null;\n } else if( arguments.length === 2 ) {\n fn = type;\n type = '';\n }\n var origFn = fn || function() {\n };\n\n if( this.shuttingDown && type === '' ) { // a global shutdown already has been called\n return fn(new Error('Shutdown already in progress'));\n }\n\n if( type === '' ) { // this is a global shutdown call\n this.shuttingDown = true;\n }\n\n var cleanup = function() {\n if( self.shuttingDown ) {\n self.workers = [];\n exports.workers = [];\n self.removeAllListeners();\n Queue.singleton = null;\n events.unsubscribe();\n // destroy redis client and pubsub\n redis.reset();\n self.client && self.client.quit();\n self.client = null;\n self.lockClient && self.lockClient.quit();\n self.lockClient = null;\n }\n };\n\n // Wrap `fn` to only call after all workers finished\n fn = function( err ) {\n if( err ) {\n return origFn(err);\n }\n if( !--n ) {\n cleanup();\n origFn.apply(null, arguments);\n }\n };\n\n // shut down promoter interval\n if( self.shuttingDown ) {\n if( self.promoter ) {\n clearInterval(self.promoter);\n self.promoter = null;\n }\n if( self.activeJobsTtlTimer ) {\n clearInterval(self.activeJobsTtlTimer);\n self.activeJobsTtlTimer = null;\n }\n\n }\n\n if( !self.workers.length ) {\n cleanup();\n origFn();\n } else {\n // Shut down workers 1 by 1\n self.workers.forEach(function( worker ) {\n if( self.shuttingDown || worker.type == type ) {\n worker.shutdown(timeout, fn);\n } else {\n fn && fn();\n }\n });\n }\n\n return this;\n};\n\n/**\n * Get the job types present and callback `fn(err, types)`.\n *\n * @param {Function} fn\n * @return {Queue} for chaining\n * @api public\n */\n\nQueue.prototype.types = function( fn ) {\n fn = fn || noop;\n this.client.smembers(this.client.getKey('job:types'), fn);\n return this;\n};\n\n/**\n * Return job ids with the given `state`, and callback `fn(err, ids)`.\n *\n * @param {String} state\n * @param {Function} fn\n * @return {Queue} for chaining\n * @api public\n */\n\nQueue.prototype.state = function( state, fn ) {\n var self = this;\n this.client.zrange(this.client.getKey('jobs:' + state), 0, -1, function(err,ids){\n var fixedIds = [];\n ids.forEach(function(id){\n fixedIds.push(self.client.stripFIFO(id));\n });\n fn(err,fixedIds);\n });\n return this;\n};\n\n/**\n * Get queue work time in milliseconds and invoke `fn(err, ms)`.\n *\n * @param {Function} fn\n * @return {Queue} for chaining\n * @api public\n */\n\nQueue.prototype.workTime = function( fn ) {\n this.client.get(this.client.getKey('stats:work-time'), function( err, n ) {\n if( err ) return fn(err);\n fn(null, parseInt(n, 10));\n });\n return this;\n};\n\n/**\n * Get cardinality of jobs with given `state` and `type` and callback `fn(err, n)`.\n *\n * @param {String} type\n * @param {String} state\n * @param {Function} fn\n * @return {Queue} for chaining\n * @api public\n */\n\nQueue.prototype.cardByType = function( type, state, fn ) {\n fn = fn || noop;\n this.client.zcard(this.client.getKey('jobs:' + type + ':' + state), fn);\n return this;\n};\n\n/**\n * Get cardinality of `state` and callback `fn(err, n)`.\n *\n * @param {String} state\n * @param {Function} fn\n * @return {Queue} for chaining\n * @api public\n */\n\nQueue.prototype.card = function( state, fn ) {\n fn = fn || noop;\n this.client.zcard(this.client.getKey('jobs:' + state), fn);\n return this;\n};\n\n/**\n * Completed jobs.\n * @param {Function} fn\n * @return {Queue} for chaining\n * @api public\n */\n\nQueue.prototype.complete = function( fn ) {\n return this.state('complete', fn);\n};\n\n/**\n * Failed jobs.\n * @param {Function} fn\n * @return {Queue} for chaining\n * @api public\n */\n\nQueue.prototype.failed = function( fn ) {\n return this.state('failed', fn);\n};\n\n/**\n * Inactive jobs (queued).\n * @param {Function} fn\n * @return {Queue} for chaining\n * @api public\n */\n\nQueue.prototype.inactive = function( fn ) {\n return this.state('inactive', fn);\n};\n\n/**\n * Active jobs (mid-process).\n * @param {Function} fn\n * @return {Queue} for chaining\n * @api public\n */\n\nQueue.prototype.active = function( fn ) {\n return this.state('active', fn);\n};\n\n/**\n * Delayed jobs.\n * @param {Function} fn\n * @return {Queue} for chaining\n * @api public\n */\n\nQueue.prototype.delayed = function( fn ) {\n return this.state('delayed', fn);\n};\n\n/**\n * Completed jobs of type `type` count.\n * @param {String} type is optional\n * @param {Function} fn\n * @return {Queue} for chaining\n * @api public\n */\n\nQueue.prototype.completeCount = function( type, fn ) {\n if( 1 == arguments.length ) {\n fn = type;\n return this.card('complete', fn);\n }\n return this.cardByType(type, 'complete', fn);\n};\n\n\n/**\n * Failed jobs of type `type` count.\n * @param {String} type is optional\n * @param {Function} fn\n * @return {Queue} for chaining\n * @api public\n */\n\nQueue.prototype.failedCount = function( type, fn ) {\n if( 1 == arguments.length ) {\n fn = type;\n return this.card('failed', fn);\n }\n return this.cardByType(type, 'failed', fn);\n};\n\n/**\n * Inactive jobs (queued) of type `type` count.\n * @param {String} type is optional\n * @param {Function} fn\n * @return {Queue} for chaining\n * @api public\n */\n\nQueue.prototype.inactiveCount = function( type, fn ) {\n if( 1 == arguments.length ) {\n fn = type;\n return this.card('inactive', fn);\n }\n return this.cardByType(type, 'inactive', fn);\n};\n\n/**\n * Active jobs (mid-process) of type `type` count.\n * @param {String} type is optional\n * @param {Function} fn\n * @return {Queue} for chaining\n * @api public\n */\n\nQueue.prototype.activeCount = function( type, fn ) {\n if( 1 == arguments.length ) {\n fn = type;\n return this.card('active', fn);\n }\n return this.cardByType(type, 'active', fn);\n};\n\n/**\n * Delayed jobs of type `type` count.\n * @param {String} type is optional\n * @param {Function} fn\n * @return {Queue} for chaining\n * @api public\n */\n\nQueue.prototype.delayedCount = function( type, fn ) {\n if( 1 == arguments.length ) {\n fn = type;\n return this.card('delayed', fn);\n }\n return this.cardByType(type, 'delayed', fn);\n};\n\n/**\n * Test mode for convenience in test suites\n * @api public\n */\n\nQueue.prototype.testMode = require('./queue/test_mode');\n" }, { "path": "lib/queue/events.js", "content": "/*!\n * kue - events\n * Copyright (c) 2013 Automattic \n * Copyright (c) 2011 LearnBoost \n * MIT Licensed\n */\n\n/**\n * Module dependencies.\n */\n\nvar redis = require('../redis');\n\n/**\n * Job map.\n */\n\nexports.jobs = {};\n\n/**\n * Pub/sub key.\n */\n\nexports.key = 'events';\n\n/**\n * Add `job` to the jobs map, used\n * to grab the in-process object\n * so we can emit relative events.\n *\n * @param {Job} job\n * @api private\n */\nexports.callbackQueue = [];\n\nexports.add = function( job, callback ) {\n if( job.id ) {\n if(!exports.jobs[ job.id ])\n exports.jobs[ job.id ] = [];\n\n exports.jobs[ job.id ].push(job);\n }\n// if (!exports.subscribed) exports.subscribe();\n if( !exports.subscribeStarted ) exports.subscribe();\n if( !exports.subscribed ) {\n exports.callbackQueue.push(callback);\n } else {\n callback();\n }\n};\n\n/**\n * Remove `job` from the jobs map.\n *\n * @param {Job} job\n * @api private\n */\n\nexports.remove = function( job ) {\n delete exports.jobs[ job.id ];\n};\n\n/**\n * Subscribe to \"q:events\".\n *\n * @api private\n */\n\nexports.subscribe = function() {\n// if (exports.subscribed) return;\n if( exports.subscribeStarted ) return;\n var client = redis.pubsubClient();\n client.on('message', exports.onMessage);\n client.subscribe(client.getKey(exports.key), function() {\n exports.subscribed = true;\n while( exports.callbackQueue.length ) {\n process.nextTick(exports.callbackQueue.shift());\n }\n });\n exports.queue = require('../kue').singleton;\n// exports.subscribed = true;\n exports.subscribeStarted = true;\n};\n\nexports.unsubscribe = function() {\n var client = redis.pubsubClient();\n client.unsubscribe();\n client.removeAllListeners();\n exports.subscribeStarted = false;\n};\n\n/**\n * Message handler.\n *\n * @api private\n */\n\nexports.onMessage = function( channel, msg ) {\n // TODO: only subscribe on {Queue,Job}#on()\n msg = JSON.parse(msg);\n\n // map to Job when in-process\n var jobs = exports.jobs[ msg.id ];\n if( jobs && jobs.length > 0 ) {\n for (var i = 0; i < jobs.length; i++) {\n var job = jobs[i];\n job.emit.apply(job, msg.args);\n if( [ 'complete', 'failed' ].indexOf(msg.event) !== -1 ) exports.remove(job);\n }\n }\n // emit args on Queues\n msg.args[ 0 ] = 'job ' + msg.args[ 0 ];\n msg.args.splice(1, 0, msg.id);\n if( exports.queue ) {\n exports.queue.emit.apply(exports.queue, msg.args);\n }\n};\n\n/**\n * Emit `event` for for job `id` with variable args.\n *\n * @param {Number} id\n * @param {String} event\n * @param {Mixed} ...\n * @api private\n */\n\nexports.emit = function( id, event ) {\n var client = redis.client()\n , msg = JSON.stringify({\n id: id, event: event, args: [].slice.call(arguments, 1)\n });\n client.publish(client.getKey(exports.key), msg, function () {});\n};\n" }, { "path": "lib/queue/job.js", "content": "/*!\n * kue - Job\n * Copyright (c) 2013 Automattic \n * Copyright (c) 2011 LearnBoost \n * MIT Licensed\n */\n\n/**\n * Module dependencies.\n */\n\nvar EventEmitter = require('events').EventEmitter\n , events = require('./events')\n , redis = require('../redis')\n , _ = require('lodash')\n , util = require('util')\n , noop = function() {};\n\n/**\n * Expose `Job`.\n */\n\nexports = module.exports = Job;\n\n\nexports.disableSearch = true;\n\n\nexports.jobEvents = true;\n\n/**\n * Search instance.\n */\nvar search;\nfunction getSearch() {\n if( search ) return search;\n var reds = require('reds');\n reds.createClient = require('../redis').createClient;\n return search = reds.createSearch(redis.client().getKey('search'));\n}\n\n/**\n * Default job priority map.\n */\n\nvar priorities = exports.priorities = {\n low: 10, normal: 0, medium: -5, high: -10, critical: -15\n};\n\n/**\n * Map `jobs` by the given array of `ids`.\n *\n * @param {Object} jobs\n * @param {Array} ids\n * @return {Array}\n * @api private\n */\n\nfunction map( jobs, ids ) {\n var ret = [];\n ids.forEach(function( id ) {\n if( jobs[ id ] ) ret.push(jobs[ id ]);\n });\n ret = ret.sort(function( a, b ) {\n return parseInt(a.id) - parseInt(b.id);\n });\n return ret;\n}\n\n/**\n * Return a function that handles fetching\n * of jobs by the ids fetched.\n *\n * @param {Function} fn\n * @param {String} order\n * @param {String} jobType\n * @return {Function}\n * @api private\n */\n\nfunction get( fn, order, jobType) {\n return function( err, ids ) {\n if( err ) return fn(err);\n var pending = ids.length\n , jobs = {};\n if( !pending ) return fn(null, ids);\n ids.forEach(function( id ) {\n id = redis.client().stripFIFO(id); // turn zid back to regular job id\n exports.get(id, jobType, function( err, job ) {\n if( err ) {\n console.error(err);\n } else {\n jobs[ redis.client().createFIFO(job.id) ] = job;\n }\n --pending || fn(null, 'desc' == order\n ? map(jobs, ids).reverse()\n : map(jobs, ids));\n\n });\n });\n }\n}\n\n/**\n * Get with the range `from`..`to`\n * and invoke callback `fn(err, ids)`.\n *\n * @param {Number} from\n * @param {Number} to\n * @param {String} order\n * @param {Function} fn\n * @api public\n */\n\nexports.range = function( from, to, order, fn ) {\n redis.client().zrange(redis.client().getKey('jobs'), from, to, get(fn, order));\n};\n\n/**\n * Get jobs of `state`, with the range `from`..`to`\n * and invoke callback `fn(err, ids)`.\n *\n * @param {String} state\n * @param {Number} from\n * @param {Number} to\n * @param {String} order\n * @param {Function} fn\n * @api public\n */\n\nexports.rangeByState = function( state, from, to, order, fn ) {\n redis.client().zrange(redis.client().getKey('jobs:' + state), from, to, get(fn, order));\n};\n\n/**\n * Get jobs of `type` and `state`, with the range `from`..`to`\n * and invoke callback `fn(err, ids)`.\n *\n * @param {String} type\n * @param {String} state\n * @param {Number} from\n * @param {Number} to\n * @param {String} order\n * @param {Function} fn\n * @api public\n */\n\nexports.rangeByType = function( type, state, from, to, order, fn ) {\n redis.client().zrange(redis.client().getKey('jobs:' + type + ':' + state), from, to, get(fn, order, type));\n};\n\n/**\n * Get job with `id` and callback `fn(err, job)`.\n *\n * @param {Number} id\n * @param {String} jobType is optional\n * @param {Function} fn\n * @api public\n */\n\nexports.get = function( id, jobType, fn ) {\n if (id === null || id === undefined) {\n return fn(new Error('invalid id param'));\n }\n if (typeof jobType === 'function' && !fn) {\n fn = jobType;\n jobType = '';\n }\n var client = redis.client()\n , job = new Job;\n\n job.id = id;\n job.zid = client.createFIFO(id);\n client.hgetall(client.getKey('job:' + job.id), function( err, hash ) {\n if( err ) return fn(err);\n if( !hash ) {\n exports.removeBadJob(job.id, jobType);\n return fn(new Error('job \"' + job.id + '\" doesnt exist'));\n }\n if( !hash.type ) {\n exports.removeBadJob(job.id, jobType);\n return fn(new Error('job \"' + job.id + '\" is invalid'))\n }\n // TODO: really lame, change some methods so\n // we can just merge these\n job.type = hash.type;\n job._ttl = hash.ttl;\n job._delay = hash.delay;\n job.priority(Number(hash.priority));\n job._progress = hash.progress;\n job._attempts = Number(hash.attempts);\n job._max_attempts = Number(hash.max_attempts);\n job._state = hash.state;\n job._error = hash.error;\n job.created_at = hash.created_at;\n job.promote_at = hash.promote_at;\n job.updated_at = hash.updated_at;\n job.failed_at = hash.failed_at;\n job.started_at = hash.started_at;\n job.duration = hash.duration;\n job.workerId = hash.workerId;\n job._removeOnComplete = hash.removeOnComplete;\n try {\n if( hash.data ) job.data = JSON.parse(hash.data);\n if( hash.result ) job.result = JSON.parse(hash.result);\n if( hash.progress_data ) job.progress_data = JSON.parse(hash.progress_data);\n if( hash.backoff ) {\n var source = 'job._backoff = ' + hash.backoff + ';';\n// require('vm').runInContext( source );\n eval(source);\n }\n } catch(e) {\n err = e;\n }\n fn(err, job);\n });\n};\n\n/**\n * Remove all references to an invalid job. Will remove leaky keys in redis keys:TYPE:STATE when\n * exports.rangeByType is used.\n *\n * @param {Number} id\n * @param {String} jobType\n */\n\nexports.removeBadJob = function( id, jobType) {\n var client = redis.client();\n var zid = client.createFIFO(id);\n client.multi()\n .del(client.getKey('job:' + id + ':log'))\n .del(client.getKey('job:' + id))\n .zrem(client.getKey('jobs:inactive'), zid)\n .zrem(client.getKey('jobs:active'), zid)\n .zrem(client.getKey('jobs:complete'), zid)\n .zrem(client.getKey('jobs:failed'), zid)\n .zrem(client.getKey('jobs:delayed'), zid)\n .zrem(client.getKey('jobs'), zid)\n .zrem(client.getKey('jobs:' + jobType + ':inactive'), zid)\n .zrem(client.getKey('jobs:' + jobType+ ':active'), zid)\n .zrem(client.getKey('jobs:' + jobType + ':complete'), zid)\n .zrem(client.getKey('jobs:' + jobType + ':failed'), zid)\n .zrem(client.getKey('jobs:' + jobType + ':delayed'), zid)\n .exec();\n if( !exports.disableSearch ) {\n getSearch().remove(id);\n }\n};\n\n/**\n * Remove job `id` if it exists and invoke callback `fn(err)`.\n *\n * @param {Number} id\n * @param {Function} fn\n * @api public\n */\n\nexports.remove = function( id, fn ) {\n fn = fn || noop;\n exports.get(id, function( err, job ) {\n if( err ) return fn(err);\n if( !job ) return fn(new Error('failed to find job ' + id));\n job.remove(fn);\n });\n};\n\n/**\n * Get log for job `id` and callback `fn(err, log)`.\n *\n * @param {Number} id\n * @param {Function} fn\n * @return {Type}\n * @api public\n */\n\nexports.log = function( id, fn ) {\n /*redis*/\n Job.client/*()*/.lrange(Job.client.getKey('job:' + id + ':log'), 0, -1, fn);\n};\n\n/**\n * Initialize a new `Job` with the given `type` and `data`.\n *\n * @param {String} type\n * @param {Object} data\n * @api public\n */\n\nfunction Job( type, data ) {\n this.type = type;\n this.data = data || {};\n this._max_attempts = 1;\n this._jobEvents = exports.jobEvents;\n// this.client = redis.client();\n this.client = Job.client/* || (Job.client = redis.client())*/;\n this.priority('normal');\n this.on('error', function( err ) {\n });// prevent uncaught exceptions on failed job errors\n}\n\n/**\n * Inherit from `EventEmitter.prototype`.\n */\n\nJob.prototype.__proto__ = EventEmitter.prototype;\n\n/**\n * Return JSON-friendly object.\n *\n * @return {Object}\n * @api public\n */\n\nJob.prototype.toJSON = function() {\n return {\n id: this.id\n , type: this.type\n , data: this.data\n , result: this.result\n , priority: this._priority\n , progress: this._progress || 0\n , progress_data: this.progress_data\n , state: this._state\n , error: this._error\n , created_at: this.created_at\n , promote_at: this.promote_at\n , updated_at: this.updated_at\n , failed_at: this.failed_at\n , started_at: this.started_at\n , duration: this.duration\n , delay: this._delay\n , workerId: this.workerId\n , ttl: this._ttl\n , attempts: {\n made: Number(this._attempts) || 0\n , remaining: this._attempts > 0 ? this._max_attempts - this._attempts : Number(this._max_attempts) || 1\n , max: Number(this._max_attempts) || 1\n }\n };\n};\n\n\nJob.prototype.refreshTtl = function() {\n ('active' === this.state() && this._ttl > 0)\n ?\n this.client.zadd(this.client.getKey('jobs:' + this.state()), Date.now() + parseInt(this._ttl), this.zid, noop)\n :\n noop();\n};\n\n\n/**\n * Log `str` with sprintf-style variable args or anything (objects,arrays,numbers,etc).\n *\n * Examples:\n *\n * job.log('preparing attachments');\n * job.log('sending email to %s at %s', user.name, user.email);\n * job.log({key: 'some key', value: 10});\n * job.log([1,2,3]);\n *\n * Specifiers:\n *\n * - %s : string\n * - %d : integer\n *\n * @param {String} str\n * @param {Mixed} ...\n * @return {Job} for chaining\n * @api public\n */\n\nJob.prototype.log = function( str ) {\n if(typeof str === 'string') {\n var formatted = util.format.apply(util, arguments);\n }else{\n var formatted = util.inspect(str);\n }\n this.client.rpush(this.client.getKey('job:' + this.id + ':log'), formatted, noop);\n this.set('updated_at', Date.now());\n return this;\n};\n\n/**\n * Set job `key` to `val`.\n *\n * @param {String} key\n * @param {String} val\n * @param {String} fn\n * @return {Job} for chaining\n * @api public\n */\n\nJob.prototype.set = function( key, val, fn ) {\n this.client.hset(this.client.getKey('job:' + this.id), key, val, fn || noop);\n return this;\n};\n\n/**\n * Get job `key`\n *\n * @param {String} key\n * @param {Function} fn\n * @return {Job} for chaining\n * @api public\n */\n\nJob.prototype.get = function( key, fn ) {\n this.client.hget(this.client.getKey('job:' + this.id), key, fn || noop);\n return this;\n};\n\n/**\n * Set the job progress by telling the job\n * how `complete` it is relative to `total`.\n * data can be used to pass extra data to job subscribers\n *\n * @param {Number} complete\n * @param {Number} total\n * @param {Object} data\n * @return {Job} for chaining\n * @api public\n */\n\nJob.prototype.progress = function( complete, total, data ) {\n if( 0 == arguments.length ) return this._progress;\n var n = Math.min(100, complete * 100 / total | 0);\n this.set('progress', n);\n\n // If this stringify fails because of a circular structure, even the one in events.emit would.\n // So it does not make sense to try/catch this.\n if( data ) this.set('progress_data', JSON.stringify(data));\n\n this.set('updated_at', Date.now());\n this.refreshTtl();\n events.emit(this.id, 'progress', n, data);\n return this;\n};\n\n/**\n * Set the job delay in `ms`.\n *\n * @param {Number|Date} delay in ms or execution date\n * @return {Job|Number}\n * @api public\n */\n\nJob.prototype.delay = function( ms ) {\n if( 0 == arguments.length ) return this._delay;\n if( _.isDate(ms) ) {\n ms = parseInt(ms.getTime() - Date.now())\n }\n if( ms > 0 ) {\n this._delay = ms;\n }\n return this;\n};\n\n/**\n * Sets the jobEvents flag for the job.\n * Can be used to override the global exports.jobEvents setting\n *\n * @param {Boolean} events True if job events should be emitted, false if job events should not be emitted.\n * @return {Job} Returns `this` for chaining\n */\nJob.prototype.events = function (events) {\n this._jobEvents = !!events;\n return this;\n};\n\nJob.prototype.removeOnComplete = function( param ) {\n if( 0 == arguments.length ) return this._removeOnComplete;\n this._removeOnComplete = param;\n return this;\n};\n\nJob.prototype.backoff = function( param ) {\n if( 0 == arguments.length ) return this._backoff;\n this._backoff = param;\n return this;\n};\n\n/**\n *\n * @param param\n * @returns {*}\n */\nJob.prototype.ttl = function( param ) {\n if( 0 == arguments.length ) return this._ttl;\n if( param > 0 ) {\n this._ttl = param;\n }\n return this;\n};\n\nJob.prototype._getBackoffImpl = function() {\n var self = this\n var supported_backoffs = {\n fixed: function( delay ) {\n return function( attempts ) {\n return delay;\n };\n }\n , exponential: function( delay ) {\n return function( attempts ) {\n return Math.round(Math.min(\n (self._backoff.maxDelay || Infinity) - (delay * 0.5),\n delay * 0.5 * ( Math.pow(2, attempts) - 1)\n ));\n };\n }\n };\n if( _.isPlainObject(this._backoff) ) {\n return supported_backoffs[ this._backoff.type ](this._backoff.delay || this._delay);\n } else {\n return this._backoff;\n }\n};\n\n/**\n * Set or get the priority `level`, which is one\n * of \"low\", \"normal\", \"medium\", and \"high\", or\n * a number in the range of -10..10.\n *\n * @param {String|Number} level\n * @return {Job|Number} for chaining\n * @api public\n */\n\nJob.prototype.priority = function( level ) {\n if( 0 == arguments.length ) return this._priority;\n this._priority = null == priorities[ level ]\n ? level\n : priorities[ level ];\n return this;\n};\n\n/**\n * Increment attempts, invoking callback `fn(remaining, attempts, max)`.\n *\n * @param {Function} fn\n * @return {Job} for chaining\n * @api public\n */\n\nJob.prototype.attempt = function( fn ) {\n var client = this.client\n , id = this.id\n , key = client.getKey('job:' + id);\n\n this._attempts = this._attempts || 0;\n if( this._attempts < this._max_attempts ) {\n client.hincrby(key, 'attempts', 1, function( err, attempts ) {\n this._attempts = attempts;\n fn(err, Math.max(0, this._max_attempts - attempts), attempts, this._max_attempts);\n }.bind(this));\n } else {\n fn(null, 0, this._attempts, this._max_attempts);\n }\n return this;\n};\n\n\n/**\n * Try to reattempt the job seand called onFailedAttempt, or call onFailed\n * @param remaining total left attempts\n * @param attempts\n * @param onFailedAttempt\n * @param onFailed\n * @param clbk\n */\n\nJob.prototype.reattempt = function( attempts, clbk ) {\n clbk = clbk || noop;\n if( this.backoff() ) {\n var delay = this.delay();\n if( _.isFunction(this._getBackoffImpl()) ) {\n try {\n delay = this._getBackoffImpl().apply(this, [ attempts, delay ]);\n } catch(e) {\n clbk(e);\n }\n }\n var self = this;\n this.delay(delay).update(function( err ) {\n if( err ) return clbk(err);\n self.delayed(clbk);\n });\n } else {\n this.inactive(clbk);\n }\n};\n\n/**\n * Set max attempts to `n`.\n *\n * @param {Number} n\n * @return {Job} for chaining\n * @api public\n */\n\nJob.prototype.attempts = function( n ) {\n this._max_attempts = n;\n return this;\n};\n\n\nJob.prototype.failedAttempt = function( theErr, fn ) {\n this.error(theErr).failed(function() {\n this.attempt(function( error, remaining, attempts/*, max*/ ) {\n if( error ) {\n this.emit( 'error', error );\n return fn && fn( error );\n }\n if( remaining > 0 ) {\n this.reattempt(attempts, function( err ) {\n if( err ) {\n this.emit( 'error', err );\n return fn && fn( err );\n }\n fn && fn( err, true, attempts );\n }.bind(this));\n } else if( remaining === 0 ) {\n fn && fn( null, false, attempts );\n } else {\n fn && fn( new Error('Attempts Exceeded') );\n }\n }.bind(this));\n }.bind(this));\n return this;\n};\n\nJob.prototype.searchKeys = function( keys ) {\n if( 0 == arguments.length ) return this._searchKeys;\n this._searchKeys = keys || [];\n if( !_.isArray(this._searchKeys) ) {\n this._searchKeys = [ this._searchKeys ];\n }\n return this;\n};\n\n/**\n * Remove the job and callback `fn(err)`.\n *\n * @param {Function} fn\n * @return {Job} for chaining\n * @api public\n */\n\nJob.prototype.remove = function( fn ) {\n var client = this.client;\n client.multi()\n .zrem(client.getKey('jobs:' + this.state()), this.zid)\n .zrem(client.getKey('jobs:' + this.type + ':' + this.state()), this.zid)\n .zrem(client.getKey('jobs'), this.zid)\n .del(client.getKey('job:' + this.id + ':log'))\n .del(client.getKey('job:' + this.id))\n .exec(function( err ) {\n// events.remove(this);\n events.emit(this.id, 'remove', this.type);\n if( !exports.disableSearch ) {\n getSearch().remove(this.id, fn);\n } else {\n fn && fn(err);\n }\n }.bind(this));\n return this;\n};\n\n/**\n * Set state to `state`.\n *\n * @param {String} state\n * @param fn\n * @return {Job} for chaining\n * @api public\n */\nJob.prototype.state = function( state, fn ) {\n if( 0 == arguments.length ) return this._state;\n var client = this.client\n , fn = fn || noop;\n var oldState = this._state;\n var multi = client.multi();\n if( oldState && oldState != '' && oldState != state ) {\n multi\n .zrem(client.getKey('jobs:' + oldState), this.zid)\n .zrem(client.getKey('jobs:' + this.type + ':' + oldState), this.zid);\n }\n multi\n .hset(client.getKey('job:' + this.id), 'state', state)\n .zadd(client.getKey('jobs:' + state), this._priority, this.zid)\n .zadd(client.getKey('jobs:' + this.type + ':' + state), this._priority, this.zid);\n\n // use promote_at as score when job moves to delayed\n ('delayed' === state) ? multi.zadd(client.getKey('jobs:' + state), parseInt(this.promote_at), this.zid) : noop();\n ('active' === state && this._ttl > 0) ? multi.zadd(client.getKey('jobs:' + state), Date.now() + parseInt(this._ttl), this.zid) : noop();\n ('active' === state && !this._ttl) ? multi.zadd(client.getKey('jobs:' + state), this._priority<0?this._priority:-this._priority, this.zid) : noop();\n ('inactive' === state) ? multi.lpush(client.getKey(this.type + ':jobs'), 1) : noop();\n\n this.set('updated_at', Date.now());\n this._state = state;\n multi.exec(function( err, replies ) {\n if( !err ) {\n (this._state === 'inactive') ? events.emit(this.id, 'enqueue', this.type) : noop();\n }\n return fn(err);\n }.bind(this));\n return this;\n};\n\n/**\n * Set the job's failure `err`.\n *\n * @param {Error} err\n * @return {Job} for chaining\n * @api public\n */\n\nJob.prototype.error = function( err ) {\n var str, summary;\n if( 0 == arguments.length ) return this._error;\n\n if( 'string' == typeof err ) {\n str = err;\n summary = '';\n } else {\n if( err.stack && 'string' === typeof err.stack ) {\n str = err.stack\n } else { //TODO what happens to CallSite[] err.stack?\n str = err.message\n }\n summary = ('string' === typeof str) ? str.split('\\n')[ 0 ] : '';\n }\n this.set('error', str);\n this.log('%s', summary);\n events.emit(this.id, 'error', str);\n return this;\n};\n\n/**\n * Set state to \"complete\", and progress to 100%.\n */\n\nJob.prototype.complete = function( clbk ) {\n return this.set('progress', 100).state('complete', clbk);\n};\n\n/**\n * Set state to \"failed\".\n */\n\nJob.prototype.failed = function( clbk ) {\n this.failed_at = Date.now();\n return this.set('failed_at', this.failed_at).state('failed', clbk);\n};\n\n/**\n * Set state to \"inactive\".\n */\n\nJob.prototype.inactive = function( clbk ) {\n return this.state('inactive', clbk);\n};\n\n/**\n * Set state to \"active\".\n */\n\nJob.prototype.active = function( clbk ) {\n return this.state('active', clbk);\n};\n\n/**\n * Set state to \"delayed\".\n */\n\nJob.prototype.delayed = function( clbk ) {\n return this.state('delayed', clbk);\n};\n\n/**\n * Save the job, optionally invoking the callback `fn(err)`.\n *\n * @param {Function} fn\n * @return {Job} for chaining\n * @api public\n */\n\nJob.prototype.save = function( fn ) {\n var client = this.client\n , fn = fn || noop\n , max = this._max_attempts\n , self = this;\n\n // update\n if( this.id ) return this.update(fn);\n\n // incr id\n client.incr(client.getKey('ids'), function( err, id ) {\n if( err ) return fn(err);\n // add the job for event mapping\n self.id = id;\n self.zid = client.createFIFO(id);\n self.subscribe(function() {\n self._state = self._state || (this._delay ? 'delayed' : 'inactive');\n if( max ) { self.set('max_attempts', max); }\n client.sadd(client.getKey('job:types'), self.type, noop);\n self.set('type', self.type);\n var now = Date.now();\n self.created_at = now;\n self.set('created_at', self.created_at);\n self.promote_at = now + (self._delay || 0);\n self.set('promote_at', self.promote_at);\n self.update(fn);\n }.bind(this));\n }.bind(this));\n return this;\n};\n\n/**\n * Update the job and callback `fn(err)`.\n *\n * @param {Function} fn\n * @api public\n */\n\nJob.prototype.update = function( fn ) {\n var json;\n\n // serialize json data\n try {\n json = JSON.stringify(this.data);\n } catch(err) {\n fn(err);\n return this;\n }\n\n // delay\n if( this._delay ) {\n this.set('delay', this._delay);\n if( this.created_at ) {\n var timestamp = parseInt(this.failed_at || this.created_at, 10)\n , delay = parseInt(this._delay);\n this.promote_at = timestamp + delay;\n this.set('promote_at', this.promote_at);\n }\n }\n if( this._ttl ) {\n this.set('ttl', this._ttl);\n }\n if( this._removeOnComplete ) this.set('removeOnComplete', this._removeOnComplete);\n if( this._backoff ) {\n if( _.isPlainObject(this._backoff) ) this.set('backoff', JSON.stringify(this._backoff));\n else this.set('backoff', this._backoff.toString());\n }\n\n // updated timestamp\n this.set('updated_at', Date.now());\n this.refreshTtl();\n\n // priority\n this.set('priority', this._priority);\n\n this.client.zadd(this.client.getKey('jobs'), this._priority, this.zid, noop);\n\n // data\n this.set('data', json, function() {\n // state\n this.state(this._state, fn);\n }.bind(this));\n\n if( !exports.disableSearch ) {\n if( this.searchKeys() ) {\n this.searchKeys().forEach(function( key ) {\n var value = _.get(this.data, key);\n if( !_.isString(value) ) {\n value = JSON.stringify(value);\n }\n getSearch().index(value, this.id);\n }.bind(this));\n } else {\n getSearch().index(json, this.id);\n }\n }\n return this;\n};\n\n/**\n * Subscribe this job for event mapping.\n *\n * @return {Job} for chaining\n * @api public\n */\n\nJob.prototype.subscribe = function( callback ) {\n if( this._jobEvents ) {\n events.add(this, callback);\n } else {\n callback && callback();\n }\n return this;\n};\n" }, { "path": "lib/queue/test_mode.js", "content": "var Job = require('./job'),\n _ = require('lodash');\n\nvar originalJobSave = Job.prototype.save,\n originalJobUpdate = Job.prototype.update,\n processQueue,\n jobs;\n\nfunction testJobSave( fn ) {\n if(processQueue) {\n jobs.push(this);\n originalJobSave.call(this, fn);\n } else {\n this.id = _.uniqueId();\n jobs.push(this);\n if( _.isFunction(fn) ) fn(); \n }\n};\n\nfunction testJobUpdate( fn ) {\n if(processQueue) {\n originalJobUpdate.call(this, fn);\n } else {\n if( _.isFunction(fn) ) fn();\n }\n};\n\n/**\n * Array of jobs added to the queue\n * @api public\n */\n\nmodule.exports.jobs = jobs = [];\nmodule.exports.processQueue = processQueue = false;\n\n/**\n * Enable test mode.\n * @api public\n */\n\nmodule.exports.enter = function(process) {\n processQueue = process || false;\n Job.prototype.save = testJobSave;\n Job.prototype.update = testJobUpdate;\n};\n\n/**\n * Disable test mode.\n * @api public\n */\n\nmodule.exports.exit = function() {\n Job.prototype.save = originalJobSave;\n Job.prototype.update = originalJobUpdate;\n};\n\n/**\n * Clear the array of queued jobs\n * @api public\n */\n\nmodule.exports.clear = function() {\n jobs.length = 0;\n};\n" }, { "path": "lib/queue/worker.js", "content": "/*!\n * kue - Worker\n * Copyright (c) 2013 Automattic \n * Copyright (c) 2011 LearnBoost \n * MIT Licensed\n * Author: behradz@gmail.com\n */\n\n/**\n * Module dependencies.\n */\n\nvar EventEmitter = require('events').EventEmitter\n , redis = require('../redis')\n , events = require('./events')\n , Job = require('./job')\n , noop = function() {};\n\n/**\n * Expose `Worker`.\n */\n\nmodule.exports = Worker;\n\n/**\n * Redis connections used by `getJob()` when blocking.\n */\n\nvar clients = {};\n\n/**\n * Initialize a new `Worker` with the given Queue\n * targetting jobs of `type`.\n *\n * @param {Queue} queue\n * @param {String} type\n * @api private\n */\n\nfunction Worker( queue, type ) {\n this.queue = queue;\n this.type = type;\n this.client = Worker.client || (Worker.client = redis.createClient());\n this.running = true;\n this.job = null;\n}\n\n/**\n * Inherit from `EventEmitter.prototype`.\n */\n\nWorker.prototype.__proto__ = EventEmitter.prototype;\n\n/**\n * Start processing jobs with the given `fn`,\n *\n * @param {Function} fn\n * @return {Worker} for chaining\n * @api private\n */\n\nWorker.prototype.start = function( fn ) {\n var self = this;\n self.idle();\n if( !self.running ) return;\n\n if (self.ttlExceededCb)\n self.queue.removeListener('job ttl exceeded', self.ttlExceededCb);\n\n self.ttlExceededCb = function(id) {\n if( self.job && self.job.id && self.job.id === id ) {\n self.failed( self.job, { error: true, message: 'TTL exceeded' }, fn );\n events.emit(id, 'ttl exceeded ack');\n }\n }\n\n /*\n listen if current job ttl received,\n so that this worker can fail current stuck job and continue,\n in case user's process callback is stuck and done is not called in time\n */\n this.queue.on( 'job ttl exceeded', self.ttlExceededCb);\n\n self.getJob(function( err, job ) {\n if( err ) self.error(err, job);\n if( !job || err ) return process.nextTick(function() {\n self.start(fn);\n });\n self.process(job, fn);\n });\n return this;\n};\n\n/**\n * Error handler, currently does nothing.\n *\n * @param {Error} err\n * @param {Job} job\n * @return {Worker} for chaining\n * @api private\n */\n\nWorker.prototype.error = function( err, job ) {\n this.emit('error', err, job);\n return this;\n};\n\n/**\n * Process a failed `job`. Set's the job's state\n * to \"failed\" unless more attempts remain, in which\n * case the job is marked as \"inactive\" or \"delayed\"\n * and remains in the queue.\n *\n * @param {Job} job\n * @param {Object} theErr\n * @param {Function} fn\n * @return {Worker} for chaining\n * @api private\n */\n\nWorker.prototype.failed = function( job, theErr, fn ) {\n var self = this;\n job.failedAttempt( theErr, function( err, hasAttempts, attempt ) {\n if( err ) return self.error(err, job);\n if( hasAttempts ) {\n self.emitJobEvent( 'failed attempt', job, theErr.message || theErr.toString(), attempt );\n } else {\n self.emitJobEvent( 'failed', job, theErr.message || theErr.toString() );\n }\n fn && self.start(fn);\n });\n return this;\n};\n\n/**\n * Process `job`, marking it as active,\n * invoking the given callback `fn(job)`,\n * if the job fails `Worker#failed()` is invoked,\n * otherwise the job is marked as \"complete\".\n *\n * @param {Job} job\n * @param {Function} fn\n * @return {Worker} for chaining\n * @api public\n */\n\nWorker.prototype.process = function( job, fn ) {\n var self = this\n , start = new Date();\n\n this.job = job;\n job.set( 'started_at', job.started_at = start.getTime() );\n job.set( 'workerId', job.workerId = this.id );\n /*\n store job.id around given done to the caller,\n so that we can later match against it when done is called\n */\n var createDoneCallback = function( jobId ) {\n return function( err, result ) {\n if( self.drop_user_callbacks ) {\n //console.warn( 'Worker started to shutdown, ignoring execution of done callback' );\n //job.log( 'Worker started to shutdown, ignoring execution of done callback' );\n return;\n }\n /*\n if no job in hand, or the current job in hand\n doesn't match called done callback's jobId\n then ignore running callers done.\n */\n if( self.job === null || self.job && self.job.id && self.job.id !== jobId ) {\n //console.warn( 'This job has already been finished, ignoring execution of done callback' );\n //job.log( 'This job has already been finished, ignoring execution of done callback' );\n return;\n }\n if( err ) {\n return self.failed(job, err, fn);\n }\n job.set('duration', job.duration = new Date - start);\n if( result ) {\n try {\n job.result = result;\n job.set('result', JSON.stringify(result), noop);\n } catch(e) {\n job.set('result', JSON.stringify({ error: true, message: 'Invalid JSON Result: \"' + result + '\"' }), noop);\n }\n }\n job.complete(function() {\n job.attempt(function() {\n if( job.removeOnComplete() ) {\n job.remove();\n }\n self.emitJobEvent('complete', job, result);\n self.start(fn);\n });\n }.bind(this));\n };\n };\n\n var doneCallback = createDoneCallback( job.id );\n\n var workerCtx = {\n /**\n * @author behrad\n * @pause: let the processor to tell worker not to continue processing new jobs\n */\n pause: function( timeout, fn ) {\n if( arguments.length === 1 ) {\n fn = timeout;\n timeout = 5000;\n }\n self.queue.shutdown(Number(timeout), self.type, fn);\n },\n /**\n * @author behrad\n * @pause: let the processor to trigger restart for they job processing\n */\n resume: function() {\n if( self.resume() ) {\n self.start(fn);\n }\n },\n shutdown: function() {\n self.shutdown();\n }\n };\n\n job.active(function() {\n self.emitJobEvent('start', job, job.type);\n if( fn.length === 2 ) { // user provided a two argument function, doesn't need workerCtx\n fn(job, doneCallback);\n } else { // user wants workerCtx parameter, make done callback the last\n fn(job, workerCtx, doneCallback);\n }\n }.bind(this));\n\n return this;\n};\n\n/**\n * Atomic ZPOP implementation.\n *\n * @param {String} key\n * @param {Function} fn\n * @api private\n */\n\nWorker.prototype.zpop = function( key, fn ) {\n this.client\n .multi()\n .zrange(key, 0, 0)\n .zremrangebyrank(key, 0, 0)\n .exec(function( err, res ) {\n if( err || !res || !res[ 0 ] || !res[ 0 ].length ) return fn(err);\n var id = res[ 0 ][ 0 ] || res[ 0 ][ 1 ][ 0 ];\n fn(null, this.client.stripFIFO(id));\n }.bind(this));\n};\n\n/**\n * Attempt to fetch the next job.\n *\n * @param {Function} fn\n * @api private\n */\n\nWorker.prototype.getJob = function( fn ) {\n var self = this;\n if( !self.running ) {\n return fn('Already Shutdown');\n }\n // alloc a client for this job type\n var client = clients[ self.type ] || (clients[ self.type ] = redis.createClient());\n // BLPOP indicates we have a new inactive job to process\n client.blpop(client.getKey(self.type + ':jobs'), 0, function( err ) {\n if( err || !self.running ) {\n if( self.client && self.client.connected && !self.client.closing ) {\n self.client.lpush(self.client.getKey(self.type + ':jobs'), 1, noop);\n }\n return fn(err);\t\t// SAE: Added to avoid crashing redis on zpop\n }\n // Set job to a temp value so shutdown() knows to wait\n self.job = true;\n self.zpop(self.client.getKey('jobs:' + self.type + ':inactive'), function( err, id ) {\n if( err || !id ) {\n self.idle();\n return fn(err /*|| \"No job to pop!\"*/);\n }\n Job.get(id, fn);\n });\n });\n};\n\n/**\n * emits worker idle event and nullifies current job in hand\n */\n\nWorker.prototype.idle = function() {\n this.job = null;\n this.emit('idle');\n return this;\n};\n\n/**\n * Gracefully shut down the worker\n *\n * @param {Function} fn\n * @param {int} timeout\n * @api private\n */\n\nWorker.prototype.shutdown = function( timeout, fn ) {\n var self = this, shutdownTimer = null;\n if( arguments.length === 1 ) {\n fn = timeout;\n timeout = null;\n }\n\n // Wrap `fn` so we don't pass `job` to it\n var _fn = function( job ) {\n if( job && self.job && job.id != self.job.id ) {\n return; // simply ignore older job events currently being received until the right one comes...\n }\n shutdownTimer && clearTimeout(shutdownTimer);\n self.removeAllListeners();\n self.job = null;\n //Safeyly kill any blpop's that are waiting.\n (self.type in clients) && clients[ self.type ].quit();\n delete clients[ self.type ];\n self.cleaned_up = true;\n //fix half-blob job fetches if any\n self.client.lpush(self.client.getKey(self.type + ':jobs'), 1, fn || noop);\n };\n\n if( !this.running ) return _fn();\n this.running = false;\n\n // As soon as we're free, signal that we're done\n if( !this.job ) {\n return _fn();\n }\n this.on('idle', _fn);\n this.on('job complete', _fn);\n this.on('job failed', _fn);\n this.on('job failed attempt', _fn);\n\n if( timeout ) {\n shutdownTimer = setTimeout(function() {\n // shutdown timeout reached...\n if( self.job ) {\n self.drop_user_callbacks = true;\n self.removeAllListeners();\n if( self.job === true ) {\n self.once('idle', _fn);\n } else {\n // a job is running, fail it and call _fn when failed\n self.once('job failed', _fn);\n self.once('job failed attempt', _fn);\n self.failed(self.job, { error: true, message: 'Shutdown' });\n }\n } else {\n // no job running, just finish immediately\n _fn();\n }\n }.bind(this), timeout);\n }\n};\n\nWorker.prototype.emitJobEvent = function( event, job, arg1, arg2 ) {\n if( this.cleaned_up ) return;\n events.emit(job.id, event, arg1, arg2);\n this.emit('job ' + event, job);\n};\n\nWorker.prototype.resume = function() {\n if( this.running ) return false;\n this.cleaned_up = false;\n this.drop_user_callbacks = false;\n this.running = true;\n return true;\n};\n" }, { "path": "lib/redis.js", "content": "/*!\n * kue - RedisClient factory\n * Copyright (c) 2013 Automattic \n * Copyright (c) 2011 LearnBoost \n * MIT Licensed\n * Author: behradz@gmail.com\n */\n\n/**\n * Module dependencies.\n */\n\nvar redis = require('redis');\nvar url = require('url');\n\n/**\n *\n * @param options\n * @param queue\n */\n\nexports.configureFactory = function( options, queue ) {\n options.prefix = options.prefix || 'q';\n\n if( typeof options.redis === 'string' ) {\n // parse the url\n var conn_info = url.parse(options.redis, true /* parse query string */);\n if( conn_info.protocol !== 'redis:' ) {\n throw new Error('kue connection string must use the redis: protocol');\n }\n\n options.redis = {\n port: conn_info.port || 6379,\n host: conn_info.hostname,\n db: (conn_info.pathname ? conn_info.pathname.substr(1) : null) || conn_info.query.db || 0,\n // see https://github.com/mranney/node_redis#rediscreateclient\n options: conn_info.query\n };\n\n if( conn_info.auth ) {\n options.redis.auth = conn_info.auth.replace(/.*?:/, '');\n }\n\n }\n\n options.redis = options.redis || {};\n\n // guarantee that redis._client has not been populated.\n // may warrant some more testing - i was running into cases where shutdown\n // would call redis.reset but an event would be emitted after the reset\n // which would re-create the client and cache it in the redis module.\n exports.reset();\n\n /**\n * Create a RedisClient.\n *\n * @return {RedisClient}\n * @api private\n */\n exports.createClient = function() {\n var clientFactoryMethod = options.redis.createClientFactory || exports.createClientFactory;\n var client = clientFactoryMethod(options);\n\n client.on('error', function( err ) {\n queue.emit('error', err);\n });\n\n client.prefix = options.prefix;\n\n // redefine getKey to use the configured prefix\n client.getKey = function( key ) {\n if( client.constructor.name == 'Redis' || client.constructor.name == 'Cluster') {\n // {prefix}:jobs format is needed in using ioredis cluster to keep they keys in same node\n // otherwise multi commands fail, since they use ioredis's pipeline.\n return '{' + this.prefix + '}:' + key;\n }\n return this.prefix + ':' + key;\n };\n\n client.createFIFO = function( id ) {\n //Create an id for the zset to preserve FIFO order\n var idLen = '' + id.toString().length;\n var len = 2 - idLen.length;\n while (len--) idLen = '0' + idLen;\n return idLen + '|' + id;\n };\n\n // Parse out original ID from zid\n client.stripFIFO = function( zid ) {\n if ( typeof zid === 'string' ) {\n return +zid.substr(zid.indexOf('|')+1);\n } else {\n // Sometimes this gets called with an undefined\n // it seems to be OK to have that not resolve to an id\n return zid;\n }\n };\n\n return client;\n };\n};\n\n/**\n * Create a RedisClient from options\n * @param options\n * @return {RedisClient}\n * @api private\n */\n\nexports.createClientFactory = function( options ) {\n var socket = options.redis.socket;\n var port = !socket ? (options.redis.port || 6379) : null;\n var host = !socket ? (options.redis.host || '127.0.0.1') : null;\n var db = !socket ? (options.redis.db || 0) : null;\n var client = redis.createClient(socket || port, host, options.redis.options);\n if( options.redis.auth ) {\n client.auth(options.redis.auth);\n }\n if( db >= 0 ){\n client.select(db);\n }\n return client;\n};\n\n/**\n * Create or return the existing RedisClient.\n *\n * @return {RedisClient}\n * @api private\n */\n\nexports.client = function() {\n return exports._client || (exports._client = exports.createClient());\n};\n\n/**\n * Return the pubsub-specific redis client.\n *\n * @return {RedisClient}\n * @api private\n */\n\nexports.pubsubClient = function() {\n return exports._pubsub || (exports._pubsub = exports.createClient());\n};\n\n/**\n * Resets internal variables to initial state\n *\n * @api private\n */\nexports.reset = function() {\n exports._client && exports._client.quit();\n exports._pubsub && exports._pubsub.quit();\n exports._client = null;\n exports._pubsub = null;\n};\n" }, { "path": "package.json", "content": "{\n \"name\": \"kue\",\n \"version\": \"0.11.5\",\n \"description\": \"Feature rich priority job queue backed by redis\",\n \"homepage\": \"http://automattic.github.io/kue/\",\n \"keywords\": [\n \"job\",\n \"queue\",\n \"worker\",\n \"redis\"\n ],\n \"license\": \"MIT\",\n \"author\": \"TJ Holowaychuk \",\n \"contributors\": [\n {\n \"name\": \"Behrad Zari\",\n \"email\": \"behradz@gmail.com\"\n }\n ],\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"https://github.com/Automattic/kue.git\"\n },\n \"bugs\": {\n \"url\": \"https://github.com/Automattic/kue/issues\"\n },\n \"dependencies\": {\n \"body-parser\": \"^1.12.2\",\n \"express\": \"^4.12.2\",\n \"lodash\": \"^4.0.0\",\n \"nib\": \"~1.1.2\",\n \"node-redis-warlock\": \"~0.2.0\",\n \"pug\": \"^2.0.0-beta3\",\n \"redis\": \"~2.6.0-2\",\n \"stylus\": \"~0.54.5\",\n \"yargs\": \"^4.0.0\"\n },\n \"devDependencies\": {\n \"async\": \"^1.4.2\",\n \"chai\": \"^3.3.0\",\n \"coffee-script\": \"~1.10.0\",\n \"mocha\": \"^2.3.3\",\n \"should\": \"^3.1.0\",\n \"sinon\": \"^1.17.2\",\n \"supertest\": \"^1.1.0\"\n },\n \"main\": \"index\",\n \"bin\": {\n \"kue-dashboard\": \"bin/kue-dashboard\"\n },\n \"scripts\": {\n \"test\": \"make test-all\"\n },\n \"optionalDependencies\": {\n \"reds\": \"^0.2.5\"\n }\n}\n" }, { "path": "test/jsonapi.js", "content": "var request = require( 'supertest' ),\r\n kue = require( '../index' ),\r\n async = require( 'async' ),\r\n chai = require( 'chai' ),\r\n queue = kue.createQueue( { disableSearch: false } ), //customize queue before accessing kue.app\r\n app = kue.app,\r\n type = 'test:inserts';\r\n\r\n\r\nexpect = chai.expect;\r\n\r\n\r\nfunction jobsPopulate( count ) {\r\n var priority = [ 10, 0, -5, -10, -15 ],\r\n jobs = [];\r\n\r\n for ( var i = 0; i < count; i++ ) {\r\n jobs.push( {\r\n type: type,\r\n data: {\r\n title: i,\r\n data: type + ':data'\r\n },\r\n options: {\r\n // random priority\r\n priority: priority[ Math.floor( Math.random() * 5 ) ]\r\n }\r\n } );\r\n }\r\n\r\n // return array only if length > 1\r\n return jobs.length === 1 ? jobs[ 0 ] : jobs;\r\n}\r\n\r\n\r\ndescribe( 'JSON API', function () {\r\n var scope = {};\r\n\r\n\r\n before( function ( done ) {\r\n scope.queue = queue;\r\n\r\n // delete all jobs to get a clean state\r\n kue.Job.rangeByType( type, 'inactive', 0, 100, 'asc', function ( err, jobs ) {\r\n if ( err ) return done( err );\r\n if ( !jobs.length ) return done();\r\n async.each( jobs, function ( job, asyncDone ) {\r\n job.remove( asyncDone );\r\n }, done );\r\n } );\r\n } );\r\n\r\n\r\n after( function ( done ) {\r\n scope.queue.shutdown( 200, function ( err ) {\r\n scope.queue = null;\r\n done( err );\r\n } );\r\n } );\r\n\r\n\r\n describe( 'create, get, update and delete', function () {\r\n it( 'should insert a job and respond with an id', function ( done ) {\r\n request( app )\r\n .post( '/job' )\r\n .send( jobsPopulate( 1 ) )\r\n .expect( 200 )\r\n .expect( function ( res ) {\r\n res.body.message.should.equal( 'job created' );\r\n res.body.id.should.be.a.Number;\r\n Object.keys( res.body ).should.have.lengthOf( 2 );\r\n\r\n scope.jobId = res.body.id;\r\n } )\r\n .end( done );\r\n } );\r\n\r\n\r\n it( 'should insert multiple jobs and respond with ids', function ( done ) {\r\n var jobCount = 5;\r\n\r\n request( app )\r\n .post( '/job' )\r\n .send( jobsPopulate( jobCount ) )\r\n .expect( 200 )\r\n .expect( function ( res ) {\r\n var created = res.body;\r\n created.should.be.ok;\r\n created.length.should.equal( jobCount );\r\n\r\n for ( var i = 0; i < jobCount; i++ ) {\r\n var job = created[ i ];\r\n job.message.should.be.equal( 'job created' );\r\n job.id.should.be.a.Number;\r\n Object.keys( job ).should.have.lengthOf( 2 );\r\n }\r\n } )\r\n .end( done );\r\n } );\r\n\r\n\r\n it( 'get job by id: job is inactive', function ( done ) {\r\n request( app )\r\n .get( '/job/' + scope.jobId )\r\n .expect( function ( res ) {\r\n res.body.id.should.eql( scope.jobId );\r\n res.body.type.should.eql( type );\r\n res.body.state.should.eql( 'inactive' );\r\n } )\r\n .end( done );\r\n } );\r\n\r\n\r\n it( 'change state', function ( done ) {\r\n request( app )\r\n .put( '/job/' + scope.jobId + '/state/active' )\r\n .expect( function ( res ) {\r\n expect( res.body.message ).to.exist;\r\n } )\r\n .end( done );\r\n } );\r\n\r\n\r\n it( 'get job by id: job is now active', function ( done ) {\r\n request( app )\r\n .get( '/job/' + scope.jobId )\r\n .expect( function ( res ) {\r\n res.body.id.should.eql( scope.jobId );\r\n res.body.type.should.eql( type );\r\n res.body.state.should.eql( 'active' );\r\n } )\r\n .end( done );\r\n } );\r\n\r\n\r\n it( 'delete job by id', function ( done ) {\r\n request( app )\r\n .del( '/job/' + scope.jobId )\r\n .expect( function ( res ) {\r\n expect( res.body.message ).to.contain( scope.jobId );\r\n } )\r\n .end( done );\r\n } );\r\n } );\r\n\r\n\r\n describe( 'search', function () {\r\n it( 'search by query: not found', function ( done ) {\r\n request( app )\r\n .get( '/job/search' )\r\n .query( {} )\r\n .expect( function ( res ) {\r\n res.body.length.should.eql( 0 );\r\n } )\r\n .end( done );\r\n } );\r\n\r\n\r\n it( 'search by query: found', function ( done ) {\r\n request( app )\r\n .get( '/job/search' )\r\n .query( {\r\n q: type + ':data'\r\n } )\r\n .expect( function ( res ) {\r\n // we created 6 jobs, one was deleted, 5 left\r\n res.body.length.should.eql( 5 );\r\n } )\r\n .end( done );\r\n } );\r\n } );\r\n\r\n\r\n describe( 'range', function () {\r\n it( 'range from...to', function ( done ) {\r\n request( app )\r\n .get( '/jobs/0..3' )\r\n .expect( function ( res ) {\r\n res.body.length.should.eql( 4 );\r\n } )\r\n .end( done );\r\n } );\r\n\r\n\r\n it( 'range from...to with type and state', function ( done ) {\r\n request( app )\r\n .get( '/jobs/' + type + '/inactive/0..20/asc' )\r\n .expect( function ( res ) {\r\n res.body.length.should.eql( 5 );\r\n } )\r\n .end( done );\r\n } );\r\n } );\r\n\r\n\r\n describe( 'stats', function () {\r\n it( 'get stats', function ( done ) {\r\n request( app )\r\n .get( '/stats' )\r\n .expect( function ( res ) {\r\n expect( res.body.inactiveCount ).to.exist;\r\n expect( res.body.completeCount ).to.exist;\r\n expect( res.body.activeCount ).to.exist;\r\n expect( res.body.delayedCount ).to.exist;\r\n } )\r\n .end( done );\r\n } );\r\n } );\r\n\r\n\r\n describe( 'error cases', function () {\r\n it( 'should return 204 status code when POST /job body is empty', function ( done ) {\r\n request( app )\r\n .post( '/job' )\r\n .send( [] )\r\n .expect( 204 )\r\n .expect( function ( res ) {\r\n res.text.should.have.lengthOf( 0 );\r\n } )\r\n .end( done );\r\n } );\r\n\r\n it( 'should insert jobs including an invalid job, respond with ids and error', function ( done ) {\r\n var jobs = jobsPopulate( 3 );\r\n delete jobs[ 1 ].type;\r\n\r\n request( app )\r\n .post( '/job' )\r\n .send( jobs )\r\n .expect( 400 ) // Expect a bad request\r\n .expect( function ( res ) {\r\n var created = res.body;\r\n\r\n created.should.be.ok;\r\n created.length.should.equal( 3 ); // should still have 3 objects in the response\r\n\r\n // The first one succeeded\r\n created[ 0 ].message.should.be.equal( 'job created' );\r\n created[ 0 ].id.should.be.a.Number;\r\n Object.keys( created[ 0 ] ).should.have.lengthOf( 2 );\r\n\r\n // The second one failed\r\n created[ 1 ].error.should.equal( 'Must provide job type' );\r\n Object.keys( created[ 1 ] ).should.have.lengthOf( 1 );\r\n\r\n // The third one succeeded\r\n created[ 2 ].message.should.be.equal( 'job created' );\r\n created[ 2 ].id.should.be.a.Number;\r\n Object.keys( created[ 2 ] ).should.have.lengthOf( 2 );\r\n } )\r\n .end( done );\r\n } );\r\n } );\r\n} );\r\n" }, { "path": "test/mocha.opts", "content": "--compilers coffee:coffee-script\n--require should\n--reporter spec\n--ui bdd\n--timeout 10000" }, { "path": "test/prefix.coffee", "content": "kue = require '../'\n\ndescribe 'Kue - Prefix', ->\n\n makeJobs = (queueName) ->\n opts =\n prefix: queueName\n promotion:\n interval: 10\n jobs = kue.createQueue opts\n return jobs\n\n stopJobs = (jobs, callback) ->\n jobs.shutdown callback\n\n # expected redis activity\n #\n # 1397744169.196792 \"subscribe\" \"q:events\"\n # 1397744169.196852 \"unsubscribe\"\n it 'should use prefix q by default', (done) ->\n jobs = kue.createQueue()\n jobs.client.prefix.should.equal 'q'\n stopJobs jobs, done\n\n # expected redis activity\n #\n # 1397744498.330456 \"subscribe\" \"testPrefix1:events\"\n # 1397744498.330638 \"unsubscribe\"\n # 1397744498.330907 \"subscribe\" \"testPrefix2:events\"\n # 1397744498.331148 \"unsubscribe\"\n it 'should accept and store prefix', (done) ->\n\n jobs = makeJobs('testPrefix1')\n\n jobs.client.prefix.should.equal 'testPrefix1'\n\n stopJobs jobs, (err) ->\n jobs2 = makeJobs('testPrefix2')\n jobs2.client.prefix.should.equal 'testPrefix2'\n stopJobs jobs2, done\n\n it 'should process and complete a job using a prefix', (testDone) ->\n\n jobs = makeJobs('simplePrefixTest')\n\n job = jobs.create('simplePrefixJob')\n job.on 'complete', () ->\n stopJobs jobs, testDone\n job.save()\n jobs.process 'simplePrefixJob', (job, done) ->\n done()\n\n # expected redis activity\n #\n # 1397744498.333423 \"subscribe\" \"jobCompleteTest:events\"\n # 1397744498.334002 \"info\"\n # 1397744498.334358 \"zcard\" \"jobCompleteTest:jobs:inactive\"\n # 1397744498.335262 \"info\"\n # 1397744498.335578 \"incr\" \"jobCompleteTest:ids\"\n # etc...\n it 'store queued jobs in different prefixes', (testDone) ->\n jobs = makeJobs('jobCompleteTest')\n\n jobs.inactiveCount (err, count) ->\n prevCount = count\n\n jobs.create( 'fakeJob', {} ).save()\n f = ->\n jobs.inactiveCount (err, count) ->\n count.should.equal prevCount + 1\n stopJobs jobs, testDone\n setTimeout f, 10\n\n it 'should not pick up an inactive job from another prefix', (testDone) ->\n jobs = makeJobs('inactiveJobs')\n # create a job but do not process\n job = jobs.create('inactiveJob', {} ).save (err) ->\n # stop the 'inactiveJobs' prefix\n stopJobs jobs, (err) ->\n jobs = makeJobs('inactiveJobs2')\n\n # verify count of inactive jobs is 0 for this prefix\n jobs.inactiveCount (err, count) ->\n count.should.equal 0\n\n stopJobs jobs, testDone\n\n\n it 'should properly switch back to default queue', (testDone) ->\n jobs = makeJobs('notDefault')\n stopJobs jobs, (err) ->\n jobs = kue.createQueue()\n\n job = jobs.create('defaultPrefixJob')\n job.on 'complete', () ->\n stopJobs jobs, testDone\n job.save()\n\n jobs.process 'defaultPrefixJob', (job, done) ->\n done()\n\n" }, { "path": "test/shutdown.coffee", "content": "should = require 'should'\n\nkue = require '../'\n\n\ndescribe 'Kue', ->\n\n before (done) ->\n jobs = kue.createQueue()\n jobs.client.flushdb done\n\n after (done) ->\n jobs = kue.createQueue()\n jobs.client.flushdb done\n\n describe 'Shutdown', ->\n it 'should return singleton from createQueue', (done) ->\n jobs = kue.createQueue()\n jobsToo = kue.createQueue()\n jobs.should.equal jobsToo\n jobs.shutdown done\n\n\n\n it 'should destroy singleton on shutdown', (done) ->\n jobs = kue.createQueue()\n jobs.shutdown (err) ->\n # test that new jobs object is a different reference\n newJobs = kue.createQueue()\n newJobs.should.not.equal jobs\n newJobs.shutdown done\n\n\n\n it 'should clear properties on shutdown', (done) ->\n jobs = kue.createQueue({promotion:{interval:200}})\n jobs.shutdown (err) ->\n should(jobs.workers).be.empty\n should(jobs.client).be.empty\n should(jobs.promoter).be.empty\n done()\n\n\n\n it 'should be able to pause/resume the worker', (done) ->\n jobs = kue.createQueue()\n job_data =\n title: 'resumable jobs'\n to: 'tj@learnboost.com'\n total_jobs = 3\n for i in [0...total_jobs]\n jobs.create('resumable-jobs', job_data).save()\n\n jobs.process 'resumable-jobs', 1, (job, ctx, job_done) ->\n job_done()\n if( !--total_jobs )\n jobs.shutdown 1000, done\n else\n ctx.pause()\n setTimeout ctx.resume, 100\n\n\n\n it 'should not clear properties on single type shutdown', (testDone) ->\n jobs = kue.createQueue()\n fn = (err) ->\n jobs.client.should.not.be.empty\n jobs.shutdown 10, testDone\n\n jobs.shutdown 10, 'fooJob', fn\n\n\n\n it 'should shutdown one worker type on single type shutdown', (testDone) ->\n jobs = kue.createQueue()\n # set up two worker types\n jobs.process 'runningTask', (job, done) ->\n done()\n jobs.workers.should.have.length 1\n jobs.process 'shutdownTask', (job, done) ->\n done()\n jobs.workers.should.have.length 2\n fn = (err) ->\n # verify shutdownTask is not running but runningTask is\n for worker in jobs.workers\n switch worker.type\n when 'shutdownTask'\n worker.should.have.property 'running', false\n when 'runningTask'\n worker.should.have.property 'running', true\n\n # kue should still be running\n jobs.promoter.should.not.be.empty\n jobs.client.should.not.be.empty\n\n jobs.shutdown 10, testDone\n jobs.shutdown 10, 'shutdownTask', fn\n\n\n it 'should fail active job when shutdown timer expires', (testDone) ->\n jobs = kue.createQueue()\n jobId = null\n jobs.process 'long-task', (job, done) ->\n jobId = job.id\n fn = ->\n done()\n setTimeout fn, 10000\n\n jobs.create('long-task', {}).save()\n # need to make sure long-task has had enough time to get into active state\n waitForJobToRun = ->\n fn = (err) ->\n kue.Job.get jobId, (err, job) ->\n job.should.have.property '_state', \"failed\"\n job.should.have.property '_error', \"Shutdown\"\n testDone()\n\n # shutdown timer is shorter than job length\n jobs.shutdown 10, fn\n\n setTimeout waitForJobToRun, 50\n\n\n\n it 'should not call graceful shutdown twice on subsequent calls', (testDone) ->\n jobs = kue.createQueue()\n jobs.process 'test-subsequent-shutdowns', (job, done) ->\n done()\n setTimeout ()->\n jobs.shutdown 100, (err)->\n should.not.exist(err)\n , 50\n\n setTimeout ()->\n jobs.shutdown 100, (err)->\n should.exist err, 'expected `err` to exist'\n err.should.be.an.instanceOf(Error)\n .with.property('message', 'Shutdown already in progress')\n testDone()\n , 60\n\n jobs.create('test-subsequent-shutdowns', {}).save()\n\n\n\n it 'should fail active re-attemptable job when shutdown timer expires', (testDone) ->\n jobs = kue.createQueue()\n jobId = null\n jobs.process 'shutdown-reattemptable-jobs', (job, done) ->\n jobId = job.id\n setTimeout done, 500\n\n jobs.create('shutdown-reattemptable-jobs', { title: 'shutdown-reattemptable-jobs' }).attempts(2).save()\n\n # need to make sure long-task has had enough time to get into active state\n waitForJobToRun = ->\n fn = (err) ->\n kue.Job.get jobId, (err, job) ->\n job.should.have.property '_state', \"inactive\"\n job.should.have.property '_attempts', \"1\"\n job.should.have.property '_error', \"Shutdown\"\n testDone()\n\n # shutdown timer is shorter than job length\n jobs.shutdown 100, fn\n\n setTimeout waitForJobToRun, 50\n" }, { "path": "test/tdd/kue.spec.js", "content": "var sinon = require('sinon');\nvar kue = require('../../lib/kue');\nvar redis = require('../../lib/redis');\nvar events = require('../../lib/queue/events');\nvar Job = require('../../lib/queue/job');\nvar Worker = require('../../lib/queue/worker');\nvar _ = require('lodash');\nvar EventEmitter = require('events').EventEmitter;\nvar redisClient = {};\n\ndescribe('Kue', function () {\n\n beforeEach(function(){\n sinon.stub(events, 'subscribe');\n sinon.stub(redis, 'configureFactory', function () {\n redis.createClient = sinon.stub();\n });\n });\n\n afterEach(function(){\n events.subscribe.restore();\n redis.configureFactory.restore();\n });\n\n describe('Function: createQueue', function () {\n\n it('should subscribe to queue events', function () {\n var queue = kue.createQueue();\n events.subscribe.called.should.be.true;\n });\n\n it('should set the correct default values', function () {\n var queue = kue.createQueue();\n queue.name.should.equal('kue');\n queue.id.should.equal([ 'kue', require(\"os\").hostname(), process.pid ].join(':'));\n (queue.promoter === null).should.be.true;\n queue.workers.should.eql(kue.workers);\n queue.shuttingDown.should.be.false;\n });\n\n it('should allow a custom name option', function () {\n it('should set the correct default values', function () {\n var queue = kue.createQueue({\n name: 'name'\n });\n queue.name.should.equal('name');\n });\n });\n });\n\n describe('Function: create', function() {\n var queue;\n beforeEach(function(){\n queue = kue.createQueue();\n });\n\n it('should return a new Job instance', function () {\n var data = {\n key: 'value'\n };\n var job = queue.create('type', data);\n\n job.type.should.equal('type');\n job.data.should.eql(data);\n });\n });\n\n describe('Function: on', function() {\n var queue, noop;\n beforeEach(function(){\n queue = kue.createQueue();\n events.subscribe.reset();\n noop = function () {};\n });\n\n it('should subscribe to events when subscribing to the job event', function () {\n queue.on('job', noop);\n events.subscribe.called.should.be.true;\n });\n\n it('should proxy the event listener', function (done) {\n queue.on('event', function (data) {\n data.should.equal('data');\n done();\n });\n queue.emit('event', 'data');\n });\n });\n\n describe('Function: setupTimers', function() {\n var queue;\n beforeEach(function(){\n queue = kue.createQueue();\n sinon.stub(queue, 'checkJobPromotion');\n sinon.stub(queue, 'checkActiveJobTtl');\n });\n\n afterEach(function(){\n queue.checkJobPromotion.restore();\n queue.checkActiveJobTtl.restore();\n });\n\n it('should setup a warlock client if it is not setup yet', function () {\n queue.warlock = undefined;\n queue.setupTimers();\n queue.warlock.should.exist;\n });\n\n it('should call checkJobPromotion', function () {\n queue.setupTimers();\n queue.checkJobPromotion.called.should.be.true;\n });\n\n it('should call checkActiveJobTtl', function () {\n queue.setupTimers();\n queue.checkActiveJobTtl.called.should.be.true;\n });\n });\n\n describe('Function: checkJobPromotion', function() {\n var queue, unlock, clock, timeout, client, ids, job;\n\n beforeEach(function(){\n unlock = sinon.spy();\n timeout = 1000;\n ids = [1, 2, 3];\n client = {\n zrangebyscore: sinon.stub().callsArgWith(6, null, ids),\n getKey: sinon.stub().returnsArg(0),\n stripFIFO: sinon.stub().returnsArg(0)\n };\n job = {\n inactive: sinon.stub().callsArg(0)\n };\n\n queue = kue.createQueue();\n queue.client = client;\n\n sinon.stub(Job, 'get').callsArgWith(1, null, job);\n sinon.stub(queue.warlock, 'lock').callsArgWith(2, null, unlock);\n sinon.stub(events, 'emit');\n clock = sinon.useFakeTimers();\n });\n\n afterEach(function(){\n Job.get.restore();\n queue.warlock.lock.restore();\n events.emit.restore();\n clock.restore();\n });\n\n it('should set the promotion lock', function () {\n queue.checkJobPromotion();\n clock.tick(timeout);\n queue.warlock.lock.calledWith('promotion', 2000).should.be.true;\n });\n\n it('should allow an override for the lockTtl', function () {\n queue.checkJobPromotion({ lockTtl: 5000 });\n clock.tick(timeout);\n queue.warlock.lock.calledWith('promotion', 5000).should.be.true;\n });\n\n it('should load all delayed jobs that should be run job', function () {\n queue.checkJobPromotion();\n clock.tick(timeout);\n client.zrangebyscore.calledWith(client.getKey('jobs:delayed'), 0, sinon.match.any, \"LIMIT\", 0, 1000).should.be.true;\n });\n\n it('should get each job', function () {\n queue.checkJobPromotion();\n clock.tick(timeout);\n Job.get.callCount.should.equal(3);\n Job.get.calledWith(ids[0]).should.be.true;\n Job.get.calledWith(ids[1]).should.be.true;\n Job.get.calledWith(ids[2]).should.be.true;\n });\n\n it('should emit promotion for each job', function () {\n queue.checkJobPromotion();\n clock.tick(timeout);\n events.emit.callCount.should.equal(3);\n events.emit.calledWith(ids[0], 'promotion').should.be.true;\n events.emit.calledWith(ids[1], 'promotion').should.be.true;\n events.emit.calledWith(ids[2], 'promotion').should.be.true;\n });\n\n it('should set each job to inactive', function () {\n queue.checkJobPromotion();\n clock.tick(timeout);\n job.inactive.callCount.should.equal(3);\n });\n\n it('should unlock promotion', function () {\n queue.checkJobPromotion();\n clock.tick(timeout);\n unlock.calledOnce.should.be.true;\n });\n\n });\n\n describe('Function: checkActiveJobTtl', function() {\n var queue, unlock, clock, timeout, client, ids, job;\n\n beforeEach(function(){\n unlock = sinon.spy();\n timeout = 1000;\n ids = [1, 2, 3];\n client = {\n zrangebyscore: sinon.stub().callsArgWith(6, null, ids),\n getKey: sinon.stub().returnsArg(0),\n stripFIFO: sinon.stub().returnsArg(0)\n };\n job = {\n failedAttempt: sinon.stub().callsArg(1)\n };\n\n queue = kue.createQueue();\n queue.client = client;\n\n sinon.spy(queue, 'removeAllListeners');\n sinon.stub(Job, 'get').callsArgWith(1, null, job);\n sinon.stub(queue.warlock, 'lock').callsArgWith(2, null, unlock);\n sinon.stub(events, 'emit');\n clock = sinon.useFakeTimers();\n });\n\n afterEach(function(){\n queue.removeAllListeners.restore();\n Job.get.restore();\n queue.warlock.lock.restore();\n events.emit.restore();\n clock.restore();\n });\n\n it('should set the activeJobsTTL lock', function () {\n queue.checkActiveJobTtl();\n clock.tick(timeout);\n queue.warlock.lock.calledWith('activeJobsTTL').should.be.true;\n });\n\n it('should load all expired jobs', function () {\n queue.checkActiveJobTtl();\n clock.tick(timeout);\n client.zrangebyscore.calledWith(client.getKey('jobs:active'), 100000, sinon.match.any, \"LIMIT\", 0, 1000).should.be.true;\n });\n\n it('should emit ttl exceeded for each job', function () {\n queue.checkActiveJobTtl();\n clock.tick(timeout);\n events.emit.callCount.should.equal(3);\n events.emit.calledWith(ids[0], 'ttl exceeded');\n events.emit.calledWith(ids[1], 'ttl exceeded');\n events.emit.calledWith(ids[2], 'ttl exceeded');\n });\n\n it('should unlock after all the job ttl exceeded acks have been received', function () {\n queue.checkActiveJobTtl('job ttl exceeded ack');\n queue.checkActiveJobTtl();\n clock.tick(timeout);\n _.each(ids, function (id) {\n // calling queue.emit since queue.on does special logic for events that start with \"job\"\n queue.emit('job ttl exceeded ack', id);\n });\n unlock.calledOnce.should.be.true;\n queue.removeAllListeners.calledWith('job ttl exceeded ack').should.be.true;\n });\n\n it('should call job.failedAttempt for each job that did not receive the ack event', function () {\n queue.removeAllListeners('job ttl exceeded ack');\n queue.checkActiveJobTtl('job ttl exceeded ack');\n clock.tick(timeout);\n var id = ids.splice(0, 1)[0];\n _.each(ids, function (id) {\n // calling queue.emit since queue.on does special logic for events that start with \"job\"\n queue.emit('job ttl exceeded ack', id);\n });\n clock.tick(timeout);\n Job.get.calledWith(id).should.be.true;\n job.failedAttempt.calledOnce.should.be.true;\n job.failedAttempt.calledWith({\n error: true,\n message: 'TTL exceeded'\n }).should.be.true;\n });\n });\n\n describe('Function: watchStuckJobs', function() {\n var queue, clock, client, sha;\n\n beforeEach(function(){\n sha = 'sha';\n client = {\n script: sinon.stub().callsArgWith(2, null, sha),\n evalsha: sinon.stub().callsArg(2)\n };\n\n queue = kue.createQueue();\n queue.client = client;\n\n clock = sinon.useFakeTimers();\n });\n\n afterEach(function(){\n clock.restore();\n });\n\n it('should load the script', function () {\n queue.watchStuckJobs();\n client.script.calledWith('LOAD').should.be.true;\n });\n\n it('should run the script on an interval', function () {\n queue.watchStuckJobs();\n clock.tick(1000);\n client.evalsha.calledWith(sha, 0).should.be.true;\n client.evalsha.callCount.should.equal(1);\n clock.tick(1000);\n client.evalsha.callCount.should.equal(2);\n });\n\n });\n\n describe('Function: setting', function() {\n var queue, client;\n\n beforeEach(function(){\n client = {\n getKey: sinon.stub().returnsArg(0),\n hget: sinon.stub().callsArg(2)\n };\n\n queue = kue.createQueue();\n queue.client = client;\n });\n\n it('should get the requested setting', function (done) {\n queue.setting('name', function () {\n client.hget.calledWith(client.getKey('settings'), 'name').should.be.true;\n done();\n });\n });\n\n });\n\n describe('Function: process', function() {\n var queue, client, worker;\n\n beforeEach(function(){\n client = {\n getKey: sinon.stub().returnsArg(0),\n incrby: sinon.stub()\n };\n worker = new EventEmitter();\n queue = kue.createQueue();\n queue.workers = [];\n queue.client = client;\n\n sinon.stub(queue, 'setupTimers');\n sinon.stub(Worker.prototype, 'start').returns(worker);\n });\n\n afterEach(function(){\n queue.setupTimers.restore();\n Worker.prototype.start.restore();\n });\n\n it('should use 1 as the default number of workers', function () {\n queue.process('type', sinon.stub());\n Worker.prototype.start.callCount.should.equal(1);\n });\n\n it('should accept a number for the number of workers', function () {\n queue.process('type', 3, sinon.stub());\n Worker.prototype.start.callCount.should.equal(3);\n });\n\n it('should add each worker to the queue.workers array', function () {\n queue.process('type', 3, sinon.stub());\n queue.workers.length.should.equal(3);\n });\n\n it('should setup each worker to respond to error events', function () {\n sinon.stub(queue, 'emit');\n queue.process('type', 3, sinon.stub());\n worker.emit('error');\n queue.emit.callCount.should.equal(3);\n queue.emit.restore();\n });\n\n it('should setup each worker to respond to job complete events', function () {\n var job = {\n duration: 100\n };\n queue.process('type', 3, sinon.stub());\n worker.emit('job complete', job);\n client.incrby.calledWith(client.getKey('stats:work-time'), job.duration).should.be.true;\n });\n\n it('should setup timers', function () {\n queue.process('type', 3, sinon.stub());\n queue.setupTimers.called.should.be.true;\n });\n\n });\n\n describe('Function: shutdown', function() {\n var queue, client, worker, lockClient;\n\n beforeEach(function(){\n client = {\n quit: sinon.stub()\n };\n lockClient = {\n quit: sinon.stub()\n };\n worker = {\n shutdown: sinon.stub().callsArg(1)\n };\n queue = kue.createQueue();\n queue.shuttingDown = false;\n queue.workers = [worker, worker, worker];\n queue.client = client;\n queue.lockClient = lockClient;\n\n sinon.stub(events, 'unsubscribe');\n sinon.stub(redis, 'reset');\n });\n\n afterEach(function(){\n events.unsubscribe.restore();\n redis.reset.restore();\n });\n\n it('should return an error if it is already shutting down', function (done) {\n queue.shuttingDown = true;\n queue.shutdown(function(err){\n err.should.exist;\n done();\n });\n });\n\n it('should shutdown each worker', function (done) {\n queue.shutdown(function () {\n worker.shutdown.callCount.should.equal(3);\n done();\n });\n });\n\n it('should clean things up', function (done) {\n queue.shutdown(function () {\n queue.workers.length.should.equal(0);\n events.unsubscribe.called.should.be.true;\n redis.reset.called.should.be.true;\n client.quit.called.should.be.true;\n (queue.client == null).should.be.true;\n lockClient.quit.called.should.be.true;\n (queue.lockClient == null).should.be.true;\n done();\n });\n });\n\n });\n\n describe('Function: types', function() {\n var queue, client, types;\n\n beforeEach(function(){\n types = ['type1', 'type2'];\n client = {\n getKey: sinon.stub().returnsArg(0),\n smembers: sinon.stub().callsArgWith(1, null, types)\n };\n queue = kue.createQueue();\n queue.client = client;\n });\n\n it('should get the jobs types', function (done) {\n queue.types(function(err, tps){\n tps.should.eql(types);\n done();\n });\n });\n });\n\n describe('Function: state', function() {\n var queue, client, jobIds, state;\n\n beforeEach(function(){\n jobIds = [1, 2];\n state = 'state';\n client = {\n getKey: sinon.stub().returnsArg(0),\n stripFIFO: sinon.stub().returnsArg(0),\n zrange: sinon.stub().callsArgWith(3, null, jobIds)\n };\n queue = kue.createQueue();\n queue.client = client;\n });\n\n it('should get all job ids for the given state', function (done) {\n queue.state(state, function (err, ids) {\n ids.should.eql(jobIds);\n done();\n });\n });\n\n });\n\n describe('Function: workTime', function() {\n var queue, client, n;\n\n beforeEach(function(){\n n = 20;\n client = {\n getKey: sinon.stub().returnsArg(0),\n get: sinon.stub().callsArgWith(1, null, n)\n };\n queue = kue.createQueue();\n queue.client = client;\n });\n\n it('should load the worktime', function (done) {\n queue.workTime(function (err, time) {\n time.should.equal(n);\n done();\n });\n });\n\n });\n\n describe('Function: cardByType', function() {\n var queue, client, type, state, total;\n\n beforeEach(function(){\n type = 'type';\n state = 'state';\n total = 20;\n client = {\n getKey: sinon.stub().returnsArg(0),\n zcard: sinon.stub().callsArgWith(1, null, total)\n };\n queue = kue.createQueue();\n queue.client = client;\n });\n\n it('should return the total number of jobs for a given type and state', function (done) {\n queue.cardByType(type, state, function (err, card) {\n card.should.equal(total);\n done();\n });\n });\n });\n\n describe('function: card', function() {\n var queue, client, state, total;\n\n beforeEach(function(){\n state = 'state';\n total = 20;\n client = {\n getKey: sinon.stub().returnsArg(0),\n zcard: sinon.stub().callsArgWith(1, null, total)\n };\n queue = kue.createQueue();\n queue.client = client;\n });\n\n it('should return the total number of jobs for a given state', function (done) {\n queue.card(state, function (err, card) {\n card.should.equal(total);\n done();\n });\n });\n });\n\n describe('Function: complete', function() {\n var queue;\n\n beforeEach(function(){\n queue = kue.createQueue();\n sinon.stub(queue, 'state').callsArg(1);\n });\n\n afterEach(function(){\n queue.state.restore();\n });\n\n it('should get the completed jobs', function (done) {\n queue.complete(function () {\n queue.state.calledWith('complete').should.be.true;\n done();\n });\n });\n });\n\n describe('Function: failed', function() {\n var queue;\n\n beforeEach(function(){\n queue = kue.createQueue();\n sinon.stub(queue, 'state').callsArg(1);\n });\n\n afterEach(function(){\n queue.state.restore();\n });\n\n it('should get the completed jobs', function (done) {\n queue.failed(function () {\n queue.state.calledWith('failed').should.be.true;\n done();\n });\n });\n });\n\n describe('Function: inactive', function() {\n var queue;\n\n beforeEach(function(){\n queue = kue.createQueue();\n sinon.stub(queue, 'state').callsArg(1);\n });\n\n afterEach(function(){\n queue.state.restore();\n });\n\n it('should get the completed jobs', function (done) {\n queue.inactive(function () {\n queue.state.calledWith('inactive').should.be.true;\n done();\n });\n });\n });\n\n describe('Function: active', function() {\n var queue;\n\n beforeEach(function(){\n queue = kue.createQueue();\n sinon.stub(queue, 'state').callsArg(1);\n });\n\n afterEach(function(){\n queue.state.restore();\n });\n\n it('should get the completed jobs', function (done) {\n queue.active(function () {\n queue.state.calledWith('active').should.be.true;\n done();\n });\n });\n });\n\n describe('Function: delayed', function() {\n var queue;\n\n beforeEach(function(){\n queue = kue.createQueue();\n sinon.stub(queue, 'state').callsArg(1);\n });\n\n afterEach(function(){\n queue.state.restore();\n });\n\n it('should get the completed jobs', function (done) {\n queue.delayed(function () {\n queue.state.calledWith('delayed').should.be.true;\n done();\n });\n });\n });\n\n describe('Function: completeCount', function() {\n var queue;\n\n beforeEach(function(){\n queue = kue.createQueue();\n sinon.stub(queue, 'card').callsArg(1);\n sinon.stub(queue, 'cardByType').callsArg(2);\n });\n\n afterEach(function(){\n queue.card.restore();\n queue.cardByType.restore();\n });\n\n it('should get all completed jobs', function (done) {\n queue.completeCount(function () {\n queue.card.calledWith('complete').should.be.true;\n done();\n });\n });\n\n it('should get all completed jobs of a certain type', function (done) {\n queue.completeCount('type', function () {\n queue.cardByType.calledWith('type', 'complete').should.be.true;\n done();\n });\n });\n });\n\n describe('Function: failedCount', function() {\n var queue;\n\n beforeEach(function(){\n queue = kue.createQueue();\n sinon.stub(queue, 'card').callsArg(1);\n sinon.stub(queue, 'cardByType').callsArg(2);\n });\n\n afterEach(function(){\n queue.card.restore();\n queue.cardByType.restore();\n });\n\n it('should get all completed jobs', function (done) {\n queue.failedCount(function () {\n queue.card.calledWith('failed').should.be.true;\n done();\n });\n });\n\n it('should get all completed jobs of a certain type', function (done) {\n queue.failedCount('type', function () {\n queue.cardByType.calledWith('type', 'failed').should.be.true;\n done();\n });\n });\n });\n\n describe('Function: inactiveCount', function() {\n var queue;\n\n beforeEach(function(){\n queue = kue.createQueue();\n sinon.stub(queue, 'card').callsArg(1);\n sinon.stub(queue, 'cardByType').callsArg(2);\n });\n\n afterEach(function(){\n queue.card.restore();\n queue.cardByType.restore();\n });\n\n it('should get all completed jobs', function (done) {\n queue.inactiveCount(function () {\n queue.card.calledWith('inactive').should.be.true;\n done();\n });\n });\n\n it('should get all completed jobs of a certain type', function (done) {\n queue.inactiveCount('type', function () {\n queue.cardByType.calledWith('type', 'inactive').should.be.true;\n done();\n });\n });\n });\n\n describe('Function: activeCount', function() {\n var queue;\n\n beforeEach(function(){\n queue = kue.createQueue();\n sinon.stub(queue, 'card').callsArg(1);\n sinon.stub(queue, 'cardByType').callsArg(2);\n });\n\n afterEach(function(){\n queue.card.restore();\n queue.cardByType.restore();\n });\n\n it('should get all completed jobs', function (done) {\n queue.activeCount(function () {\n queue.card.calledWith('active').should.be.true;\n done();\n });\n });\n\n it('should get all completed jobs of a certain type', function (done) {\n queue.activeCount('type', function () {\n queue.cardByType.calledWith('type', 'active').should.be.true;\n done();\n });\n });\n });\n\n describe('Function: delayedCount', function() {\n var queue;\n\n beforeEach(function(){\n queue = kue.createQueue();\n sinon.stub(queue, 'card').callsArg(1);\n sinon.stub(queue, 'cardByType').callsArg(2);\n });\n\n afterEach(function(){\n queue.card.restore();\n queue.cardByType.restore();\n });\n\n it('should get all completed jobs', function (done) {\n queue.delayedCount(function () {\n queue.card.calledWith('delayed').should.be.true;\n done();\n });\n });\n\n it('should get all completed jobs of a certain type', function (done) {\n queue.delayedCount('type', function () {\n queue.cardByType.calledWith('type', 'delayed').should.be.true;\n done();\n });\n });\n });\n\n});" }, { "path": "test/tdd/redis.spec.js", "content": "var sinon = require('sinon');\nvar r = require('redis');\nvar redis = require('../../lib/redis');\n\ndescribe('redis', function() {\n\n describe('Function: configureFactory', function() {\n\n beforeEach(function(){\n sinon.stub(redis, 'reset');\n });\n\n afterEach(function(){\n redis.reset.restore();\n });\n\n it('should parse a url connection string', function () {\n var options = {\n redis: 'redis://:password@host:1234/db'\n };\n redis.configureFactory(options);\n options.redis.port.should.equal('1234');\n options.redis.host.should.equal('host');\n options.redis.db.should.equal('db');\n });\n\n it('should reset everything', function () {\n var options = {\n redis: 'redis://:password@host:1234/db'\n };\n redis.configureFactory(options);\n redis.reset.called.should.be.true;\n });\n\n it('should export the createClient function', function () {\n var options = {\n redis: 'redis://:password@host:1234/db'\n };\n redis.createClient = null;\n redis.configureFactory(options);\n (typeof redis.createClient == 'function').should.be.true;\n });\n\n });\n\n describe('Function: createClient', function() {\n var options;\n beforeEach(function(){\n options = {\n prefix: 'prefix',\n redis: 'redis://:password@host:1234/db'\n };\n redis.configureFactory(options);\n sinon.stub(redis, 'createClientFactory').returns({\n on: sinon.stub()\n });\n });\n\n afterEach(function(){\n redis.createClientFactory.restore();\n });\n\n it('should create a client object', function () {\n var client = redis.createClient();\n client.prefix.should.equal(options.prefix);\n ('function' === typeof client.getKey).should.be.true;\n ('function' === typeof client.createFIFO).should.be.true;\n ('function' === typeof client.stripFIFO).should.be.true;\n });\n\n describe('Function: client.getKey', function() {\n\n it('should return the key with the prefix', function () {\n var client = redis.createClient();\n var key = client.getKey('key');\n key.should.equal('prefix:key');\n });\n\n it('should return key with prefix and curly braces for ioredis cluster', function () {\n var client = redis.createClient();\n client.constructor = {\n name: 'Redis'\n };\n var key = client.getKey('key');\n key.should.equal('{prefix}:key');\n });\n\n });\n\n describe('Function: client.createFIFO', function() {\n\n it('should prefix with the length of the id', function () {\n var client = redis.createClient();\n var id = client.createFIFO('12345678910');\n id.should.equal('11|12345678910');\n });\n\n it('should pad with a zero for single digit length ids', function () {\n var client = redis.createClient();\n var id = client.createFIFO('123');\n id.should.equal('03|123');\n });\n\n });\n\n describe('Function: client.stripFIFO', function() {\n\n it('should strip the prefix on the id', function () {\n var client = redis.createClient();\n var id = client.stripFIFO( '03|123' );\n id.should.equal(123);\n });\n });\n\n });\n\n describe('Function: createClientFactory', function() {\n var options, client;\n beforeEach(function(){\n options = {\n prefix: 'prefix',\n redis: {\n port: 'port',\n host: 'host',\n db: 'db',\n options: {}\n }\n };\n client = {\n auth: sinon.stub(),\n select: sinon.stub()\n };\n sinon.stub(r, 'createClient').returns(client);\n });\n\n afterEach(function(){\n r.createClient.restore();\n });\n\n it('should create a client', function () {\n var c = redis.createClientFactory(options);\n r.createClient.called.should.be.true;\n r.createClient.calledWith(options.redis.port, options.redis.host, options.redis.options).should.be.true;\n });\n\n it('should authenticate if auth is present', function () {\n options.redis.auth = 'auth';\n var c = redis.createClientFactory(options);\n client.auth.calledWith(options.redis.auth).should.be.true;\n });\n\n it('should select the passed in db', function () {\n options.redis.db = 1;\n var c = redis.createClientFactory(options);\n client.select.calledWith(options.redis.db).should.be.true;\n });\n\n });\n\n describe('Function: client', function() {\n\n it('should return the existing client if there is one', function () {\n redis._client = 'client';\n (redis.client()).should.equal('client');\n });\n\n it('should create a client if one is not present', function () {\n redis._client = null;\n sinon.stub(redis, 'createClient');\n redis.client();\n redis.createClient.called.should.be.true;\n redis.createClient.restore();\n });\n\n });\n\n describe('Function: pubsubClient', function() {\n\n it('should return the existing client if there is one', function () {\n redis._pubsub = 'pubsubClient';\n (redis.pubsubClient()).should.equal('pubsubClient');\n });\n\n it('should create a pubsubClient if one is not present', function () {\n redis._pubsub = null;\n sinon.stub(redis, 'createClient');\n redis.pubsubClient();\n redis.createClient.called.should.be.true;\n redis.createClient.restore();\n });\n\n });\n\n describe('Function: reset', function() {\n var client, pubsub;\n beforeEach(function(){\n client = {\n quit: sinon.stub()\n };\n pubsub = {\n quit: sinon.stub()\n };\n redis._client = client;\n redis._pubsub = pubsub;\n });\n\n it('should quit and remove the client', function () {\n redis.reset();\n (redis._client == null).should.be.true;\n client.quit.called.should.be.true;\n });\n\n it('should quick and remove the pubsub client', function () {\n redis.reset();\n (redis._pubsub == null).should.be.true;\n pubsub.quit.called.should.be.true;\n });\n\n });\n\n});" }, { "path": "test/test.coffee", "content": "_ = require 'lodash'\nasync = require 'async'\nshould = require 'should'\nkue = require '../'\nutil = require 'util'\n\ndescribe 'Kue Tests', ->\n\n jobs = null\n Job = null\n\n beforeEach ->\n jobs = kue.createQueue({promotion:{interval:50}})\n Job = kue.Job\n\n afterEach (done) ->\n jobs.shutdown 50, done\n\n # before (done) ->\n # jobs = kue.createQueue({promotion:{interval:100}})\n # jobs.client.flushdb done\n\n # after (done) ->\n # jobs = kue.createQueue({promotion:{interval:100}})\n # jobs.client.flushdb done\n\n\n describe 'Job Producer', ->\n it 'should save jobs having a new id', (done) ->\n job_data =\n title: 'Test Email Job'\n to: 'tj@learnboost.com'\n job = jobs.create('email-to-be-saved', job_data)\n jobs.process('email-to-be-saved', _.noop)\n job.save (err) ->\n job.id.should.be.an.instanceOf(Number)\n done err\n\n\n it 'should set worker id on job hash', (done) ->\n job_data =\n title: 'Test workerId Job'\n to: 'tj@learnboost.com'\n job = jobs.create('worker-id-test', job_data)\n jobs.process 'worker-id-test', (job, jdone)->\n jdone()\n Job.get job.id, (err, j) ->\n j.toJSON().workerId.should.be.not.null;\n done()\n job.save()\n\n\n it 'should receive job complete event', (done) ->\n jobs.process 'email-to-be-completed', (job, done)->\n done()\n job_data =\n title: 'Test Email Job'\n to: 'tj@learnboost.com'\n jobs.create('email-to-be-completed', job_data)\n .on 'complete', ->\n done()\n .save()\n\n\n\n it 'should receive job result in complete event', (done) ->\n jobs.process 'email-with-results', (job, done)->\n done( null, {finalResult:123} )\n job_data =\n title: 'Test Email Job With Results'\n to: 'tj@learnboost.com'\n jobs.create('email-with-results', job_data)\n .on 'complete', (result)->\n result.finalResult.should.be.equal 123\n done()\n .save()\n\n\n\n it 'should receive job progress event', (done) ->\n jobs.process 'email-to-be-progressed', (job, done)->\n job.progress 1, 2\n done()\n job_data =\n title: 'Test Email Job'\n to: 'tj@learnboost.com'\n jobs.create('email-to-be-progressed', job_data)\n .on 'progress', (progress)->\n progress.should.be.equal 50\n done()\n .save()\n\n\n\n it 'should receive job progress event with extra data', (done) ->\n jobs.process 'email-to-be-progressed', (job, done)->\n job.progress 1, 2,\n notifyTime : \"2014-11-22\"\n done()\n job_data =\n title: 'Test Email Job'\n to: 'tj@learnboost.com'\n jobs.create('email-to-be-progressed', job_data)\n .on 'progress', (progress, extraData)->\n progress.should.be.equal 50\n extraData.notifyTime.should.be.equal \"2014-11-22\"\n done()\n .save()\n\n\n\n it 'should receive job failed attempt events', (done) ->\n total = 2\n errorMsg = 'myError'\n jobs.process 'email-to-be-failed', (job, jdone)->\n jdone errorMsg\n job_data =\n title: 'Test Email Job'\n to: 'tj@learnboost.com'\n jobs.create('email-to-be-failed', job_data).attempts(2)\n .on 'failed attempt', (errMsg,doneAttempts) ->\n errMsg.should.be.equal errorMsg\n doneAttempts.should.be.equal 1\n total--\n .on 'failed', (errMsg)->\n errMsg.should.be.equal errorMsg\n (--total).should.be.equal 0\n done()\n .save()\n\n\n\n it 'should receive queue level complete event', (done) ->\n jobs.process 'email-to-be-completed', (job, jdone)->\n jdone( null, { prop: 'val' } )\n jobs.on 'job complete', (id, result) ->\n id.should.be.equal testJob.id\n result.prop.should.be.equal 'val'\n done()\n job_data =\n title: 'Test Email Job'\n to: 'tj@learnboost.com'\n testJob = jobs.create('email-to-be-completed', job_data).save()\n\n\n\n it 'should receive queue level failed attempt events', (done) ->\n total = 2\n errorMsg = 'myError'\n jobs.process 'email-to-be-failed', (job, jdone)->\n jdone errorMsg\n job_data =\n title: 'Test Email Job'\n to: 'tj@learnboost.com'\n jobs.on 'job failed attempt', (id, errMsg, doneAttempts) ->\n id.should.be.equal newJob.id\n errMsg.should.be.equal errorMsg\n doneAttempts.should.be.equal 1\n total--\n .on 'job failed', (id, errMsg)->\n id.should.be.equal newJob.id\n errMsg.should.be.equal errorMsg\n (--total).should.be.equal 0\n done()\n newJob = jobs.create('email-to-be-failed', job_data).attempts(2).save()\n\n\n\n\n describe 'Job', ->\n it 'should be processed after delay', (done) ->\n now = Date.now()\n jobs.create( 'simple-delay-job', { title: 'simple delay job' } ).delay(300).save()\n jobs.process 'simple-delay-job', (job, jdone) ->\n processed = Date.now()\n (processed - now).should.be.approximately( 300, 100 )\n jdone()\n done()\n\n\n it 'should have promote_at timestamp', (done) ->\n now = Date.now()\n job = jobs.create( 'simple-delayed-job', { title: 'simple delay job' } ).delay(300).save()\n jobs.process 'simple-delayed-job', (job, jdone) ->\n job.promote_at.should.be.approximately(now + 300, 100)\n jdone()\n done()\n\n\n it 'should update promote_at after delay change', (done) ->\n now = Date.now()\n job = jobs.create( 'simple-delayed-job-1', { title: 'simple delay job' } ).delay(300).save()\n job.delay(100).save()\n jobs.process 'simple-delayed-job-1', (job, jdone) ->\n job.promote_at.should.be.approximately(now + 100, 100)\n jdone()\n done()\n\n\n\n it 'should update promote_at after failure with backoff', (done) ->\n now = Date.now()\n job = jobs.create( 'simple-delayed-job-2', { title: 'simple delay job' } ).delay(100).attempts(2).backoff({delay: 100, type: 'fixed'}).save()\n calls = 0\n jobs.process 'simple-delayed-job-2', (job, jdone) ->\n processed = Date.now()\n if calls == 1\n (processed - now).should.be.approximately(300, 100)\n jdone()\n done()\n else\n (processed - now).should.be.approximately(100, 100)\n jdone('error')\n\n calls++\n\n\n\n it 'should be processed at a future date', (done) ->\n now = Date.now()\n jobs.create( 'future-job', { title: 'future job' } ).delay(new Date(now + 200)).save()\n jobs.process 'future-job', (job, jdone) ->\n processed = Date.now()\n (processed - now).should.be.approximately( 200, 100 )\n jdone()\n done()\n\n\n\n it 'should receive promotion event', (done) ->\n job_data =\n title: 'Test Email Job'\n to: 'tj@learnboost.com'\n jobs.process('email-to-be-promoted', (job,done)-> )\n jobs.create('email-to-be-promoted', job_data).delay(200)\n .on 'promotion', ()->\n done()\n .save()\n\n\n\n it 'should be re tried after failed attempts', (done) ->\n [total, remaining] = [2,2]\n jobs.create( 'simple-multi-attempts-job', { title: 'simple-multi-attempts-job' } ).attempts(total).save()\n jobs.process 'simple-multi-attempts-job', (job, jdone) ->\n job.toJSON().attempts.remaining.should.be.equal remaining\n (job.toJSON().attempts.made + job.toJSON().attempts.remaining).should.be.equal total\n if( !--remaining )\n jdone()\n done()\n else\n jdone( new Error('reaattempt') )\n\n\n\n it 'should honor original delay at fixed backoff', (done) ->\n [total, remaining] = [2,2]\n start = Date.now()\n jobs.create( 'backoff-fixed-job', { title: 'backoff-fixed-job' } ).delay( 200 ).attempts(total).backoff( true ).save()\n jobs.process 'backoff-fixed-job', (job, jdone) ->\n if( !--remaining )\n now = Date.now()\n (now - start).should.be.approximately(400,120)\n jdone()\n done()\n else\n jdone( new Error('reaattempt') )\n\n\n\n it 'should honor original delay at exponential backoff', (done) ->\n [total, remaining] = [3,3]\n start = Date.now()\n jobs.create( 'backoff-exponential-job', { title: 'backoff-exponential-job' } )\n .delay( 50 ).attempts(total).backoff( {type:'exponential', delay: 100} ).save()\n jobs.process 'backoff-exponential-job', (job, jdone) ->\n job._backoff.type.should.be.equal \"exponential\"\n job._backoff.delay.should.be.equal 100\n now = Date.now()\n if( !--remaining )\n (now - start).should.be.approximately(350,100)\n jdone()\n done()\n else\n jdone( new Error('reaattempt') )\n\n it 'should honor max delay at exponential backoff', (done) ->\n [total, remaining] = [10,10]\n last = Date.now()\n jobs.create( 'backoff-exponential-job', { title: 'backoff-exponential-job' } )\n .attempts(total).backoff( {type:'exponential', delay: 50, maxDelay: 100} ).save()\n jobs.process 'backoff-exponential-job', (job, jdone) ->\n job._backoff.type.should.be.equal \"exponential\"\n job._backoff.delay.should.be.equal 50\n job._backoff.maxDelay.should.be.equal 100\n now = Date.now()\n (now - last).should.be.lessThan 120\n if( !--remaining )\n jdone()\n done()\n else\n last = now\n jdone( new Error('reaattempt') )\n\n\n it 'should honor users backoff function', (done) ->\n [total, remaining] = [2,2]\n start = Date.now()\n jobs.create( 'backoff-user-job', { title: 'backoff-user-job' } )\n .delay( 50 ).attempts(total).backoff( ( attempts, delay ) -> 250 ).save()\n jobs.process 'backoff-user-job', (job, jdone) ->\n now = Date.now()\n if( !--remaining )\n (now - start).should.be.approximately(350, 100)\n jdone()\n done()\n else\n jdone( new Error('reaattempt') )\n\n it 'should log with a sprintf-style string', (done) ->\n jobs.create( 'log-job', { title: 'simple job' } ).save()\n jobs.process 'log-job', (job, jdone) ->\n job.log('this is %s number %d','test',1)\n Job.log job.id, (err,logs) ->\n logs[0].should.be.equal('this is test number 1');\n done()\n jdone()\n\n\n\n it 'should log objects, errors, arrays, numbers, etc', (done) ->\n jobs.create( 'log-job', { title: 'simple job' } ).save()\n jobs.process 'log-job', (job, jdone) ->\n testErr = new Error('test error')# to compare the same stack\n job.log()\n job.log(undefined)\n job.log(null)\n job.log({test: 'some text'})\n job.log(testErr)\n job.log([1,2,3])\n job.log(123)\n job.log(1.23)\n job.log(0)\n job.log(NaN)\n job.log(true)\n job.log(false)\n\n Job.log job.id, (err,logs) ->\n logs[0].should.be.equal(util.format(undefined));\n logs[1].should.be.equal(util.format(undefined));\n logs[2].should.be.equal(util.format(null));\n logs[3].should.be.equal(util.format({ test: 'some text' }));\n logs[4].should.be.equal(util.format(testErr));\n logs[5].should.be.equal(util.format([ 1, 2, 3 ]));\n logs[6].should.be.equal(util.format(123));\n logs[7].should.be.equal(util.format(1.23));\n logs[8].should.be.equal(util.format(0));\n logs[9].should.be.equal(util.format(NaN));\n logs[10].should.be.equal(util.format(true));\n logs[11].should.be.equal(util.format(false));\n done()\n jdone()\n\n\n\n\n describe 'Kue Core', ->\n\n it 'should receive a \"job enqueue\" event', (done) ->\n jobs.on 'job enqueue', (id, type) ->\n if type == 'email-to-be-enqueued'\n id.should.be.equal job.id\n done()\n jobs.process 'email-to-be-enqueued', (job, jdone) -> jdone()\n job = jobs.create('email-to-be-enqueued').save()\n\n\n it 'should receive a \"job remove\" event', (done) ->\n jobs.on 'job remove', (id, type) ->\n if type == 'removable-job'\n id.should.be.equal job.id\n done()\n jobs.process 'removable-job', (job, jdone) -> jdone()\n job = jobs.create('removable-job').save().remove()\n\n\n it 'should fail a job with TTL is exceeded', (done) ->\n jobs.process('test-job-with-ttl', (job, jdone) ->\n # do nothing to sample a stuck worker\n )\n jobs.create('test-job-with-ttl', title: 'a ttl job').ttl(500)\n .on 'failed', (err) ->\n err.should.be.equal 'TTL exceeded'\n done()\n .save()\n\n\n describe 'Kue Job Concurrency', ->\n\n it 'should process 2 concurrent jobs at the same time', (done) ->\n now = Date.now()\n jobStartTimes = []\n jobs.process('test-job-parallel', 2, (job,jdone) ->\n jobStartTimes.push Date.now()\n if( jobStartTimes.length == 2 )\n (jobStartTimes[0] - now).should.be.approximately( 0, 100 )\n (jobStartTimes[1] - now).should.be.approximately( 0, 100 )\n done()\n setTimeout(jdone, 500)\n )\n jobs.create('test-job-parallel', title: 'concurrent job 1').save()\n jobs.create('test-job-parallel', title: 'concurrent job 2').save()\n\n it 'should process non concurrent jobs serially', (done) ->\n now = Date.now()\n jobStartTimes = []\n jobs.process('test-job-serial', 1, (job,jdone) ->\n jobStartTimes.push Date.now()\n if( jobStartTimes.length == 2 )\n (jobStartTimes[0] - now).should.be.approximately( 0, 100 )\n (jobStartTimes[1] - now).should.be.approximately( 500, 100 )\n done()\n setTimeout(jdone, 500)\n )\n jobs.create('test-job-serial', title: 'non concurrent job 1').save()\n jobs.create('test-job-serial', title: 'non concurrent job 2').save()\n\n it 'should process a new job after a previous one fails with TTL is exceeded', (done) ->\n failures = 0\n now = Date.now()\n jobStartTimes = []\n jobs.process('test-job-serial-failed', 1, (job,jdone) ->\n jobStartTimes.push Date.now()\n if( jobStartTimes.length == 2 )\n (jobStartTimes[0] - now).should.be.approximately( 0, 100 )\n (jobStartTimes[1] - now).should.be.approximately( 500, 100 )\n failures.should.be.equal 1\n done()\n # do not call jdone to simulate a stuck worker\n )\n jobs.create('test-job-serial-failed', title: 'a ttl job 1').ttl(500).on( 'failed', ()->\n ++failures\n ).save()\n jobs.create('test-job-serial-failed', title: 'a ttl job 2').ttl(500).on( 'failed', ()->\n ++failures\n ).save()\n\n it 'should not stuck in inactive mode if one of the workers failed because of ttl', (done) ->\n jobs.create('jobsA',\n title: 'titleA'\n metadata: {}).delay(1000).attempts(3).backoff(\n delay: 1 * 1000\n type: 'exponential').removeOnComplete(true).ttl(1 * 1000).save()\n jobs.create('jobsB',\n title: 'titleB'\n metadata: {}).delay(1500).attempts(3).backoff(\n delay: 1 * 1000\n type: 'exponential').removeOnComplete(true).ttl(1 * 1000).save()\n\n jobs.process 'jobsA', 1, (job, jdone) ->\n if job._attempts == '2'\n done()\n return\n jobs.process 'jobsB', 1, (job, jdone) ->\n done()\n return\n\n\n describe 'Kue Job Removal', ->\n\n beforeEach (done) ->\n jobs.process 'sample-job-to-be-cleaned', (job, jdone) -> jdone()\n async.each([1..10], (id, next) ->\n jobs.create( 'sample-job-to-be-cleaned', {id: id} ).save(next)\n , done)\n\n\n it 'should be able to remove completed jobs', (done) ->\n jobs.complete (err, ids) ->\n should.not.exist err\n async.each(ids, (id, next) ->\n Job.remove(id, next)\n , done)\n\n\n it 'should be able to remove failed jobs', (done) ->\n jobs.failed (err, ids) ->\n should.not.exist err\n async.each(ids, (id, next) ->\n Job.remove(id, next)\n , done)\n" }, { "path": "test/test.js", "content": "var kue = require( '../' );\n\ndescribe('CONNECTION', function(){\n\tvar jobs = null;\n\n\tafterEach( function ( done ) {\n\t\tjobs.shutdown( 50, function () {\n\t\t\tdone()\n\t\t} );\n\t} );\n\n it( 'should configure properly with string', function ( done ) {\n\t jobs = new kue( {\n\t\t redis: 'redis://localhost:6379/15?foo=bar'\n\t } );\n\n\t jobs.client.options.port.should.be.eql( 6379 );\n\t jobs.client.options.host.should.be.eql( 'localhost' );\n\t jobs.client.options.foo.should.be.eql( 'bar' );\n\n\t var jobData = {\n\t\t title: 'welcome email for tj',\n\t\t to: '\"TJ\" ',\n\t\t template: 'welcome-email'\n\t };\n\t jobs.create( 'email-should-be-processed-3', jobData ).priority( 'high' ).save();\n\t jobs.process( 'email-should-be-processed-3', function ( job, jdone ) {\n\t\t job.data.should.be.eql( jobData );\n\t\t job.log( '

    This is a formatted log

    ' );\n\t\t // Needs to be here to support the async client.select statement where the return happens sync but the call is async\n\t\t jobs.client.selected_db.should.be.eql(15);\n\t\t jdone();\n\t\t done();\n\t } );\n });\n\n\tit( 'should configure properly with dictionary', function ( done ) {\n\t\tjobs = new kue( {\n\t\t\tredis: {\n\t\t\t\thost: 'localhost',\n\t\t\t\tport: 6379,\n\t\t\t\tdb: 15,\n\t\t\t\toptions: {\n\t\t\t\t\tfoo: 'bar'\n\t\t\t\t}\n\t\t\t}\n\t\t} );\n\n\t\tjobs.client.options.port.should.be.eql( 6379 );\n\t\tjobs.client.options.host.should.be.eql( 'localhost' );\n\t\tjobs.client.options.foo.should.be.eql( 'bar' );\n\n\t\tvar jobData = {\n\t\t\ttitle: 'welcome email for tj',\n\t\t\tto: '\"TJ\" ',\n\t\t\ttemplate: 'welcome-email'\n\t\t};\n\t\tjobs.create( 'email-should-be-processed-4', jobData ).priority( 'high' ).save();\n\t\tjobs.process( 'email-should-be-processed-4', function ( job, jdone ) {\n\t\t\tjob.data.should.be.eql( jobData );\n\t\t\tjob.log( '

    This is a formatted log

    ' );\n\t\t\t// Needs to be here to support the async client.select statement where the return happens sync but the call is async\n\t\t\tjobs.client.selected_db.should.be.eql(15);\n\t\t\tjdone();\n\t\t\tdone();\n\t\t} );\n\t});\n\n\tit( 'should default to 0 db with string', function ( done ) {\n\t\tvar jobs = new kue( {\n\t\t\tredis: 'redis://localhost:6379/?foo=bar'\n\t\t} );\n\n\t\tjobs.client.options.port.should.be.eql( 6379 );\n\t\tjobs.client.options.host.should.be.eql( 'localhost' );\n\t\tjobs.client.options.foo.should.be.eql( 'bar' );\n\n\t\tvar jobData = {\n\t\t\ttitle: 'welcome email for tj',\n\t\t\tto: '\"TJ\" ',\n\t\t\ttemplate: 'welcome-email'\n\t\t};\n\t\tjobs.create( 'email-should-be-processed-5', jobData ).priority( 'high' ).save();\n\t\tjobs.process( 'email-should-be-processed-5', function ( job, jdone ) {\n\t\t\tjob.data.should.be.eql( jobData );\n\t\t\tjob.log( '

    This is a formatted log

    ' );\n\t\t\tjobs.client.selected_db.should.be.eql(0);\n\t\t\tjdone();\n\t\t\tdone();\n\t\t} );\n\n\t});\n\n\tit( 'should default to 0 db with string and no /', function ( done ) {\n\t\tvar jobs = new kue( {\n\t\t\tredis: 'redis://localhost:6379?foo=bar'\n\t\t} );\n\n\t\tjobs.client.options.port.should.be.eql( 6379 );\n\t\tjobs.client.options.host.should.be.eql( 'localhost' );\n\t\tjobs.client.options.foo.should.be.eql( 'bar' );\n\n\t\tvar jobData = {\n\t\t\ttitle: 'welcome email for tj',\n\t\t\tto: '\"TJ\" ',\n\t\t\ttemplate: 'welcome-email'\n\t\t};\n\t\tjobs.create( 'email-should-be-processed-6', jobData ).priority( 'high' ).save();\n\t\tjobs.process( 'email-should-be-processed-6', function ( job, jdone ) {\n\t\t\tjob.data.should.be.eql( jobData );\n\t\t\tjob.log( '

    This is a formatted log

    ' );\n\t\t\tjobs.client.selected_db.should.be.eql(0);\n\t\t\tjdone();\n\t\t\tdone();\n\t\t} );\n\n\t});\n\n\tit( 'should configure properly with dictionary', function ( done ) {\n\t\tjobs = new kue( {\n\t\t\tredis: {\n\t\t\t\thost: 'localhost',\n\t\t\t\tport: 6379,\n\t\t\t\toptions: {\n\t\t\t\t\tfoo: 'bar'\n\t\t\t\t}\n\t\t\t}\n\t\t} );\n\n\t\tjobs.client.options.port.should.be.eql( 6379 );\n\t\tjobs.client.options.host.should.be.eql( 'localhost' );\n\t\tjobs.client.options.foo.should.be.eql( 'bar' );\n\n\t\tvar jobData = {\n\t\t\ttitle: 'welcome email for tj',\n\t\t\tto: '\"TJ\" ',\n\t\t\ttemplate: 'welcome-email'\n\t\t};\n\t\tjobs.create( 'email-should-be-processed-7', jobData ).priority( 'high' ).save();\n\t\tjobs.process( 'email-should-be-processed-7', function ( job, jdone ) {\n\t\t\tjob.data.should.be.eql( jobData );\n\t\t\tjob.log( '

    This is a formatted log

    ' );\n\t\t\t// Needs to be here to support the async client.select statement where the return happens sync but the call is async\n\t\t\tjobs.client.selected_db.should.be.eql(0);\n\t\t\tjdone();\n\t\t\tdone();\n\t\t} );\n\t});\n});\n\ndescribe( 'JOBS', function () {\n\n var jobs = null;\n\n beforeEach( function ( done ) {\n jobs = kue.createQueue( { promotion: { interval: 100 } } );\n done();\n } );\n\n afterEach( function ( done ) {\n jobs.shutdown( 50, function () {\n done()\n } );\n } );\n\n it( 'should be processed', function ( done ) {\n var jobData = {\n title: 'welcome email for tj',\n to: '\"TJ\" ',\n template: 'welcome-email'\n };\n jobs.create( 'email-should-be-processed', jobData ).priority( 'high' ).save();\n jobs.process( 'email-should-be-processed', function ( job, jdone ) {\n job.data.should.be.eql( jobData );\n job.log( '

    This is a formatted log

    ' );\n jdone();\n done();\n } );\n } );\n\n it( 'should retry on failure if attempts is set', function ( testDone ) {\n var job = jobs.create( 'failure-attempts', {} );\n var failures = 0;\n job.attempts( 5 )\n .on( 'complete', function () {\n attempts.should.be.equal( 5 );\n failures.should.be.equal( 4 );\n testDone();\n } )\n .on( 'failed attempt', function ( attempt ) {\n failures++;\n } )\n .save();\n var attempts = 0;\n jobs.process( 'failure-attempts', function ( job, done ) {\n attempts++;\n if ( attempts == 5 )\n done();\n else\n done( new Error( \"error\" ) );\n } );\n } );\n\n it( 'should accept url strings for redis when making an new queue', function ( done ) {\n var jobs = new kue( {\n redis: 'redis://localhost:6379/?foo=bar'\n } );\n\n jobs.client.options.port.should.be.eql( 6379 );\n jobs.client.options.host.should.be.eql( 'localhost' );\n jobs.client.options.foo.should.be.eql( 'bar' );\n\n var jobData = {\n title: 'welcome email for tj',\n to: '\"TJ\" ',\n template: 'welcome-email'\n };\n jobs.create( 'email-should-be-processed-2', jobData ).priority( 'high' ).save();\n jobs.process( 'email-should-be-processed-2', function ( job, jdone ) {\n job.data.should.be.eql( jobData );\n job.log( '

    This is a formatted log

    ' );\n jdone();\n done();\n } );\n } );\n} );\n" }, { "path": "test/test_mode.js", "content": "var kue = require('../'),\n _ = require('lodash'),\n queue = kue.createQueue();\n\ndescribe('Test Mode', function() {\n context('when enabled', function() {\n before(function() {\n queue.testMode.enter();\n });\n\n afterEach(function() {\n queue.testMode.clear();\n });\n\n it('adds jobs to an array in memory', function() {\n queue.createJob('myJob', { foo: 'bar' }).save();\n\n var jobs = queue.testMode.jobs;\n expect(jobs.length).to.equal(1);\n\n var job = _.last(jobs);\n expect(job.type).to.equal('myJob');\n expect(job.data).to.eql({ foo: 'bar' });\n });\n\n it('adds jobs to an array in memory and processes them when processQueue is true', function(done) {\n queue.testMode.exit();\n queue.testMode.enter(true);\n\n queue.createJob('test-testMode-process', { foo: 'bar' }).save();\n\n var jobs = queue.testMode.jobs;\n expect(jobs.length).to.equal(1);\n\n var job = _.last(jobs);\n expect(job.type).to.equal('test-testMode-process');\n expect(job.data).to.eql({ foo: 'bar' });\n\n job.on('complete', function() {\n queue.testMode.exit();\n queue.testMode.enter();\n done();\n });\n\n queue.process('test-testMode-process', function(job, jdone) {\n job.data.should.be.eql({ foo: 'bar' });\n\n jdone();\n });\n });\n\n describe('#clear', function() {\n it('resets the list of jobs', function() {\n queue.createJob('myJob', { foo: 'bar' }).save();\n queue.testMode.clear();\n\n var jobs = queue.testMode.jobs;\n expect(jobs.length).to.equal(0);\n });\n });\n });\n\n context('when disabled', function() {\n before(function() {\n // Simulate entering and exiting test mode to ensure\n // state is restored correctly.\n queue.testMode.enter();\n queue.testMode.exit();\n });\n\n it('processes jobs regularly', function(done) {\n queue.createJob('myJob', { foo: 'bar' }).save();\n\n var jobs = queue.testMode.jobs;\n expect(jobs.length).to.equal(0);\n\n queue.process('myJob', function (job, jdone) {\n expect(job.data).to.eql({ foo: 'bar' });\n jdone();\n done();\n });\n });\n });\n});\n" } ]