[ { "path": ".editorconfig", "content": "; EditorConfig file: http://EditorConfig.org\n; Install the \"EditorConfig\" plugin into Sublime Text to use\n\nroot = true\n\n[*]\ncharset = utf-8\nend_of_line = lf\ninsert_final_newline = true\nindent_style = space\nindent_size = 2\ntrim_trailing_whitespace = true\n" }, { "path": ".eslintrc.yml", "content": "env:\n node: true\n\nparserOptions:\n ecmaVersion: 2018\n\nextends:\n - eslint:recommended\n - plugin:mocha/recommended\n - plugin:node/recommended\n\nplugins:\n - mocha\n - node\n\nrules:\n valid-jsdoc: 0\n func-style: 0\n no-use-before-define: 0\n camelcase: 1\n no-unused-vars: 2\n no-alert: 2\n no-console: [2, { allow: ['warn', 'error'] }]\n no-underscore-dangle: 0\n object-shorthand: 0\n\n strict: [2, 'global']\n no-var: 2\n prefer-arrow-callback: 2\n prefer-const: 2\n no-inner-declarations: 0\n newline-per-chained-call: 2\n\n mocha/no-exclusive-tests: 2\n mocha/no-hooks-for-single-case: 0\n mocha/no-mocha-arrows: 0\n mocha/no-setup-in-describe: 0\n mocha/no-sibling-hooks: 0\n mocha/no-skipped-tests: 0\n\n node/no-deprecated-api: 0\n" }, { "path": ".github/issue_template.md", "content": "\n\n## Description\n\n## Minimal, Working Test code to reproduce the issue.\n#### (An easy to reproduce test case will dramatically decrease the resolution time.)\n\n## Bull version\n\n## Additional information\n\n" }, { "path": ".github/stale.yml", "content": "# Number of days of inactivity before an issue becomes stale\ndaysUntilStale: 60\n# Number of days of inactivity before a stale issue is closed\ndaysUntilClose: 7\n# Issues with these labels will never be considered stale\nexemptLabels:\n - pinned\n - security\n - enhancement\n - BETTER DOC\n - bug\n# Label to use when marking an issue as stale\nstaleLabel: wontfix\n# Comment to post when marking an issue as stale. Set to `false` to disable\nmarkComment: >\n This issue has been automatically marked as stale because it has not had\n recent activity. It will be closed if no further activity occurs. Thank you\n for your contributions.\n# Comment to post when closing a stale issue. Set to `false` to disable\ncloseComment: false\n" }, { "path": ".github/workflows/codeql-analysis.yml", "content": "# For most projects, this workflow file will not need changing; you simply need\n# to commit it to your repository.\n#\n# You may wish to alter this file to override the set of languages analyzed,\n# or to provide custom queries or build logic.\n#\n# ******** NOTE ********\n# We have attempted to detect the languages in your repository. Please check\n# the `language` matrix defined below to confirm you have the correct set of\n# supported CodeQL languages.\n#\nname: \"CodeQL\"\n\non:\n push:\n branches: [ develop ]\n pull_request:\n # The branches below must be a subset of the branches above\n branches: [ develop ]\n schedule:\n - cron: '24 2 * * 0'\n\njobs:\n analyze:\n name: Analyze\n runs-on: ubuntu-latest\n permissions:\n actions: read\n contents: read\n security-events: write\n\n strategy:\n fail-fast: false\n matrix:\n language: [ 'javascript' ]\n # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]\n # Learn more about CodeQL language support at https://git.io/codeql-language-support\n\n steps:\n - name: Checkout repository\n uses: actions/checkout@v2\n\n # Initializes the CodeQL tools for scanning.\n - name: Initialize CodeQL\n uses: github/codeql-action/init@v1\n with:\n languages: ${{ matrix.language }}\n # If you wish to specify custom queries, you can do so here or in a config file.\n # By default, queries listed here will override any specified in a config file.\n # Prefix the list here with \"+\" to use these queries and those in the config file.\n # queries: ./path/to/local/query, your-org/your-repo/queries@main\n\n # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).\n # If this step fails, then you should remove it and run the build manually (see below)\n - name: Autobuild\n uses: github/codeql-action/autobuild@v1\n\n # ℹī¸ Command-line programs to run using the OS shell.\n # 📚 https://git.io/JvXDl\n\n # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines\n # and modify them (or add more) to build your code if your project\n # uses a compiled language\n\n #- run: |\n # make bootstrap\n # make release\n\n - name: Perform CodeQL Analysis\n uses: github/codeql-action/analyze@v1\n" }, { "path": ".github/workflows/node.js.yml", "content": "# This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node\n# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions\n\nname: Node.js CI\n\non:\n push:\n branches: [develop]\n pull_request:\n branches: [develop]\n\npermissions:\n contents: read # to fetch code (actions/checkout)\n\njobs:\n build:\n runs-on: ubuntu-latest\n\n services:\n redis:\n image: redis\n ports:\n - 6379:6379\n\n strategy:\n matrix:\n node-version: [12.x, 14.x, 16.x]\n redis-version: [7.0-alpine]\n include:\n - node-version: '16.x'\n redis-version: 6-alpine\n\n steps:\n - uses: actions/checkout@v2\n - name: Use Node.js ${{ matrix.node-version }}\n uses: actions/setup-node@v1\n with:\n node-version: ${{ matrix.node-version }}\n - run: yarn install --frozen-lockfile --non-interactive\n - run: yarn prettier -- --list-different\n - run: yarn test\n" }, { "path": ".github/workflows/release.yml", "content": "name: Release\non:\n push:\n branches:\n - develop\npermissions: {}\njobs:\n release:\n permissions:\n contents: write # for release publishing\n\n name: Release\n runs-on: ubuntu-latest\n steps:\n - name: Checkout\n uses: actions/checkout@v2\n with:\n fetch-depth: 0\n - name: Setup Node.js\n uses: actions/setup-node@v1\n with:\n node-version: 12\n - name: Install dependencies\n run: yarn install --frozen-lockfile --non-interactive\n - name: Generate scripts\n run: yarn pretest\n - name: Release\n env:\n GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}\n NPM_TOKEN: ${{ secrets.NPM_TOKEN }}\n run: npx semantic-release\n" }, { "path": ".gitignore", "content": "node_modules\ntmp\ncoverage\n*.rdb\n.vscode\npackage-lock.json\n.nyc_output\nrawScripts\nlib/scripts\n" }, { "path": ".mocharc.json", "content": "{\n \"timeout\": 5000,\n \"reporter\": \"spec\",\n \"exit\": true\n}\n" }, { "path": ".npmignore", "content": "node_modules\ntmp\ntest\nbugs\ndocs\nsupport\n.github\n.editorconfig\n.eslintrc.yml\n.travis.yml\n.gitignore\n*.md\ncommitlint.config.js\nrawScripts\ncommandTransform.js\ndocker-compose.yml\ngenerateRawScripts.js\n.nyc_output\n.mocharc.json\nlib/commands/*.js\n" }, { "path": "CHANGELOG.md", "content": "## [4.16.5](https://github.com/OptimalBits/bull/compare/v4.16.4...v4.16.5) (2024-12-18)\n\n\n### Bug Fixes\n\n* upgrade cron-parser dependency for Luxon CVE-2023-22467 ([e45698e](https://github.com/OptimalBits/bull/commit/e45698eb91b91a4676f9bafdb7bdb35043d2316d))\n\n## [4.16.4](https://github.com/OptimalBits/bull/compare/v4.16.3...v4.16.4) (2024-11-01)\n\n\n### Bug Fixes\n\n* **deps:** bump msgpackr to 1.1.2 to resolve ERR_BUFFER_OUT_OF_BOUNDS error ([#2783](https://github.com/OptimalBits/bull/issues/2783)) fixes [#2782](https://github.com/OptimalBits/bull/issues/2782) ([bc0ae0a](https://github.com/OptimalBits/bull/commit/bc0ae0ae5b7fb2b067de94c3e51ce1bdd609806d))\n\n## [4.16.3](https://github.com/OptimalBits/bull/compare/v4.16.2...v4.16.3) (2024-09-10)\n\n\n### Bug Fixes\n\n* **metrics:** differentiate points in different minutes to be more accurate ([#2770](https://github.com/OptimalBits/bull/issues/2770)) ([fbf2fa3](https://github.com/OptimalBits/bull/commit/fbf2fa397c865e6620aac39b291704e486f424d2))\n\n## [4.16.2](https://github.com/OptimalBits/bull/compare/v4.16.1...v4.16.2) (2024-09-05)\n\n\n### Performance Improvements\n\n* **metrics:** save zeros as much as max data points ([#2767](https://github.com/OptimalBits/bull/issues/2767)) ([3a09840](https://github.com/OptimalBits/bull/commit/3a098406a86573150ad13d1487d16211030c4200))\n\n## [4.16.1](https://github.com/OptimalBits/bull/compare/v4.16.0...v4.16.1) (2024-08-28)\n\n\n### Bug Fixes\n\n* **metrics:** use batches include when collecting metrics ([#2765](https://github.com/OptimalBits/bull/issues/2765)) fixes [#2764](https://github.com/OptimalBits/bull/issues/2764) [#2763](https://github.com/OptimalBits/bull/issues/2763) ([8276f72](https://github.com/OptimalBits/bull/commit/8276f72b30a7abc2f3fb5509f980d931027d0f0c))\n\n# [4.16.0](https://github.com/OptimalBits/bull/compare/v4.15.1...v4.16.0) (2024-07-30)\n\n\n### Features\n\n* **job:** support debouncing ([#2760](https://github.com/OptimalBits/bull/issues/2760)) ([603befe](https://github.com/OptimalBits/bull/commit/603befe439fb4e318137b88550ab4d0c6c2bbcb3))\n\n## [4.15.1](https://github.com/OptimalBits/bull/compare/v4.15.0...v4.15.1) (2024-07-04)\n\n\n### Bug Fixes\n\n* **job:** check jobKey when saving stacktrace ([#2755](https://github.com/OptimalBits/bull/issues/2755)) ([96675f5](https://github.com/OptimalBits/bull/commit/96675f51d1a9e5a30df0a0fe24a40a2d14078203))\n\n# [4.15.0](https://github.com/OptimalBits/bull/compare/v4.14.0...v4.15.0) (2024-06-30)\n\n\n### Features\n\n* **queue:** emit internal duplicated event ([#2754](https://github.com/OptimalBits/bull/issues/2754)) ([021ab7f](https://github.com/OptimalBits/bull/commit/021ab7fd8d6c89b01767572b0e53422f34f7bc04))\n\n# [4.14.0](https://github.com/OptimalBits/bull/compare/v4.13.1...v4.14.0) (2024-06-25)\n\n\n### Features\n\n* **queue:** add global:duplicated event when a duplicated is added ([#2749](https://github.com/OptimalBits/bull/issues/2749)) ([d632ac1](https://github.com/OptimalBits/bull/commit/d632ac1010d7d7faaa8f731eafdea67d6129e111))\n\n## [4.13.1](https://github.com/OptimalBits/bull/compare/v4.13.0...v4.13.1) (2024-06-21)\n\n\n### Bug Fixes\n\n* **priority:** consider paused state when calling getCountsPerPriority ([#2748](https://github.com/OptimalBits/bull/issues/2748)) ([6c2719a](https://github.com/OptimalBits/bull/commit/6c2719a9292bf0a11136d575d333600be0b9a422))\n\n# [4.13.0](https://github.com/OptimalBits/bull/compare/v4.12.9...v4.13.0) (2024-06-12)\n\n\n### Features\n\n* **queue:** add getCountsPerPriority method ([#2746](https://github.com/OptimalBits/bull/issues/2746)) ([0376dcc](https://github.com/OptimalBits/bull/commit/0376dcc128d9af13ecbd658d8ea7ff19fce56915))\n\n## [4.12.9](https://github.com/OptimalBits/bull/compare/v4.12.8...v4.12.9) (2024-05-24)\n\n\n### Bug Fixes\n\n* **retry-job:** throw error when job is not in active state ([#2741](https://github.com/OptimalBits/bull/issues/2741)) ([c29e3b0](https://github.com/OptimalBits/bull/commit/c29e3b061a168c3d40a336a733c382bab14caa57))\n\n## [4.12.8](https://github.com/OptimalBits/bull/compare/v4.12.7...v4.12.8) (2024-05-22)\n\n\n### Bug Fixes\n\n* **move-to-finished:** throw error when job is not in active state ([#2739](https://github.com/OptimalBits/bull/issues/2739)) ([7b12be1](https://github.com/OptimalBits/bull/commit/7b12be13ea39e309d6a6a39a2e56f646f06fdfa8))\n\n## [4.12.7](https://github.com/OptimalBits/bull/compare/v4.12.6...v4.12.7) (2024-05-21)\n\n\n### Bug Fixes\n\n* **scripts:** throw error when moving non-active job to delayed ([#2740](https://github.com/OptimalBits/bull/issues/2740)) ([63636b1](https://github.com/OptimalBits/bull/commit/63636b181d0a166b1702059e5abb53ce1589d90f))\n\n## [4.12.6](https://github.com/OptimalBits/bull/compare/v4.12.5...v4.12.6) (2024-05-18)\n\n\n### Bug Fixes\n\n* **stalled:** take in count removeOnFail option ([#2734](https://github.com/OptimalBits/bull/issues/2734)) ([2112269](https://github.com/OptimalBits/bull/commit/21122697461b551055192ff3b9c02c6f37cb331e))\n\n## [4.12.5](https://github.com/OptimalBits/bull/compare/v4.12.4...v4.12.5) (2024-05-17)\n\n\n### Bug Fixes\n\n* **job:** validate job existence when adding log ([#2738](https://github.com/OptimalBits/bull/issues/2738)) ([1fb1562](https://github.com/OptimalBits/bull/commit/1fb15628dc5912e83220d430aa14e19e90d1cc53))\n\n## [4.12.4](https://github.com/OptimalBits/bull/compare/v4.12.3...v4.12.4) (2024-05-15)\n\n\n### Bug Fixes\n\n* **retry-job:** consider priority ([#2737](https://github.com/OptimalBits/bull/issues/2737)) fixes [#1755](https://github.com/OptimalBits/bull/issues/1755) ([09ce146](https://github.com/OptimalBits/bull/commit/09ce146563871519cda638bafa82ce6af34bdd25))\n\n## [4.12.3](https://github.com/OptimalBits/bull/compare/v4.12.2...v4.12.3) (2024-05-10)\n\n\n### Bug Fixes\n\n* **job:** validate jobKey in updateProgress and update ([#2730](https://github.com/OptimalBits/bull/issues/2730)) ([6d84156](https://github.com/OptimalBits/bull/commit/6d8415696606d3b7ec891f7fca9ab0508923c321))\n\n\n### Performance Improvements\n\n* **scripts:** remove token after moving to wait or delayed ([#2731](https://github.com/OptimalBits/bull/issues/2731)) ([7ee8f74](https://github.com/OptimalBits/bull/commit/7ee8f7430a68492c9ce768e7108443592f49d74c))\n\n## [4.12.2](https://github.com/OptimalBits/bull/compare/v4.12.1...v4.12.2) (2024-01-17)\n\n\n### Bug Fixes\n\n* **dependencies:** upgrade msgpackr ([cc83ae2](https://github.com/OptimalBits/bull/commit/cc83ae297e96344f94039401be5097d7f05ab10b))\n\n## [4.12.1](https://github.com/OptimalBits/bull/compare/v4.12.0...v4.12.1) (2024-01-15)\n\n\n### Bug Fixes\n\n* **deps:** bump msgpackr from 1.5.2 to 1.10.1 ([#2697](https://github.com/OptimalBits/bull/issues/2697)) ([b27c90d](https://github.com/OptimalBits/bull/commit/b27c90d8b106cc5319e712df2386ddc35946ec3d))\n\n# [4.12.0](https://github.com/OptimalBits/bull/compare/v4.11.5...v4.12.0) (2023-12-18)\n\n\n### Features\n\n* add missing extendLock definition ([14432ff](https://github.com/OptimalBits/bull/commit/14432ff8a5e743d97c82a360c9d6a92204b1a684))\n\n## [4.11.5](https://github.com/OptimalBits/bull/compare/v4.11.4...v4.11.5) (2023-11-11)\n\n\n### Bug Fixes\n\n* pass redis string as opts into queue ([e94f568](https://github.com/OptimalBits/bull/commit/e94f568085de079fc42b876233a060ba11ec946e))\n\n## [4.11.4](https://github.com/OptimalBits/bull/compare/v4.11.3...v4.11.4) (2023-10-14)\n\n\n### Bug Fixes\n\n* catch pause errors when closing ([ccb6cc7](https://github.com/OptimalBits/bull/commit/ccb6cc7ecca8c726fc14509536831f665ac49701))\n\n## [4.11.3](https://github.com/OptimalBits/bull/compare/v4.11.2...v4.11.3) (2023-08-11)\n\n\n### Bug Fixes\n\n* **types:** make repeat option key optional ([934ec98](https://github.com/OptimalBits/bull/commit/934ec9875c6e04c3e771b6c6ba212e3693d3a25b))\n\n## [4.11.2](https://github.com/OptimalBits/bull/compare/v4.11.1...v4.11.2) (2023-08-08)\n\n\n### Bug Fixes\n\n* **worker:** better client name support ([5910f44](https://github.com/OptimalBits/bull/commit/5910f44b03a264d979c8ade54d64d13fdc908b51))\n\n## [4.11.1](https://github.com/OptimalBits/bull/compare/v4.11.0...v4.11.1) (2023-08-08)\n\n\n### Bug Fixes\n\n* **queue:** deep clone opts ([#2634](https://github.com/OptimalBits/bull/issues/2634)) fixes [#2633](https://github.com/OptimalBits/bull/issues/2633) ([35f1da3](https://github.com/OptimalBits/bull/commit/35f1da3cf631bee97e96a774d9f1127466e7a66a))\n\n# [4.11.0](https://github.com/OptimalBits/bull/compare/v4.10.4...v4.11.0) (2023-08-08)\n\n\n### Bug Fixes\n\n* add mising getMetrics type ([#2640](https://github.com/OptimalBits/bull/issues/2640)) ([a217a7d](https://github.com/OptimalBits/bull/commit/a217a7d56d52385eb56ffe386b7503eca9a24604))\n* remove deprecated debuglog ([4ce36fe](https://github.com/OptimalBits/bull/commit/4ce36febe3a63a45198e2fe24b46fc371ee3f6e5))\n* **types:** add missing keys to repeat opts ([e4e6457](https://github.com/OptimalBits/bull/commit/e4e64572a3ad259d9cb90d5dec81e8565eeadca1))\n* **types:** rename strategyOptions to options to reflect js file ([bae6427](https://github.com/OptimalBits/bull/commit/bae6427ce9d9fac26b198402068bd84647fd8208))\n* **typings:** return type of getJobCountByTypes ([#2622](https://github.com/OptimalBits/bull/issues/2622)) ([47722ed](https://github.com/OptimalBits/bull/commit/47722ed791429b087128ce5f35847663b2d8fc9c))\n* **worker:** high-memory-usage-when-providing-float-to-concurrency ([#2620](https://github.com/OptimalBits/bull/issues/2620)) ([dcca1e8](https://github.com/OptimalBits/bull/commit/dcca1e8c39b121fb01ac299bec30a3d011059c1f))\n* change option name to match ts declaration ([909a07e](https://github.com/OptimalBits/bull/commit/909a07e27075a63b9ca178a3074b0b5c80d86355))\n* ts declaration metrics option and getMetrics function ([11331b7](https://github.com/OptimalBits/bull/commit/11331b718a8e534ac6822917a536eab32b10446b))\n\n\n### Features\n\n* upgrade ioredis to 5.3.2 ([e1883f0](https://github.com/OptimalBits/bull/commit/e1883f01c2cb23a51b5485ef8048c4268ee968ea))\n\n## [4.10.4](https://github.com/OptimalBits/bull/compare/v4.10.3...v4.10.4) (2023-02-09)\n\n\n### Bug Fixes\n\n* **retry:** handle pause queue status ([9f945d6](https://github.com/OptimalBits/bull/commit/9f945d60c69e8b5d7b46f58189a1c49a83897099))\n\n## [4.10.3](https://github.com/OptimalBits/bull/compare/v4.10.2...v4.10.3) (2023-02-03)\n\n\n### Bug Fixes\n\n* don't reschedule delay timer if closing ([#2535](https://github.com/OptimalBits/bull/issues/2535)) ([8a0292a](https://github.com/OptimalBits/bull/commit/8a0292a574df82a62d718e13d8995800fd8529d0))\n\n## [4.10.2](https://github.com/OptimalBits/bull/compare/v4.10.1...v4.10.2) (2022-11-24)\n\n\n### Bug Fixes\n\n* **queue:** throw error when needed instead of hiding it in a closure ([8a742c1](https://github.com/OptimalBits/bull/commit/8a742c1176e7147e2069602f18089d8becb4cb15))\n\n## [4.10.1](https://github.com/OptimalBits/bull/compare/v4.10.0...v4.10.1) (2022-10-13)\n\n\n### Bug Fixes\n\n* support for instantiation using redisUrl ([6288f7d](https://github.com/OptimalBits/bull/commit/6288f7de9b82e712e480510eb10c03bd4d1cd24e))\n\n# [4.10.0](https://github.com/OptimalBits/bull/compare/v4.9.0...v4.10.0) (2022-09-29)\n\n\n### Features\n\n* **types:** add typescript types to package ([e793f8d](https://github.com/OptimalBits/bull/commit/e793f8d1502bc5ed4a2e15087dc048c18e5e8644))\n\n# [4.9.0](https://github.com/OptimalBits/bull/compare/v4.8.5...v4.9.0) (2022-09-05)\n\n\n### Features\n\n* support .cjs files ([75e6775](https://github.com/OptimalBits/bull/commit/75e6775fc3720563aac9cd9a07d1722dfbdfa177))\n\n## [4.8.5](https://github.com/OptimalBits/bull/compare/v4.8.4...v4.8.5) (2022-07-27)\n\n\n### Performance Improvements\n\n* **clean:** use ZRANGEBYSCORE to improve performance ([#2363](https://github.com/OptimalBits/bull/issues/2363)) ([3331188](https://github.com/OptimalBits/bull/commit/3331188bce510e0bb4749d92cb63f4c73203d076))\n\n## [4.8.4](https://github.com/OptimalBits/bull/compare/v4.8.3...v4.8.4) (2022-06-16)\n\n\n### Bug Fixes\n\n* **worker:** better disconnect when blocking connection ([b9ea7f4](https://github.com/OptimalBits/bull/commit/b9ea7f4780948d4556548e6bf13e2c3271939d12))\n\n## [4.8.3](https://github.com/OptimalBits/bull/compare/v4.8.2...v4.8.3) (2022-05-12)\n\n\n### Bug Fixes\n\n* **stalled-jobs:** move stalled jobs to wait in batches ([2f1fb6c](https://github.com/OptimalBits/bull/commit/2f1fb6cdc1329f98b2dd30e847b5a79839db0346))\n\n## [4.8.2](https://github.com/OptimalBits/bull/compare/v4.8.1...v4.8.2) (2022-04-21)\n\n\n### Bug Fixes\n\n* unlock job when moving it to delayed ([#2329](https://github.com/OptimalBits/bull/issues/2329)) ([11eae6b](https://github.com/OptimalBits/bull/commit/11eae6b960c83b47dadb22991b2b3e239c177508))\n\n## [4.8.1](https://github.com/OptimalBits/bull/compare/v4.8.0...v4.8.1) (2022-03-21)\n\n\n### Performance Improvements\n\n* speed up clean operation ([#2326](https://github.com/OptimalBits/bull/issues/2326)) ([ef5f471](https://github.com/OptimalBits/bull/commit/ef5f4717258940042fa11f980622034cde765860))\n\n# [4.8.0](https://github.com/OptimalBits/bull/compare/v4.7.0...v4.8.0) (2022-03-19)\n\n\n### Features\n\n* have Queue#clean consult job.{finishedOn,processedOn,timestamp} ([#2309](https://github.com/OptimalBits/bull/issues/2309)) ([b7058e6](https://github.com/OptimalBits/bull/commit/b7058e6e8f4bb56febebaf9452f993883240fa9d))\n\n# [4.7.0](https://github.com/OptimalBits/bull/compare/v4.6.2...v4.7.0) (2022-03-02)\n\n\n### Features\n\n* **metrics:** add support for collecting queue metrics ([886d764](https://github.com/OptimalBits/bull/commit/886d7643819dcf52902d2e92394267dbd495c71b))\n\n## [4.6.2](https://github.com/OptimalBits/bull/compare/v4.6.1...v4.6.2) (2022-02-23)\n\n\n### Bug Fixes\n\n* better handling of maxRetriesPerRequest ([d3b9138](https://github.com/OptimalBits/bull/commit/d3b91386e30d7205efdc19bcd18fe1e5fefa3542))\n\n## [4.6.1](https://github.com/OptimalBits/bull/compare/v4.6.0...v4.6.1) (2022-02-21)\n\n\n### Bug Fixes\n\n* **sandbox:** better error reporting broken processor file ([10db479](https://github.com/OptimalBits/bull/commit/10db479731bcb8ba27c3a0e2dd4094c8e9ff1c57))\n\n# [4.6.0](https://github.com/OptimalBits/bull/compare/v4.5.6...v4.6.0) (2022-02-21)\n\n\n### Features\n\n* handle redis uri queries ([54e5463](https://github.com/OptimalBits/bull/commit/54e5463bffee1b8b56b460460c79d8751142d859))\n\n## [4.5.6](https://github.com/OptimalBits/bull/compare/v4.5.5...v4.5.6) (2022-02-20)\n\n\n### Bug Fixes\n\n* **sandbox:** wait for result of sending start command ([232ed85](https://github.com/OptimalBits/bull/commit/232ed85d4c980d94dee5d9e4c5b6f8758dbb82d9))\n\n## [4.5.5](https://github.com/OptimalBits/bull/compare/v4.5.4...v4.5.5) (2022-02-16)\n\n\n### Bug Fixes\n\n* **worker:** better closing when disconnected ([41b9404](https://github.com/OptimalBits/bull/commit/41b940457b3447619c3c2887674a8cebf1508b07))\n\n## [4.5.4](https://github.com/OptimalBits/bull/compare/v4.5.3...v4.5.4) (2022-02-14)\n\n\n### Bug Fixes\n\n* **queue:** return correct workers with getWorkers() ([193644c](https://github.com/OptimalBits/bull/commit/193644c5ed290901448f8c35ede99e1063a90f4a))\n\n## [4.5.3](https://github.com/OptimalBits/bull/compare/v4.5.2...v4.5.3) (2022-02-14)\n\n\n### Bug Fixes\n\n* **commands:** do not wait for redis to load commands ([ad7b647](https://github.com/OptimalBits/bull/commit/ad7b6474db426b4970a0d1d3ddb8a032a22c481b))\n\n## [4.5.2](https://github.com/OptimalBits/bull/compare/v4.5.1...v4.5.2) (2022-02-14)\n\n\n### Bug Fixes\n\n* **scripts:** make it easier for tools like vercel to find the .lua scripts ([8ab5b1d](https://github.com/OptimalBits/bull/commit/8ab5b1d1c3eecf41b20d97f464030377ece01640))\n\n## [4.5.1](https://github.com/OptimalBits/bull/compare/v4.5.0...v4.5.1) (2022-02-06)\n\n\n### Bug Fixes\n\n* **sandbox:** broken processor files should fail jobs ([dd0b853](https://github.com/OptimalBits/bull/commit/dd0b853a516c32baac535395377a19e361964dc8))\n\n# [4.5.0](https://github.com/OptimalBits/bull/compare/v4.4.0...v4.5.0) (2022-02-01)\n\n\n### Features\n\n* **queue:** add retryJobs for failed status ([501b2cc](https://github.com/OptimalBits/bull/commit/501b2cc49ccd0d7de82fea50144b52fa9879d1f5))\n\n# [4.4.0](https://github.com/OptimalBits/bull/compare/v4.3.0...v4.4.0) (2022-01-26)\n\n\n### Features\n\n* add support for removeOn based on time ([90f040c](https://github.com/OptimalBits/bull/commit/90f040c052325da302c99f17111a12d1afbe88bd))\n\n# [4.3.0](https://github.com/OptimalBits/bull/compare/v4.2.1...v4.3.0) (2022-01-26)\n\n\n### Features\n\n* upgrade cron version enabling new cron expressions ([79337a3](https://github.com/OptimalBits/bull/commit/79337a30758ea7e6d6b1536eb0edebcd0b3c8274))\n\n## [4.2.1](https://github.com/OptimalBits/bull/compare/v4.2.0...v4.2.1) (2022-01-17)\n\n\n### Bug Fixes\n\n* **sandbox:** exit if uncaughtException ([43dc2e6](https://github.com/OptimalBits/bull/commit/43dc2e69dfa0cbaf960ce6f1bd6c4125b1052ff9))\n\n# [4.2.0](https://github.com/OptimalBits/bull/compare/v4.1.4...v4.2.0) (2021-12-21)\n\n\n### Features\n\n* **queue:** enabled queues to share childPool instance ([#2237](https://github.com/OptimalBits/bull/issues/2237)) ([16fdbe9](https://github.com/OptimalBits/bull/commit/16fdbe90a05ae2e2bcb134a550a97dc84dfe573e))\n\n## [4.1.4](https://github.com/OptimalBits/bull/compare/v4.1.3...v4.1.4) (2021-12-14)\n\n\n### Bug Fixes\n\n* **queue:** check redisOptions is available fixes [#2186](https://github.com/OptimalBits/bull/issues/2186) ([071c51d](https://github.com/OptimalBits/bull/commit/071c51d16db58ed9f71138058dbbc25f73383e56))\n\n## [4.1.3](https://github.com/OptimalBits/bull/compare/v4.1.2...v4.1.3) (2021-12-14)\n\n\n### Bug Fixes\n\n* typo on url ([#2195](https://github.com/OptimalBits/bull/issues/2195)) ([7e7d9cb](https://github.com/OptimalBits/bull/commit/7e7d9cb58d7cb5c1a92020f7c4333932526d8e98))\n\n## [4.1.2](https://github.com/OptimalBits/bull/compare/v4.1.1...v4.1.2) (2021-12-14)\n\n\n### Performance Improvements\n\n* speed up performance of queue.clean when called with a limit ([#2205](https://github.com/OptimalBits/bull/issues/2205)) ([c20e469](https://github.com/OptimalBits/bull/commit/c20e469dcd71fd13e23e922f2720f55450311d22))\n\n## [4.1.1](https://github.com/OptimalBits/bull/compare/v4.1.0...v4.1.1) (2021-11-16)\n\n\n### Bug Fixes\n\n* **emit:** protect emit calls fixes [#2213](https://github.com/OptimalBits/bull/issues/2213) ([4978a2b](https://github.com/OptimalBits/bull/commit/4978a2b40ee840ba91e0939e86e1e3b15e8b16e9))\n\n# [4.1.0](https://github.com/OptimalBits/bull/compare/v4.0.0...v4.1.0) (2021-10-31)\n\n\n### Features\n\n* emit event on job lock extend failure ([7247b3b](https://github.com/OptimalBits/bull/commit/7247b3bb9741c5eb18ce4027ea14349cbc8504c5))\n\n# [4.0.0](https://github.com/OptimalBits/bull/compare/v3.29.3...v4.0.0) (2021-10-27)\n\n\n### Bug Fixes\n\n* force options to guarantee correct reconnects ([3ade8e6](https://github.com/OptimalBits/bull/commit/3ade8e6727d7b906a30b09bccb6dc10d76ed1b5f))\n\n\n### BREAKING CHANGES\n\n* If redis opts are missing:\n { maxRetriesPerRequest: null,\n enableReadyCheck: false }\nthen a exception will be thrown.\n\n## [3.29.3](https://github.com/OptimalBits/bull/compare/v3.29.2...v3.29.3) (2021-10-13)\n\n\n### Bug Fixes\n\n* **name-processors:** wait for all processors when closing fixes [#1618](https://github.com/OptimalBits/bull/issues/1618) ([79ce013](https://github.com/OptimalBits/bull/commit/79ce013af695f96ff57106b213982647e0783d3f))\n\n## [3.29.2](https://github.com/OptimalBits/bull/compare/v3.29.1...v3.29.2) (2021-09-08)\n\n\n### Bug Fixes\n\n* **connection:** fail only if redis connection does not recover ([0ca4c6b](https://github.com/OptimalBits/bull/commit/0ca4c6b4d57efa78e5ca484cb8bed2a6961646a3))\n\n## [3.29.1](https://github.com/OptimalBits/bull/compare/v3.29.0...v3.29.1) (2021-08-26)\n\n\n### Bug Fixes\n\n* protect getJob with isReady, fixes [#1386](https://github.com/OptimalBits/bull/issues/1386) ([2f27faa](https://github.com/OptimalBits/bull/commit/2f27faa410f70504f24df2ebd6bb7831df21660d))\n\n# [3.29.0](https://github.com/OptimalBits/bull/compare/v3.28.1...v3.29.0) (2021-08-20)\n\n\n### Features\n\n* **jobs:** add extendLock method ([30d5959](https://github.com/OptimalBits/bull/commit/30d59590c34cb664e3b9a62695c4092c9b1ae3f3))\n\n## [3.28.1](https://github.com/OptimalBits/bull/compare/v3.28.0...v3.28.1) (2021-08-06)\n\n\n### Bug Fixes\n\n* **queue:** changed user prop to username for redisOptions ([71baea9](https://github.com/OptimalBits/bull/commit/71baea91de702d9bd1d5516f09b22599b7f13045))\n\n# [3.28.0](https://github.com/OptimalBits/bull/compare/v3.27.0...v3.28.0) (2021-08-05)\n\n\n### Features\n\n* **queue:** handle redis url containing username ([a245fc4](https://github.com/OptimalBits/bull/commit/a245fc403827fd22b08dbe1499fe843eb1ad633a))\n\n# [3.27.0](https://github.com/OptimalBits/bull/compare/v3.26.0...v3.27.0) (2021-07-27)\n\n\n### Features\n\n* support job.discard function in sandboxed processors ([5adcf2c](https://github.com/OptimalBits/bull/commit/5adcf2ceed263d46089cceefcdcaa658151c53b7))\n\n# [3.26.0](https://github.com/OptimalBits/bull/compare/v3.25.2...v3.26.0) (2021-07-16)\n\n\n### Features\n\n* **repeatable:** store key in repeat options ([dab0d82](https://github.com/OptimalBits/bull/commit/dab0d8266174f1d25ec914cff8450594d85db511))\n\n## [3.25.2](https://github.com/OptimalBits/bull/compare/v3.25.1...v3.25.2) (2021-07-16)\n\n\n### Bug Fixes\n\n* **repeatable:** honor endDate fixes [#1573](https://github.com/OptimalBits/bull/issues/1573) ([7f0db0e](https://github.com/OptimalBits/bull/commit/7f0db0e293367752971be8a1bf7b6c8cf4190350))\n\n## [3.25.1](https://github.com/OptimalBits/bull/compare/v3.25.0...v3.25.1) (2021-07-16)\n\n\n### Bug Fixes\n\n* error when .lua scripts missing in built bundle ([85307c3](https://github.com/OptimalBits/bull/commit/85307c3a8e48b910590b2682700c5062c84d32b0))\n\n# [3.25.0](https://github.com/OptimalBits/bull/compare/v3.24.0...v3.25.0) (2021-07-15)\n\n\n### Features\n\n* pass clientName to createClient function ([2a29569](https://github.com/OptimalBits/bull/commit/2a295691b88318ff64beaa7b83e03487854a7fe4))\n\n# [3.24.0](https://github.com/OptimalBits/bull/compare/v3.23.3...v3.24.0) (2021-07-15)\n\n\n### Features\n\n* **backoff:** add option to specify options for custom backoff strategy ([e573010](https://github.com/OptimalBits/bull/commit/e5730107688a41c7268a717f16302c5959eaf6f6))\n\n## [3.23.3](https://github.com/OptimalBits/bull/compare/v3.23.2...v3.23.3) (2021-07-15)\n\n\n### Bug Fixes\n\n* **delayed:** do not get lock twice fixes [#2033](https://github.com/OptimalBits/bull/issues/2033) ([2800cf8](https://github.com/OptimalBits/bull/commit/2800cf8923ccac52dcd4da4e7fd19b3404c68fe8))\n\n## [3.23.2](https://github.com/OptimalBits/bull/compare/v3.23.1...v3.23.2) (2021-07-15)\n\n\n### Bug Fixes\n\n* **job:** add default err object fixes [#2029](https://github.com/OptimalBits/bull/issues/2029) ([39684e9](https://github.com/OptimalBits/bull/commit/39684e9f941e2ea4191809fdc2aa52b3d7b267ae))\n\n## [3.23.1](https://github.com/OptimalBits/bull/compare/v3.23.0...v3.23.1) (2021-07-15)\n\n\n### Bug Fixes\n\n* wait in queue to be ready in getNextJob fixes [#1852](https://github.com/OptimalBits/bull/issues/1852) ([4e224e5](https://github.com/OptimalBits/bull/commit/4e224e5533f729b9781b1db81e1875b1bd50afb0))\n\n# [3.23.0](https://github.com/OptimalBits/bull/compare/v3.22.12...v3.23.0) (2021-07-13)\n\n\n### Features\n\n* support job.update function in sandboxed processors ([ff79fb4](https://github.com/OptimalBits/bull/commit/ff79fb494ea3e123256b35d18b46b24fbb9b8365)), closes [#1279](https://github.com/OptimalBits/bull/issues/1279) [#1608](https://github.com/OptimalBits/bull/issues/1608) [#1056](https://github.com/OptimalBits/bull/issues/1056)\n\n## [3.22.12](https://github.com/OptimalBits/bull/compare/v3.22.11...v3.22.12) (2021-07-13)\n\n\n### Bug Fixes\n\n* remove stalled job when finishing fixes [#1600](https://github.com/OptimalBits/bull/issues/1600) ([90763fd](https://github.com/OptimalBits/bull/commit/90763fd66404d8bc5a47ff8555cbe2da776c030d))\n\n## [3.22.11](https://github.com/OptimalBits/bull/compare/v3.22.10...v3.22.11) (2021-07-08)\n\n\n### Bug Fixes\n\n* **close:** clear timers after waiting jobs fixes [#1415](https://github.com/OptimalBits/bull/issues/1415) ([77b319d](https://github.com/OptimalBits/bull/commit/77b319da6acaa93351830f0a4e38b5ad1d3d8cf5))\n\n## [3.22.10](https://github.com/OptimalBits/bull/compare/v3.22.9...v3.22.10) (2021-07-01)\n\n\n### Bug Fixes\n\n* deep clone options ([#2083](https://github.com/OptimalBits/bull/issues/2083)) ([1e00a90](https://github.com/OptimalBits/bull/commit/1e00a90d1216083cdf33bc78a9753ece1acdddb2))\n\n## [3.22.9](https://github.com/OptimalBits/bull/compare/v3.22.8...v3.22.9) (2021-06-22)\n\n\n### Bug Fixes\n\n* **reprocess:** do not store job.id in added list ([3fbc506](https://github.com/OptimalBits/bull/commit/3fbc506aee4c36fd612ed18fc3a1619c0c8069d8))\n\n## [3.22.8](https://github.com/OptimalBits/bull/compare/v3.22.7...v3.22.8) (2021-06-09)\n\n\n### Bug Fixes\n\n* upgrade ioredis fixes [#1445](https://github.com/OptimalBits/bull/issues/1445) ([f6a2364](https://github.com/OptimalBits/bull/commit/f6a23648146f5a53bb2e4084f8de9b783b782bed))\n\n## [3.22.7](https://github.com/OptimalBits/bull/compare/v3.22.6...v3.22.7) (2021-05-31)\n\n\n### Bug Fixes\n\n* **obliterate:** remove job logs fixes [#2050](https://github.com/OptimalBits/bull/issues/2050) ([6ccf2b9](https://github.com/OptimalBits/bull/commit/6ccf2b90892bf16b4edb0095d6e770f978bc660a))\n\n## [3.22.6](https://github.com/OptimalBits/bull/compare/v3.22.5...v3.22.6) (2021-05-17)\n\n\n### Bug Fixes\n\n* **job:** fix job log pagination, fixes [#2031](https://github.com/OptimalBits/bull/issues/2031) ([30aa0a9](https://github.com/OptimalBits/bull/commit/30aa0a99acb4a04a12f988840bc8ccc7a014fed3))\n\n## [3.22.5](https://github.com/OptimalBits/bull/compare/v3.22.4...v3.22.5) (2021-05-11)\n\n\n### Bug Fixes\n\n* emit failed event when stalled job fails ([f68da41](https://github.com/OptimalBits/bull/commit/f68da4176658b9935cf4b63b218130008619b25b))\n\n## [3.22.4](https://github.com/OptimalBits/bull/compare/v3.22.3...v3.22.4) (2021-04-27)\n\n\n### Bug Fixes\n\n* also populate retriedOn when loading from id with excludeData ([0964b39](https://github.com/OptimalBits/bull/commit/0964b390d9190510f0d016f4eb2a3f1bc6cdb4e5))\n\n## [3.22.3](https://github.com/OptimalBits/bull/compare/v3.22.2...v3.22.3) (2021-04-23)\n\n\n### Bug Fixes\n\n* **delayed:** re-schedule updateDelay in case of error fixes [#2015](https://github.com/OptimalBits/bull/issues/2015) ([16bbfad](https://github.com/OptimalBits/bull/commit/16bbfadb270bc6c7d6df9cf5ab30b7f66028b2b3))\n\n## [3.22.2](https://github.com/OptimalBits/bull/compare/v3.22.1...v3.22.2) (2021-04-23)\n\n\n### Bug Fixes\n\n* **obliterate:** obliterate many jobs fixes [#2016](https://github.com/OptimalBits/bull/issues/2016) ([7a923b4](https://github.com/OptimalBits/bull/commit/7a923b468d5299bbdfe06d1ee7447fd810e2779b))\n\n\n## v.3.22.1\n\n- fix(obliterate): remove repeatable jobs fixes #2012\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.22.0...v3.22.1)\n\n## v.3.22.0\n\n- feat: do not rely on comma to encode jobid in progress fixes #2003\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.21.1...v3.22.0)\n\n\n## v.3.21.1\n\n- fix: safer implementation of obliterate.\nNote: If you want to use the new method \"obliterate\" it is highly recommended\nthat you upgrade to this version, since previous version is not safe to use\nif using the colon ':' character in your queue names.\n\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.21.0...v3.21.1)\n\n## v.3.21.0\n\n- feat: add a method to \"obliterate\" a queue\n- fix: upgrade lodash fixes #1996\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.20.1...v3.21.0)\n\n## v.3.20.1\n\n- fix(queue): possible infinite loop when disconnect fixes #1746\n- fix(clean-priority): remove job from priority set on clean (#1405)\n- fix(sandbox): job update (#1957)\n- fix: use async version of process.send for progress and log (#1948)\n- fix: promote jobs to the right \"list\" when paused\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.20.0...v3.20.1)\n\n## v.3.20.0\n\n- feat(job): implement Job#retriedOn (#1868)\n- fix: job default opts fixes #1904\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.19.1...v3.20.0)\n\n## v.3.19.1\n\n- fix(getters): properly zip hmget result\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.19.0...v3.19.1)\n\n## v.3.19.0\n\n- feat: add option to exclude data on getters (#1910)\n- fix: lock ioredis to 4.18.0 to avoid breakage with newer 4.19.0+.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.18.1...v3.19.0)\n\n## v.3.18.1\n\n- fix(repeat): remove last delayed job.\n- fix(rate-limiter): increment jobCounter only when a job is processed. fixes #1875.\n- fix(sandbox): clear dangling timeout.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.18.0...v3.18.1)\n\n## v.3.18.0\n\n- feat: make pause forward compatible with bullmq (#1818) (@manast)\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.17.0...v3.18.0)\n\n## v.3.17.0\n\n- feat: better rate limiter (#1816) (@manast)\n- feat(sandbox): kill child workers gracefully (#1802) (@GabrielCastro)\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.16.0...v3.17.0)\n\n## v.3.16.0\n\n- feat(rate-limiter): add grouping support.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.15.0...v3.16.0)\n\n## v.3.15.0\n\n- feat: add isPaused fixes #1274\n- fix: emit waiting event when adding a priority job (#1134)\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.14.0...v3.15.0)\n\n## v.3.14.0\n\n- feat(queue): add removeJobs function\n- fix: clamp negative job delay values to 0 to prevent thrashing\n- fix: use DEFAULT_JOB_NAME (#1585)\n- fix: remove the lazy client error handler on close (#1605)\n- fix: prevent exceeding the maximum stack call size when emptying large queues (#1660)\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.13.0...v3.14.0)\n\n## v.3.13.0\n\n- feat: add \"preventParsingData\" job option to prevent data parsing\n- fix: queue.clean clean job logs as well\n- fix: whenCurrentJobsFinished should wait for all jobs\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.12.1...v3.13.0)\n\n## v.3.12.1\n\n- fix: catch errors parsing invalid progress data\n- fix(pause): don't initialize bclient if not waiting for jobs to finish\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.12.0...v3.12.1)\n\n## v.3.12.0\n\n- feat: support async custom backoffs.\n- feat(sandbox): emulate job.progress function.\n\n- fix: queue.pause(true, true) doesn't pause queue.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.11.0...v3.12.0)\n\n## v.3.11.0\n\n- feat(queue): basic support for bulk adding jobs.\n- feat(job): save data on job instance when updated.\n\n- fix(queue): whenCurrentJobsFinished shouldn't initialize bclient. Fixes #1346.\n- fix(queue): unhandled promise warning in updateDelayTimer.\n- fix(sandbox): if the child process is killed, remove it from the pool.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.10.0...v3.11.0)\n\n## v.3.10.0\n\n- fix: remove logs automtically when using autoremove fixes #1330\n- feat: add support for keeping a specified number of jobs when using auto-remove.\n- feat: add support for node 12\n- fix: fix check for supported file types in sandboxed processors #1311\n- ci: drop support for node 6\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.9.1...v3.10.0)\n\n## v.3.9.1\n\n- fix: add log to job wrapper\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.9.0...v3.9.1)\n\n## v.3.9.0\n\n- feat: add job.log #1165\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.8.1...v3.9.0)\n\n## v.3.8.1\n\n- fix: wait for ready in cleanJobsInSet fixes #1298\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.8.0...v3.8.1)\n\n## v.3.8.0\n\n- fix: improve delay logic fixing #1226, #1222\n- feat: store finishedOn on the job instance\n- fix: return every in getRepeatableJobs #1284\n- fix: remove broken child processes #1098\n- feat: update sandbox exit handler to log signals #1252\n- fix: Ignore unknown command client error #1240\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.7.0...v3.8.0)\n\n## v.3.7.0\n\n- perf: improve update delay set logic.\n- feat: consider priority when promoting a job #1205.\n- fix: better delay for rate limited jobs.\n- feat: update port selection mechanism for child node process inspect flag.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.6.0...v3.7.0)\n\n## v.3.6.0\n\n- feat: add function to remove repeatable jobs by key.\n- fix: properly remove sandbox events. Fixes #1179.\n- fix: make progress functions in sandbox consistently return a promise.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.5.3...v3.6.0)\n\n## v.3.5.3\n\n- chore: upgrade ioredis to ^4.5.1.\n- fix: fix #1044 support for typescript processors.\n- chore: remove bluebird.\n- chore: use lockKey method instead of private property.\n- fix(queue): convert delay setting to number.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.5.2...v3.5.3)\n\n## v.3.5.2\n\n- chore(queue): remove bluebird config from the codebase.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.5.1...v3.5.2)\n\n## v.3.5.1\n\n- chore(yarn): updated yarn.lock\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.5.0...v3.5.1)\n\n## v.3.5.0\n\n- fix(delayed): pause delayed jobs #1087\n- fix(lua): correct numJobs fetch in moveToActive\n- perf(moveToActive): used local var for rate limiter\n- perf(queue): replace bluebird by native promises where possible\n- chore(queue): fix typo in forcedReconnection variable\n- feat(child-processes): catch sub process crashes\n- fix(jobs): reset 'failedReason', 'finishedOn' and 'processedOn' fields on job retry\n- fix(queue): fix Warning: cancellation is disabled\n- fix(queue): remove the correct listener in isRedisReady\n- feat(jobs): allow cancelling of retries when using custom backoff strategy\n- feat(rate-limiter): add discard config for rate-limiter\n- feat(jobs): make job progress accepts variant types\n- fix(repeatable): Fixed wrong repeatable count updates\n- fix(jobs): fix copy paste mistake for stacktrace in job toData\n- feat(child-processes): Propagate stack traces\n- feat(repeatable): add ability for cron repeatable job with startDate\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.4.8...v3.5.0)\n\n## v.3.4.8\n\n- emit waiting event when waking up sleep jobs fixing #792\n- throw error if missing processor file fixing #954\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.4.7...v3.4.8)\n\n## v.3.4.7\n\n- Fixes to deal with removing correctly in priority queues #984\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.4.6...v3.4.7)\n\n## v.3.4.6\n\n- Reverted use of arrow function that was incompatible with older versions of node.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.4.5...v3.4.6)\n\n## v.3.4.5\n\n- Fixed Unhandled promise rejections #1012.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.4.4...v3.4.5)\n\n## v.3.4.4\n\n- Partially fixed #845. When call queue.close() bull throws Error: Connection is closed.\n- Fixed #998. Check for existence of rate limiter options.\n- Fixed #1003. Fixed fixed repeatable jobs duplication using every.\n- Feature/provide error to custom backoff strategy.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.4.3...v3.4.4)\n\n## v.3.4.3\n\n- Fixed #994 queue.getJobs() race condition.\n- Fixed #966 Race conditions reviving repeatable jobs.\n- Fixed getters: Update types array to include paused.\n- Fixed #958 job.finished slowdown.\n- Fixed #949 TypeError: job.queue.client.isFinished is not a function.\n- Fixed #870 TypeError when retrying jobs.\n- Fixed #942 Support for milliseconds intervals in repeatable jobs.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.4.2...v3.4.3)\n\n## v.3.4.2\n\n- Fixed #903 Globally paused queue cannot receive job (or not shown in Arena untill queue is globally resumed).\n- Workaround for #911 Seperate process worker fails to launch when Node is started with --inspect flag\n- added missing retain on reused child job #908.\n- added more tests for child jobs.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.4.1...v3.4.2)\n\n## v.3.4.1\n\n- Better check for closing in moveUnlockedJobsToWait, possibly fixing #806.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.4.0...v3.4.1)\n\n## v.3.4.0\n\n- Added support for prioritized delayed jobs.\n- Added ability to process all named jobs from one process function.\n- Fixed #893, warning 'a promise was rejected with a non-error' for external queues in case of an error.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.3.10...v3.4.0)\n\n## v.3.3.10\n\n- Faster next job fetch #868\n- Added global default options for jobs. Fixes #706.\n- Added a limit for repeatable jobs. #854.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.3.9...v3.3.10)\n\n## v.3.3.9\n\n- Support custom backoff strategies.\n- Fixed #786. Handling of ES6 default export.\n- Fixed #782. Better handling of \"isReady\".\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.3.8...v3.3.9)\n\n## v.3.3.8\n\n- Fixed #812. External process doesn't terminate on `queue.close()`.\n- Fixed #830. Named Process Sent to Wrong Processor.\n- Fixed #572. Do not close external connections.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.3.7...v3.3.8)\n\n## v.3.3.7\n\n- Fixed #807.\n- Adding ability to limit by stacktrace. #798.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.3.6...v3.3.7)\n\n## v.3.3.6\n\n- Fixed #766, #781, #716.\n- Correctly accept DB in redis connection string.\n- Fixed global progress event.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.3.5...v3.3.6)\n\n## v.3.3.5\n\n- Fixed #764, #762, #759.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.3.4...v3.3.5)\n\n## v.3.3.4\n\n- Fixed #748.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.3.3...v3.3.4)\n\n## v.3.3.3\n\n- Re-fixed #739.\n- Possibly fixed for #747.\n- Fixed removeRepeatable (missing file)\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.3.2...v3.3.3)\n\n## v.3.3.2\n\n- Fixed #721. SyntaxError: Unexpected token u in JSON at position 0.\n- Fixed #739. childs are not added to the retained set.\n- Fixed #734. fixed Promise warnings.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.3.1...v3.3.2)\n\n## v.3.3.1\n\n- Fixed #714\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.3.0...v3.3.1)\n\n## v.3.3.0\n\n- Added a method `Queue##removeRepeatable` to remove repeatable jobs.\n- Now also emits drained as a global event.\n- Fixed #518, #624\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.2.0...v3.3.0)\n\n## v.3.2.0\n\n- Added support for running jobs in child processes #488\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.1.0...v3.2.0)\n\n## v.3.1.0\n\n- Added rate limiter support.\n- Added method to update jobs data.\n- Implemented stalled as global event.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.0.0...v3.1.0)\n\n## v.3.0.0\n\n- No changes.\n\n## v.3.0.0-rc.10\n\n- Fixed #666.\n- Small improvements in the repeat code.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.0.0-rc.9...v3.0.0-rc.10)\n\n## v.3.0.0-rc.9\n\n- Fixed #672.\n- Fixed #670\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.0.0-rc.8...v3.0.0-rc.9)\n\n## v.3.0.0-rc.8\n\n- Enhanced job fetching #651 (faster and more reliable).\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.0.0-rc.7...v3.0.0-rc.8)\n\n## v.3.0.0-rc.7\n\n- Fixed #659\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.0.0-rc.6...v3.0.0-rc.7)\n\n## v.3.0.0-rc.6\n\n- Fixed #645.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.0.0-rc.5...v3.0.0-rc.6)\n\n## v.3.0.0-rc.5\n\n- Improved performance, specially when having many concurrent workers.\n- Fixed #609 using zsets for storing repeatable jobs.\n- Fixed #608 Event chaining no longer works.\n- Improved getters.\n- Fixed #601 Add multiple repeatable jobs with the same cron pattern.\n\n[Changes](https://github.com/OptimalBits/bull/compare/3.0.0-rc.4...v3.0.0-rc.5)\n\n## v.3.0.0-rc.4\n\n- Added support for naming workers in redis connections #530.\n- Lazy instantiation of redis clients. Fixes #526.\n- job.finished captures result from queue process. #588.\n- Caches LUA scripts to avoid reading files in every queue instantiation. #591.\n- Emit 'drain' event when queue is empty. #596.\n- store finished and processed timestamps. #594, #606.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.0.0-rc.3...3.0.0-rc.4)\n\n## v.3.0.0-rc.3\n\n- Fixed #579.\n- Lazy subscription to events for better performance.\n- Corrected calculation of next repeat job. #563.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.0.0-rc.2...v3.0.0-rc.3)\n\n## v.3.0.0-rc.2\n\n- Improved performance of moveToActive #550.\n- Fixed issue with cancelable promise #546.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.0.0-rc.1...v3.0.0-rc.2)\n\n## v.3.0.0-rc.1\n\n- Improved error and lock handling for failed jobs #499, #539.\n- Corrected instantiation from urls #538.\n- Return jobs in proper order in jobs getters.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.0.0-alpha.4...v3.0.0-rc.1)\n\n## v.3.0.0-alpha.4\n\n- Implemented repeatable jobs. #252.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.0.0-alpha.3...v3.0.0-alpha.4)\n\n## v.3.0.0-alpha.3\n\n- Simplified global events #501.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.0.0-alpha.2...v3.0.0-alpha.3)\n\n## v.3.0.0-alpha.2\n\n- Eliminated possible memory leak #503\n\n[Changes](https://github.com/OptimalBits/bull/compare/v3.0.0-alpha.1...v3.0.0-alpha.2)\n\n## v.3.0.0-alpha.1\n\n- improved job fetch mechanism. #480.\n- job.jobId changed to job.id.\n- refactored error messages into separate error module.\n- refactored lua scripts into separate files, and preloaded.\n- many atomizations and clean ups.\n- completed and failed job states are now represented in ZSETs. #190.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v2.2.6...v3.0.0-alpha.1)\n\n## v.2.2.6\n\n- Persisted failedReason when storing job data.\n- added queue##isReady()\n\n[Changes](https://github.com/OptimalBits/bull/compare/v2.2.5...v2.2.6)\n\n## v.2.2.5\n\n- Fixed so that redis key prefix works properly.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v2.2.4...v2.2.5)\n\n## v.2.2.4\n\n- Allow reusing certain redis connections.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v2.2.3...v2.2.4)\n\n## v.2.2.3\n\n- Added getJobCounts.\n- Fixed global events #394.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v2.2.2...v2.2.3)\n\n## v.2.2.2\n\n- Fixed redis script cache gets bloated after update to bull 2.0 #426\n\n[Changes](https://github.com/OptimalBits/bull/compare/v2.2.1...v2.2.2)\n\n## v.2.2.1\n\n- Re-added createClient option that was removed by mistake.\n- Corrected getJobCountByTypes, fixes #419 and #401\n\n[Changes](https://github.com/OptimalBits/bull/compare/v2.2.0...v2.2.1)\n\n## v.2.2.0\n\n- Much improved priority queues, simpler, faster and more reliable.\n- Fixed issue where lua scripts where leaking memory.\n- Improvements in local pause, fixing #446 and #447.\n- Fix to increase delay time over 24 days #244\n\n[Changes](https://github.com/OptimalBits/bull/compare/v2.1.2...v2.2.0)\n\n## v.2.1.2\n\n- Fixed Error renewing lock LockError: Exceeded 0 attempts to lock the resource #437\n- Fixed Unable to renew nonexisting lock on job fail #441\n\n[Changes](https://github.com/OptimalBits/bull/compare/v2.1.1...v2.1.2)\n\n## v.2.1.1\n\n- Catch errors produced in timers. Related to #441\n\n[Changes](https://github.com/OptimalBits/bull/compare/v2.1.0...v2.1.1)\n\n## v.2.1.0\n\n- Fixed #397, Error: Unable to renew nonexisting lock\n- Fixed #402, Job.prototype.finished contains error in promise\n- Fixed #371, \"Unexpected token u in JSON at position 0\" while processing job\n- New feature #363, \"method to permanently fail a job\"\n- Fix job.progress() to return the correct progress\n\n[Changes](https://github.com/OptimalBits/bull/compare/v2.0.0...v2.1.0)\n\n## v.2.0.0\n\n- Changed redis module to ioredis fixing many issues along the way, see changes.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v1.1.3...v2.0.0)\n\n## v.1.1.3\n\n- fixed \"Broken constructor pattern from recent commit\" #384\n- fixed \"Queue.prototype.getWaiting() returns empty list if Queue is paused\" #342\n\n[Changes](https://github.com/OptimalBits/bull/compare/v1.1.2...v1.1.3)\n\n## v1.1.2\n\n- regained backwards compatibility in events by using disturbed 1.0.6\n\n[Changes](https://github.com/OptimalBits/bull/compare/v1.1.1...v1.1.2)\n\n## v1.1.1\n\n- Returned this in queue##on and queue##once for backwards compatibility.\n- [Fixes PriorityQueue Events and Local Worker Pause/Resume](https://github.com/OptimalBits/bull/pull/341)\n\n[Changes](https://github.com/OptimalBits/bull/compare/v1.1.0...v1.1.1)\n\n## v1.1.0\n\n- Fixed [job corruption issue](https://github.com/OptimalBits/bull/pull/359)\n- The job id can be [overridden](https://github.com/OptimalBits/bull/pull/335) to implement job throttling behavior\n- Added [`removeOnComplete` job option](https://github.com/OptimalBits/bull/pull/361)\n- [More robust job retry](https://github.com/OptimalBits/bull/pull/318)\n- Events are [now broadcast to all workers](https://github.com/OptimalBits/bull/commit/d55ad1c8f44f86be9b4e9f4fa9a3fc8a16c6e02d)\n\n[Changes](https://github.com/OptimalBits/bull/compare/v1.0.0...v1.1.0)\n\n## v1.0.0\n\n- improvements in clean (fixes and performance).\n\n[Changes](https://github.com/OptimalBits/bull/compare/v1.0.0-rc4...v1.0.0)\n\n## v1.0.0-rc4\n\n- fixed lock renew logic.\n- atomized code for getting stalled jobs.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v1.0.0-rc3...v1.0.0-rc4)\n\n## v1.0.0-rc3\n\n- smaller fixes.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v1.0.0-rc2...v1.0.0-rc3)\n\n## v1.0.0-rc2\n\n- Improved locking when removing and processing stalled jobs.\n- Fixed #302 EVALSHA failure.\n- Fixed #295 error with redis 3.2.\n- Correctly allows the specification of the db\n- Honor start/end range for complete/failed jobs.\n- Fixed #277 Memory Leaks With Large Queue.\n- Support for custom key prefix for redis keys.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v1.0.0-rc1...v1.0.0-rc2)\n\n## v1.0.0-rc1\n\n- Removed all potential dangerous hazards by atomizing many operations using\n cached LUA scripts.\n- Improved performance around 400% compared to previous version.\n- Better pause/resume (#266), and added pause for local workers.\n- Fixed #272, #271, #261, #253, #240, #239\n\n[Changes](https://github.com/OptimalBits/bull/compare/v0.7.2...v1.0.0-rc1)\n\n## v0.7.2\n\n- Added local pause/resume functionality\n- fixed memory leaks present in the run promise chain.\n- fixed \"Illegal access to a strict mode caller function\".\n\n[Changes](https://github.com/OptimalBits/bull/compare/v0.7.1...v0.7.2)\n\n## v0.7.1\n\n- fixed storing of stacktraces\n\n[Changes](https://github.com/OptimalBits/bull/compare/v0.7.0...v0.7.1)\n\n## v0.7.0\n\n- store the return value from the job handlers.\n- store stacktraces.\n- improvements in delayed jobs.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v0.6.0...v0.7.0)\n\n## v0.4.0\n\n- added a Queue##clean method\n\n[Changes](https://github.com/OptimalBits/bull/compare/v0.3.0...v0.4.0)\n\n## v0.3.0\n\n- added support for custom clients.\n- added test support for node 0.12.\n- timeout improvements.\n- unit test improvements.\n- added timeout to queue pop blocking call.\n- removed when dependency.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v0.2.7...v0.3.0)\n\n## v0.2.7\n\n[Changes](https://github.com/OptimalBits/bull/compare/v0.2.6...v0.2.7)\n\n## v0.2.6\n\n- [Fix] #103 When a queue start it do not process delayed job.\n [Changes](https://github.com/OptimalBits/bull/compare/v0.2.5...v0.2.6)\n\n## v0.2.5\n\n- [upgrade] Upgraded node redis to version 0.12.x\n- [improvement] eslinted all code.\n- [fix] added missing token when calling takeLock on jobs.\n\n[Changes](https://github.com/OptimalBits/bull/compare/v0.2.4...v0.2.5)\n\n## v0.2.4\n\n[Changes](https://github.com/OptimalBits/bull/compare/v0.2.3...v0.2.4)\n\n## v0.2.3\n\n[Changes](https://github.com/OptimalBits/bull/compare/v0.1.9...v0.2.3)\n\n## v0.1.9\n\n- [Improvement] Faster job removal. (manast)\n\n## v0.1.8\n\n- [Improvement] Better promisification of redis methods. (manast)\n\n## v0.1.7\n\n- [Feature] Added a convenience method for getting a job. (loginx)\n- [Fix] Only set a redis db from options if defined. (jboga)\n- [Fix] Fixed issue #52. (manast)\n\n## v0.1.6\n\n- [Fix] Improved and corrected job's getters.\n- [Fix] Automatically restart queues in the event of redis disconnections.\n- [Feature] Added support for adding jobs in a LIFO fashion.\n" }, { "path": "CODE_OF_CONDUCT.md", "content": "# Contributor Covenant Code of Conduct\n\n## Our Pledge\n\nIn the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.\n\n## Our Standards\n\nExamples of behavior that contributes to creating a positive environment include:\n\n* Using welcoming and inclusive language\n* Being respectful of differing viewpoints and experiences\n* Gracefully accepting constructive criticism\n* Focusing on what is best for the community\n* Showing empathy towards other community members\n\nExamples of unacceptable behavior by participants include:\n\n* The use of sexualized language or imagery and unwelcome sexual attention or advances\n* Trolling, insulting/derogatory comments, and personal or political attacks\n* Public or private harassment\n* Publishing others' private information, such as a physical or electronic address, without explicit permission\n* Other conduct which could reasonably be considered inappropriate in a professional setting\n\n## Our Responsibilities\n\nProject maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.\n\nProject maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.\n\n## Scope\n\nThis Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.\n\n## Enforcement\n\nInstances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at manuel@optimalbits.com. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.\n\nProject maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.\n\n## Attribution\n\nThis Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version]\n\n[homepage]: http://contributor-covenant.org\n[version]: http://contributor-covenant.org/version/1/4/\n" }, { "path": "CONTRIBUTING.md", "content": "Release process\n---------------\n\nFirst, update `CHANGELOG.md` with the release number about to be released.\n\n npm outdated --depth 0 # See if you can upgrade any dependencies\n npm version [major|minor|patch] # Update package.json\n npm publish # Tag repo and publish npm package\n" }, { "path": "LICENSE.md", "content": "\nLicense\n-------\n\n(The MIT License)\n\nCopyright © 2013-2018 Manuel Astudillo \n\nPermission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the 'Software'), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" }, { "path": "MIGRATION.md", "content": "# Migration from 2.x to 3.0.0\n\nAlthough version 3.0 is almost backwards compatible with 2.x, there are some important changes that needs\nto be taking in consideration before upgrading to 3.0.\n\n# Complete and failed sets.\n\nIn 3.x, the jobs that are completed and failed end in two ZSETS, instead of a standard SET.\nThis gives the possibility of retrieving a subset of the jobs in a high performant way, which\nis useful for graphical tools and scripts. However an old queue will not be compatible with 3.x.\nYou will need to either delete the complete and failed keys, or create a new queue.\n\n# Data structure changes\n\njob.jobId to job.id\n\ntoJSON ->\n job.data\n job.opts\n\n# Queue instantiation options\n\nSanitized and cleaned all the options. Check the [Reference](./REFERENCE.md) to see the new structure.\n\n\n# Events\n\nAll events are now published atomically in the scripts where they are relevant, this increases efficiency and\nreduces chances for hazards.\n\n'ready' event has been removed, you can use ```Queue##isReady()``` instead if you want to know when the queue\nhas been initialized. Normally you will never need to wait for readyness since this is taken care internally\nby the queue methods that require the queue to be ready.\n\nEvents arguments are now the same for local and global events. This affects events such as completed and failed,\nwhere in 2.x the first argument was a job instance for local jobs. Now both local and global events pass\njobId as first argument to the event handler. If the job instance is needed it can be easily retrieved with\n```Job.fromId()```.\n\n" }, { "path": "PATTERNS.md", "content": "\nPatterns\n========\n\nHere are a few examples of useful patterns that are often implemented with Bull:\n\n- [Message Queue](#message-queue)\n- [Returning Job Completions](#returning-job-completions)\n- [Reusing Redis Connections](#reusing-redis-connections)\n- [Redis Cluster](#redis-cluster)\n- [Debugging](#debugging)\n- [Custom backoff strategy](#custom-backoff-strategy)\n- [Manually fetching jobs](#manually-fetching-jobs)\n\nIf you have any other common patterns you want to add, pull request them!\n\n\nMessage Queue\n-------------\n\nBull can also be used for persistent message queues. This is a quite useful\nfeature in some use cases. For example, you can have two servers that need to\ncommunicate with each other. By using a queue the servers do not need to be online at the same time, so this creates a very robust communication channel. You can treat `add` as *send* and `process` as *receive*:\n\nServer A:\n\n```js\nconst Queue = require('bull');\n\nconst sendQueue = new Queue('Server B');\nconst receiveQueue = new Queue('Server A');\n\nreceiveQueue.process(function (job, done) {\n console.log('Received message', job.data.msg);\n done();\n});\n\nsendQueue.add({ msg: 'Hello' });\n```\n\nServer B:\n\n```js\nconst Queue = require('bull');\n\nconst sendQueue = new Queue('Server A');\nconst receiveQueue = new Queue('Server B');\n\nreceiveQueue.process(function (job, done) {\n console.log('Received message', job.data.msg);\n done();\n});\n\nsendQueue.add({ msg: 'World' });\n```\n\n\nReturning Job Completions\n-------------------------\n\nA common pattern is where you have a cluster of queue processors that just process jobs as fast as they can, and some other services that need to take the result of this processors and do something with it, maybe storing results in a database.\n\nThe most robust and scalable way to accomplish this is by combining the standard job queue with the message queue pattern: a service sends jobs to the cluster just by opening a job queue and adding jobs to it, and the cluster will start processing as fast as it can. Everytime a job gets completed in the cluster a message is sent to a results message queue with the result data, and this queue is listened by some other service that stores the results in a database.\n\n\nReusing Redis Connections\n-------------------------\n\nA standard queue requires **3 connections** to the Redis server. In some situations you might want to re-use connections—for example on Heroku where the connection count is restricted. You can do this with the `createClient` option in the `Queue` constructor.\n\nNotes:\n- bclient connections [cannot be re-used](https://github.com/OptimalBits/bull/issues/880), so you should return a new connection each time this is called.\n- client and subscriber connections can be shared and will not be closed when the queue is closed. When you are shutting down the process, first close the queues, then the shared connections (if they are shared).\n- if you are not sharing connections but still using `createClient` to do some custom connection logic, you may still need to keep a list of all the connections you created so you can manually close them later when the queue shuts down, if you need a graceful shutdown for your process\n- do not set a `keyPrefix` on the connection you create, use bull's built-in prefix feature if you need a key prefix\n\n```js\nconst { REDIS_URL } = process.env;\n\nconst Redis = require('ioredis');\nlet client;\nlet subscriber;\n\nconst opts = {\n // redisOpts here will contain at least a property of connectionName which will identify the queue based on its name\n createClient: function (type, redisOpts) {\n switch (type) {\n case 'client':\n if (!client) {\n client = new Redis(REDIS_URL, redisOpts);\n }\n return client;\n case 'subscriber':\n if (!subscriber) {\n subscriber = new Redis(REDIS_URL, redisOpts);\n }\n return subscriber;\n case 'bclient':\n return new Redis(REDIS_URL, redisOpts);\n default:\n throw new Error('Unexpected connection type: ' + type);\n }\n }\n}\n\nconst queueFoo = new Queue('foobar', opts);\nconst queueQux = new Queue('quxbaz', opts);\n```\n\nRedis cluster\n-------------\n\nBull internals require atomic operations that span different keys. This behavior breaks Redis's\nrules for cluster configurations. However, it is still possible to use a cluster environment\nby using the proper bull prefix option as a cluster \"hash tag\". Hash tags are used to guarantee\nthat certain keys are placed in the same hash slot, read more about hash tags in the [redis cluster\ntutorial](https://redis.io/topics/cluster-tutorial). A hash tag is defined with brackets. I.e. a key that has a substring inside brackets will use that\nsubstring to determine in which hash slot the key will be placed.\n\nIn summary, to make bull compatible with Redis cluster, use a queue prefix inside brackets.\nFor example:\n\n```js\nconst queue = new Queue('cluster', {\n prefix: '{myprefix}'\n});\n```\n\nIf you use several queues in the same cluster, you should use different prefixes so that the\nqueues are evenly placed in the cluster nodes.\n\nDebugging\n---------\n\nTo see debug statements set or add `bull` to the `NODE_DEBUG` environment variable:\n\n```bash\nexport NODE_DEBUG=bull\n```\n\n```bash\nNODE_DEBUG=bull node ./your-script.js\n```\n\nCustom backoff strategy\n-----------------------\n\nWhen the builtin backoff strategies on retries are not sufficient, a custom strategy can be defined. Custom backoff strategies are defined by a function on the queue. The number of attempts already made to process the job is passed to this function as the first parameter, and the error that the job failed with as the second parameter.\nThe function returns either the time to delay the retry with, 0 to retry immediately or -1 to fail the job immediately.\n\n```js\nconst Queue = require('bull');\n\nconst myQueue = new Queue('Server B', {\n settings: {\n backoffStrategies: {\n jitter: function (attemptsMade, err) {\n return 5000 + Math.random() * 500;\n }\n }\n }\n});\n```\n\nThe new backoff strategy can then be specified on the job, using the name defined above:\n\n```js\nmyQueue.add({foo: 'bar'}, {\n attempts: 3,\n backoff: {\n type: 'jitter'\n }\n});\n```\n\nYou may specify options for your strategy:\n```js\nconst Queue = require('bull');\n\nconst myQueue = new Queue('Server B', {\n settings: {\n backoffStrategies: {\n // truncated binary exponential backoff\n binaryExponential: function (attemptsMade, err, options) {\n // Options can be undefined, you need to handle it by yourself\n if (!options) {\n options = {}\n }\n const delay = options.delay || 1000;\n const truncate = options.truncate || 1000;\n console.error({ attemptsMade, err, options });\n return Math.round(Math.random() * (Math.pow(2, Math.max(attemptsMade, truncate)) - 1) * delay)\n }\n }\n }\n});\n\nmyQueue.add({ foo: 'bar' }, {\n attempts: 10,\n backoff: {\n type: 'binaryExponential',\n options: {\n delay: 500,\n truncate: 5\n }\n }\n});\n\n```\n\nYou may base your backoff strategy on the error that the job throws:\n```js\nconst Queue = require('bull');\n\nfunction MySpecificError() {};\n\nconst myQueue = new Queue('Server C', {\n settings: {\n backoffStrategies: {\n foo: function (attemptsMade, err) {\n if (err instanceof MySpecificError) {\n return 10000;\n }\n return 1000;\n }\n }\n }\n});\n\nmyQueue.process(function (job, done) {\n if (job.data.msg === 'Specific Error') {\n throw new MySpecificError();\n } else {\n throw new Error();\n }\n});\n\nmyQueue.add({ msg: 'Hello' }, {\n attempts: 3,\n backoff: {\n type: 'foo'\n }\n});\n\nmyQueue.add({ msg: 'Specific Error' }, {\n attempts: 3,\n backoff: {\n type: 'foo'\n }\n});\n```\n\nManually fetching jobs\n----------------------------------\n\nIf you want the actual job processing to be done in a seperate repo/service than where `bull` is running, this pattern may be for you.\n\nManually transitioning states for jobs can be done with a few simple methods.\n\n1. Adding a job to the 'waiting' queue. Grab the queue and call `add`.\n\n```typescript\nimport Queue from 'bull';\n\nconst queue = new Queue({\n limiter: {\n max: 5,\n duration: 5000,\n bounceBack: true // important\n },\n ...queueOptions\n});\nqueue.add({ random_attr: 'random_value' });\n```\n\n2. Pulling a job from 'waiting' and moving it to 'active'.\n\n```typescript\nconst job: Job = await queue.getNextJob();\n```\n\n3. Move the job to the 'failed' queue if something goes wrong.\n\n```typescript\nconst (nextJobData, nextJobId) = await job.moveToFailed(\n {\n message: 'Call to external service failed!',\n },\n true,\n);\n```\n\n3. Move the job to the 'completed' queue.\n\n```typescript\nconst (nextJobData, nextJobId) = await job.moveToCompleted('succeeded', true);\n```\n\n4. Return the next job if one is returned.\n\n```typescript\nif (nextJobdata) {\n return Job.fromJSON(queue, nextJobData, nextJobId);\n}\n```\n\n**Note**\n\nBy default the lock duration for a job that has been returned by ```getNextJob``` or ```moveToCompleted``` is 30 seconds, if it takes more time than that the job will be automatically\nmarked as stalled and depending on the max stalled options be moved back to the wait state or marked as failed. In order to avoid this you must use [```job.extendLock(duration)```](REFERENCE.md#jobextendlock) in order to give you some more time before the lock expires. The recommended is to extend the lock when half the lock time has passsed.\n\n" }, { "path": "README.md", "content": "\n
\n
\n \n
\n
\n

\n The fastest, most reliable, Redis-based queue for Node.
\n Carefully written for rock solid stability and atomicity.\n

\n
\n

\n Sponsors ¡\n Features ¡\n UIs ¡\n Install ¡\n Quick Guide ¡\n Documentation\n

\n

Check the new Guide!

\n
\n

\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n

\n
\n\n### 🚀 Sponsors 🚀\n\n\n \n \n \n \n
\n \n \"Dragonfly\"\n \n \n Dragonfly is a new Redisâ„ĸ drop-in replacement that is fully compatible with BullMQ and brings some important advantages over Redisâ„ĸ such as massive\n better performance by utilizing all CPU cores available and faster and more memory efficient data structures. Read more here on how to use it with BullMQ.\n
\n\n### đŸ“ģ News and updates\n\nBull is currently in maintenance mode, we are only fixing bugs. For new features check [BullMQ](https://github.com/taskforcesh/bullmq), a modern rewritten\nimplementation in Typescript. You are still very welcome to use Bull if it suits your needs, which is a safe, battle tested library.\n\nFollow me on [Twitter](http://twitter.com/manast) for other important news and updates.\n\n### 🛠 Tutorials\n\nYou can find tutorials and news in this blog: https://blog.taskforce.sh/\n\n---\n\n### Used by\n\nBull is popular among large and small organizations, like the following ones:\n\n\n \n \n \n \n \n \n\n \n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n\n---\n\n---\n\n### Official FrontEnd\n\n[\"Taskforce.sh,](https://taskforce.sh)\n\nSupercharge your queues with a professional front end:\n- Get a complete overview of all your queues.\n- Inspect jobs, search, retry, or promote delayed jobs.\n- Metrics and statistics.\n- and many more features.\n\nSign up at [Taskforce.sh](https://taskforce.sh)\n\n---\n\n### Bull Features\n\n- [x] Minimal CPU usage due to a polling-free design.\n- [x] Robust design based on Redis.\n- [x] Delayed jobs.\n- [x] Schedule and repeat jobs according to a cron specification.\n- [x] Rate limiter for jobs.\n- [x] Retries.\n- [x] Priority.\n- [x] Concurrency.\n- [x] Pause/resume—globally or locally.\n- [x] Multiple job types per queue.\n- [x] Threaded (sandboxed) processing functions.\n- [x] Automatic recovery from process crashes.\n\nAnd coming up on the roadmap...\n\n- [ ] Job completion acknowledgement (you can use the message queue [pattern](https://github.com/OptimalBits/bull/blob/develop/PATTERNS.md#returning-job-completions) in the meantime).\n- [ ] Parent-child jobs relationships.\n\n---\n\n### UIs\n\nThere are a few third-party UIs that you can use for monitoring:\n\n**BullMQ**\n\n- [Taskforce](https://taskforce.sh)\n\n**Bull v3**\n\n- [Taskforce](https://taskforce.sh)\n- [bull-board](https://github.com/vcapretz/bull-board)\n- [bull-repl](https://github.com/darky/bull-repl)\n- [bull-monitor](https://github.com/s-r-x/bull-monitor)\n- [Monitoro](https://github.com/AbhilashJN/monitoro)\n\n**Bull <= v2**\n\n- [Matador](https://github.com/ShaneK/Matador)\n- [react-bull](https://github.com/kfatehi/react-bull)\n- [Toureiro](https://github.com/Epharmix/Toureiro)\n\n---\n\n### Monitoring & Alerting\n\n- With Prometheus [Bull Queue Exporter](https://github.com/UpHabit/bull_exporter)\n\n---\n\n### Feature Comparison\n\nSince there are a few job queue solutions, here is a table comparing them:\n\n| Feature | [BullMQ-Pro](https://bullmq.io/#bullmq-pro) | [BullMQ](https://bullmq.io) | Bull | Kue | Bee | Agenda |\n| :------------------------ | :-------------: | :-------------: | :-------------: | :---: | -------- | ------ |\n| Backend | redis | redis | redis | redis | redis | mongo |\n| Observables | ✓ | | | | | |\n| Group Rate Limit | ✓ | | | | | |\n| Group Support | ✓ | | | | | |\n| Batches Support | ✓ | | | | | |\n| Parent/Child Dependencies | ✓ | ✓ | | | | |\n| Priorities | ✓ | ✓ | ✓ | ✓ | | ✓ |\n| Concurrency | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ |\n| Delayed jobs | ✓ | ✓ | ✓ | ✓ | | ✓ |\n| Global events | ✓ | ✓ | ✓ | ✓ | | |\n| Rate Limiter | ✓ | ✓ | ✓ | | | |\n| Pause/Resume | ✓ | ✓ | ✓ | ✓ | | |\n| Sandboxed worker | ✓ | ✓ | ✓ | | | |\n| Repeatable jobs | ✓ | ✓ | ✓ | | | ✓ |\n| Atomic ops | ✓ | ✓ | ✓ | | ✓ | |\n| Persistence | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ |\n| UI | ✓ | ✓ | ✓ | ✓ | | ✓ |\n| Optimized for | Jobs / Messages | Jobs / Messages | Jobs / Messages | Jobs | Messages | Jobs |\n\n\n### Install\n\n```bash\nnpm install bull --save\n```\nor\n\n```bash\nyarn add bull\n```\n\n_**Requirements:** Bull requires a Redis version greater than or equal to `2.8.18`._\n\n\n### Typescript Definitions\n\n```bash\nnpm install @types/bull --save-dev\n```\n```bash\nyarn add --dev @types/bull\n```\n\nDefinitions are currently maintained in the [DefinitelyTyped](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/bull) repo.\n\n\n## Contributing\n\nWe welcome all types of contributions, either code fixes, new features or doc improvements.\nCode formatting is enforced by [prettier](https://prettier.io/).\nFor commits please follow conventional [commits convention](https://www.conventionalcommits.org/en/v1.0.0-beta.2/).\nAll code must pass lint rules and test suites before it can be merged into develop.\n\n---\n\n### Quick Guide\n\n#### Basic Usage\n```js\nconst Queue = require('bull');\n\nconst videoQueue = new Queue('video transcoding', 'redis://127.0.0.1:6379');\nconst audioQueue = new Queue('audio transcoding', { redis: { port: 6379, host: '127.0.0.1', password: 'foobared' } }); // Specify Redis connection using object\nconst imageQueue = new Queue('image transcoding');\nconst pdfQueue = new Queue('pdf transcoding');\n\nvideoQueue.process(function (job, done) {\n\n // job.data contains the custom data passed when the job was created\n // job.id contains id of this job.\n\n // transcode video asynchronously and report progress\n job.progress(42);\n\n // call done when finished\n done();\n\n // or give an error if error\n done(new Error('error transcoding'));\n\n // or pass it a result\n done(null, { framerate: 29.5 /* etc... */ });\n\n // If the job throws an unhandled exception it is also handled correctly\n throw new Error('some unexpected error');\n});\n\naudioQueue.process(function (job, done) {\n // transcode audio asynchronously and report progress\n job.progress(42);\n\n // call done when finished\n done();\n\n // or give an error if error\n done(new Error('error transcoding'));\n\n // or pass it a result\n done(null, { samplerate: 48000 /* etc... */ });\n\n // If the job throws an unhandled exception it is also handled correctly\n throw new Error('some unexpected error');\n});\n\nimageQueue.process(function (job, done) {\n // transcode image asynchronously and report progress\n job.progress(42);\n\n // call done when finished\n done();\n\n // or give an error if error\n done(new Error('error transcoding'));\n\n // or pass it a result\n done(null, { width: 1280, height: 720 /* etc... */ });\n\n // If the job throws an unhandled exception it is also handled correctly\n throw new Error('some unexpected error');\n});\n\npdfQueue.process(function (job) {\n // Processors can also return promises instead of using the done callback\n return pdfAsyncProcessor();\n});\n\nvideoQueue.add({ video: 'http://example.com/video1.mov' });\naudioQueue.add({ audio: 'http://example.com/audio1.mp3' });\nimageQueue.add({ image: 'http://example.com/image1.tiff' });\n```\n\n#### Using promises\n\nAlternatively, you can return promises instead of using the `done` callback:\n\n```javascript\nvideoQueue.process(function (job) { // don't forget to remove the done callback!\n // Simply return a promise\n return fetchVideo(job.data.url).then(transcodeVideo);\n\n // Handles promise rejection\n return Promise.reject(new Error('error transcoding'));\n\n // Passes the value the promise is resolved with to the \"completed\" event\n return Promise.resolve({ framerate: 29.5 /* etc... */ });\n\n // If the job throws an unhandled exception it is also handled correctly\n throw new Error('some unexpected error');\n // same as\n return Promise.reject(new Error('some unexpected error'));\n});\n```\n\n#### Separate processes\n\nThe process function can also be run in a separate process. This has several advantages:\n- The process is sandboxed so if it crashes it does not affect the worker.\n- You can run blocking code without affecting the queue (jobs will not stall).\n- Much better utilization of multi-core CPUs.\n- Less connections to redis.\n\nIn order to use this feature just create a separate file with the processor:\n```js\n// processor.js\nmodule.exports = function (job) {\n // Do some heavy work\n\n return Promise.resolve(result);\n}\n```\n\nAnd define the processor like this:\n\n```js\n// Single process:\nqueue.process('/path/to/my/processor.js');\n\n// You can use concurrency as well:\nqueue.process(5, '/path/to/my/processor.js');\n\n// and named processors:\nqueue.process('my processor', 5, '/path/to/my/processor.js');\n```\n\n#### Repeated jobs\n\nA job can be added to a queue and processed repeatedly according to a cron specification:\n\n```js\n paymentsQueue.process(function (job) {\n // Check payments\n });\n\n // Repeat payment job once every day at 3:15 (am)\n paymentsQueue.add(paymentsData, { repeat: { cron: '15 3 * * *' } });\n\n```\n\nAs a tip, check your expressions here to verify they are correct:\n[cron expression generator](https://crontab.cronhub.io)\n\n#### Pause / Resume\n\nA queue can be paused and resumed globally (pass `true` to pause processing for\njust this worker):\n```js\nqueue.pause().then(function () {\n // queue is paused now\n});\n\nqueue.resume().then(function () {\n // queue is resumed now\n})\n```\n\n#### Events\n\nA queue emits some useful events, for example...\n```js\n.on('completed', function (job, result) {\n // Job completed with output result!\n})\n```\n\nFor more information on events, including the full list of events that are fired, check out the [Events reference](./REFERENCE.md#events)\n\n#### Queues performance\n\nQueues are cheap, so if you need many of them just create new ones with different\nnames:\n```javascript\nconst userJohn = new Queue('john');\nconst userLisa = new Queue('lisa');\n.\n.\n.\n```\n\nHowever every queue instance will require new redis connections, check how to [reuse connections](https://github.com/OptimalBits/bull/blob/master/PATTERNS.md#reusing-redis-connections) or you can also use [named processors](https://github.com/OptimalBits/bull/blob/master/REFERENCE.md#queueprocess) to achieve a similar result.\n\n#### Cluster support\n\nNOTE: From version 3.2.0 and above it is recommended to use threaded processors instead.\n\nQueues are robust and can be run in parallel in several threads or processes\nwithout any risk of hazards or queue corruption. Check this simple example\nusing cluster to parallelize jobs across processes:\n```js\nconst Queue = require('bull');\nconst cluster = require('cluster');\n\nconst numWorkers = 8;\nconst queue = new Queue('test concurrent queue');\n\nif (cluster.isMaster) {\n for (let i = 0; i < numWorkers; i++) {\n cluster.fork();\n }\n\n cluster.on('online', function (worker) {\n // Let's create a few jobs for the queue workers\n for (let i = 0; i < 500; i++) {\n queue.add({ foo: 'bar' });\n };\n });\n\n cluster.on('exit', function (worker, code, signal) {\n console.log('worker ' + worker.process.pid + ' died');\n });\n} else {\n queue.process(function (job, jobDone) {\n console.log('Job done by worker', cluster.worker.id, job.id);\n jobDone();\n });\n}\n```\n\n---\n\n\n### Documentation\n\nFor the full documentation, check out the reference and common patterns:\n\n- [Guide](https://optimalbits.github.io/bull/) — Your starting point for developing with Bull.\n- [Reference](./REFERENCE.md) — Reference document with all objects and methods available.\n- [Patterns](./PATTERNS.md) — a set of examples for common patterns.\n- [License](./LICENSE.md) — the Bull license—it's MIT.\n\nIf you see anything that could use more docs, please submit a pull request!\n\n\n\n---\n\n### Important Notes\n\nThe queue aims for an \"at least once\" working strategy. This means that in some situations, a job\ncould be processed more than once. This mostly happens when a worker fails to keep a lock\nfor a given job during the total duration of the processing.\n\nWhen a worker is processing a job it will keep the job \"locked\" so other workers can't process it.\n\nIt's important to understand how locking works to prevent your jobs from losing their lock - becoming _stalled_ -\nand being restarted as a result. Locking is implemented internally by creating a lock for `lockDuration` on interval\n`lockRenewTime` (which is usually half `lockDuration`). If `lockDuration` elapses before the lock can be renewed,\nthe job will be considered stalled and is automatically restarted; it will be __double processed__. This can happen when:\n1. The Node process running your job processor unexpectedly terminates.\n2. Your job processor was too CPU-intensive and stalled the Node event loop, and as a result, Bull couldn't renew the job lock (see [#488](https://github.com/OptimalBits/bull/issues/488) for how we might better detect this). You can fix this by breaking your job processor into smaller parts so that no single part can block the Node event loop. Alternatively, you can pass a larger value for the `lockDuration` setting (with the tradeoff being that it will take longer to recognize a real stalled job).\n\nAs such, you should always listen for the `stalled` event and log this to your error monitoring system, as this means your jobs are likely getting double-processed.\n\nAs a safeguard so problematic jobs won't get restarted indefinitely (e.g. if the job processor always crashes its Node process), jobs will be recovered from a stalled state a maximum of `maxStalledCount` times (default: `1`).\n" }, { "path": "REFERENCE.md", "content": "# Reference\n\n- [Queue](#queue)\n\n - [Queue#process](#queueprocess)\n - [Queue#add](#queueadd)\n - [Queue#addBulk](#queueaddBulk)\n - [Queue#pause](#queuepause)\n - [Queue#isPaused](#queueispaused)\n - [Queue#resume](#queueresume)\n - [Queue#whenCurrentJobsFinished](#queuewhencurrentjobsfinished)\n - [Queue#count](#queuecount)\n - [Queue#removeJobs](#queueremovejobs)\n - [Queue#empty](#queueempty)\n - [Queue#clean](#queueclean)\n - [Queue#obliterate](#queueobliterate)\n - [Queue#close](#queueclose)\n - [Queue#getJob](#queuegetjob)\n - [Queue#getJobs](#queuegetjobs)\n - [Queue#getJobLogs](#queuegetjoblogs)\n - [Queue#getRepeatableJobs](#queuegetrepeatablejobs)\n - [Queue#removeRepeatable](#queueremoverepeatable)\n - [Queue#removeRepeatableByKey](#queueremoverepeatablebykey)\n - [Queue#getJobCounts](#queuegetjobcounts)\n - [Queue#getCompletedCount](#queuegetcompletedcount)\n - [Queue#getFailedCount](#queuegetfailedcount)\n - [Queue#getDelayedCount](#queuegetdelayedcount)\n - [Queue#getActiveCount](#queuegetactivecount)\n - [Queue#getWaitingCount](#queuegetwaitingcount)\n - [Queue#getPausedCount](#queuegetpausedcount)\n - [Queue#getWaiting](#queuegetwaiting)\n - [Queue#getActive](#queuegetactive)\n - [Queue#getDelayed](#queuegetdelayed)\n - [Queue#getCompleted](#queuegetcompleted)\n - [Queue#getFailed](#queuegetfailed)\n - [Queue#getWorkers](#queuegetworkers)\n - [Queue#getMetrics](#queuegetmetrics)\n\n- [Job](#job)\n\n - [Job#progress](#jobprogress)\n - [Job#log](#joblog)\n - [Job#getState](#jobgetstate)\n - [Job#update](#jobupdate)\n - [Job#remove](#jobremove)\n - [Job#retry](#jobretry)\n - [Job#discard](#jobdiscard)\n - [Job#promote](#jobpromote)\n - [Job#finished](#jobfinished)\n - [Job#moveToCompleted](#jobmovetocompleted)\n - [Job#moveToFailed](#jobmovetofailed)\n - [Job#lockKey](#joblockkey)\n - [Job#releaseLock](#jobreleaselock)\n - [Job#takeLock](#jobtakelock)\n - [Job#extendLock](#jobextendlock)\n\n- [Events](#events)\n - [Global events](#global-events)\n\n## Queue\n\n```ts\nQueue(queueName: string, url?: string, opts?: QueueOptions): Queue\n```\n\nThis is the Queue constructor. It creates a new Queue that is persisted in\nRedis. Everytime the same queue is instantiated it tries to process all the\nold jobs that may exist from a previous unfinished session.\n\nThe optional `url` argument, allows to specify a redis connection string such as for example:\n`redis://mypassword@myredis.server.com:1234`\n\n```typescript\ninterface QueueOptions {\n createClient?: (type: 'client' | 'subscriber' | 'bclient', config?: Redis.RedisOptions) => Redis.Redis | Redis.Cluster;\n limiter?: RateLimiter;\n redis?: RedisOpts;\n prefix?: string = 'bull'; // prefix for all queue keys.\n metrics?: MetricsOpts; // Configure metrics\n defaultJobOptions?: JobOpts;\n settings?: AdvancedSettings;\n}\n```\n\n```typescript\ninterface MetricsOpts {\n maxDataPoints?: number; // Max number of data points to collect, granularity is fixed at one minute.\n}\n```\n\n```typescript\ninterface RateLimiter {\n max: number; // Max number of jobs processed\n duration: number; // per duration in milliseconds\n bounceBack?: boolean = false; // When jobs get rate limited, they stay in the waiting queue and are not moved to the delayed queue\n groupKey?: string; // allows grouping of jobs with the specified key from the data object passed to the Queue#add (ex. \"network.handle\")\n}\n```\n\n`RedisOpts` are passed directly to ioredis constructor, check [ioredis](https://github.com/luin/ioredis/blob/master/API.md)\nfor details. We document here just the most important ones.\n\n```typescript\ninterface RedisOpts {\n port?: number = 6379;\n host?: string = localhost;\n db?: number = 0;\n password?: string;\n}\n```\n\n```typescript\ninterface AdvancedSettings {\n lockDuration: number = 30000; // Key expiration time for job locks.\n lockRenewTime: number = 15000; // Interval on which to acquire the job lock\n stalledInterval: number = 30000; // How often check for stalled jobs (use 0 for never checking).\n maxStalledCount: number = 1; // Max amount of times a stalled job will be re-processed.\n guardInterval: number = 5000; // Poll interval for delayed jobs and added jobs.\n retryProcessDelay: number = 5000; // delay before processing next job in case of internal error.\n backoffStrategies: {}; // A set of custom backoff strategies keyed by name.\n drainDelay: number = 5; // A timeout for when the queue is in drained state (empty waiting for jobs).\n isSharedChildPool: boolean = false; // enables multiple queues on the same instance of child pool to share the same instance.\n}\n```\n\n#### Custom or Shared IORedis Connections\n\n`createClient` is passed a `type` to specify the type of connection that Bull is trying to create, and some options that `bull` would like to set for that connection.\n\nYou can merge the provided options with some of your own and create an `ioredis` connection.\n\nWhen type is `client` or `subscriber` you can return the same connection for multiple queues, which can reduce the number of connections you open to the redis server. Bull\ndoes not close or disconnect these connections when queues are closed, so if you need to have your app do a graceful shutdown, you will need to keep references to these\nRedis connections somewhere and disconnect them after you shut down all the queues.\n\nThe `bclient` connection however is a \"blocking client\" and is used to wait for new jobs on a single queue at a time. For this reason it cannot be shared and a\nnew connection should be returned each time.\n\n#### Advanced Settings\n\n**Warning:** Do not override these advanced settings unless you understand the internals of the queue.\n\n`lockDuration`: Time in milliseconds to acquire the job lock. Set this to a higher value if you find that your jobs are being stalled because your job processor is CPU-intensive and blocking the event loop (see note below about stalled jobs). Set this to a lower value if your jobs are extremely time-sensitive and it might be OK if they get double-processed (due to them be falsly considered stalled).\n\n`lockRenewTime`: Interval in milliseconds on which to acquire the job lock. It is set to `lockDuration / 2` by default to give enough buffer to renew the lock each time before the job lock expires. It should never be set to a value larger than `lockDuration`. Set this to a lower value if you're finding that jobs are becoming stalled due to a CPU-intensive job processor function. Generally you shouldn't change this though.\n\n`stalledInterval`: Interval in milliseconds on which each worker will check for stalled jobs (i.e. unlocked jobs in the `active` state). See note below about stalled jobs. Set this to a lower value if your jobs are extremely time-sensitive. Set this to a higher value if your Redis CPU usage is high as this check can be expensive. Note that because each worker runs this on its own interval and checks the entire queue, the stalled job actually run much more frequently than this value would imply.\n\n`maxStalledCount`: The maximum number of times a job can be restarted before it will be permamently failed with the error `job stalled more than allowable limit`. This is set to a default of `1` with the assumption that stalled jobs should be very rare (only due to process crashes) and you want to be on the safer side of not restarting jobs. Set this higher if stalled jobs are common (e.g. processes crash a lot) and it's generally OK to double process jobs.\n\n`guardInterval`: Interval in milliseconds on which the delayed job watchdog will run. When running multiple concurrent workers with delayed tasks, the default value of `guardInterval` will cause spikes on network bandwidth, cpu usage and memory usage. Each concurrent worker will run the delayed job watchdog. In this case set this value to something much higher, e.g. `guardInterval = numberOfWorkers*5000`. Set to a lower value if your Redis connection is unstable and delayed jobs aren't being processed in time.\n\n`retryProcessDelay`: Time in milliseconds in which to wait before trying to process jobs, in case of a Redis error. Set to a lower value on an unstable Redis connection.\n\n`backoffStrategies`: An object containing custom backoff strategies. The key in the object is the name of the strategy and the value is a function that should return the delay in milliseconds. For a full example see [Patterns](./PATTERNS.md#custom-backoff-strategy).\n\n`drainDelay`: A timeout for when the queue is in `drained` state (empty waiting for jobs). It is used when calling `queue.getNextJob()`, which will pass it to `.brpoplpush` on the Redis client.\n\n```js\nbackoffStrategies: {\n jitter: function () {\n return 5000 + Math.random() * 500;\n }\n}\n```\n\n---\n\n### Queue#process\n\n```ts\n/**\n * Consider these as overloaded functions. Since method overloading doesn't exist in JavaScript,\n * Bull recognizes the desired function call by checking the parameters' types.\n * Make sure you comply with one of the below defined patterns.\n *\n * Note: Concurrency defaults to 1 if not specified.\n */\nprocess(processor: ((job, done?) => Promise) | string)\nprocess(concurrency: number, processor: ((job, done?) => Promise) | string)\nprocess(name: string, processor: ((job, done?) => Promise) | string)\nprocess(name: string, concurrency: number, processor: ((job, done?) => Promise) | string)\n```\n\nDefines a processing function for the jobs in a given Queue.\n\nThe callback is called every time a job is placed in the queue. It is passed an instance of the job as first argument.\n\nIf the callback signature contains the second optional `done` argument, the callback will be passed a `done` callback to be called after the job has been completed. The `done` callback can be called with an Error instance, to signal that the job did not complete successfully, or with a result as second argument (e.g.: `done(null, result);`) when the job is successful. Errors will be passed as a second argument to the \"failed\" event;\nresults, as a second argument to the \"completed\" event.\n\nIf, however, the callback signature does not contain the `done` argument, a promise must be returned to signal job completion. If the promise is rejected, the error will be passed as a second argument to the \"failed\" event.\nIf it is resolved, its value will be the \"completed\" event's second argument.\n\nYou can specify a `concurrency` argument. Bull will then call your handler in parallel respecting this maximum value.\n\nA process function can also be declared as a separate process. This will make a better use of the available CPU cores\nand run the jobs in parallel. This is a perfect way to run blocking code. Just specify an absolute path to a processor module.\ni.e. a file exporting the process function like this:\n\n```js\n// my-processor.js\nmodule.exports = function (job) {\n // do some job\n\n return value;\n};\n```\n\nYou can return a value or a promise to signal that the job has been completed.\n\nA `name` argument can be provided so that multiple process functions can be defined per queue. A named process will only process jobs that matches the given name. However, if you define multiple named process functions in one Queue, the defined concurrency for each process function stacks up for the Queue. See the following examples:\n\n```js\n/***\n * For each named processor, concurrency stacks up, so any of these three process functions\n * can run with a concurrency of 125. To avoid this behaviour you need to create an own queue\n * for each process function.\n */\nconst loadBalancerQueue = new Queue('loadbalancer');\nloadBalancerQueue.process('requestProfile', 100, requestProfile);\nloadBalancerQueue.process('sendEmail', 25, sendEmail);\nloadBalancerQueue.process('sendInvitation', 0, sendInvite);\n\nconst profileQueue = new Queue('profile');\n// Max concurrency for requestProfile is 100\nprofileQueue.process('requestProfile', 100, requestProfile);\n\nconst emailQueue = new Queue('email');\n// Max concurrency for sendEmail is 25\nemailQueue.process('sendEmail', 25, sendEmail);\n```\n\nSpecifying `*` as the process name will make it the default processor for all named jobs.\nIt is frequently used to process all named jobs from one process function:\n\n```js\nconst differentJobsQueue = new Queue('differentJobsQueue');\ndifferentJobsQueue.process('*', processFunction);\ndifferentJobsQueue.add('jobA', data, opts);\ndifferentJobsQueue.add('jobB', data, opts);\n```\n\n**Note:** in order to determine whether job completion is signaled by\nreturning a promise or calling the `done` callback, Bull looks at\nthe length property of the callback you pass to it.\nSo watch out, as the following won't work:\n\n```js\n// THIS WON'T WORK!!\nqueue.process(function (job, done) {\n // Oops! done callback here!\n return Promise.resolve();\n});\n```\n\nThis, however, will:\n\n```js\nqueue.process(function (job) {\n // No done callback here :)\n return Promise.resolve();\n});\n```\n\n---\n\n### Queue#add\n\n```ts\nadd(name?: string, data: object, opts?: JobOpts): Promise\n```\n\nCreates a new job and adds it to the queue. If the queue is empty the job will be executed directly, otherwise it will be placed in the queue and executed as soon as possible.\n\nAn optional name can be added, so that only process functions defined for that name (also called job type) will process the job.\n\n**Note:**\nYou need to define _processors_ for all the named jobs that you add to your queue or the queue will complain that you are missing a processor for the given job, unless you use the `*` as job name when defining the processor.\n\n**Note:**\nConsidering all jobs in a finished state (`failed` or `completed`) are stored in Redis, depending on the number of jobs running and your Redis setup, you might want to setup a default maximum number of jobs kept, using the `removeOnComplete` and `removeOnFail` options when creating a queue so Redis does not end up running out of memory.\n\n```typescript\ninterface JobOpts {\n priority: number; // Optional priority value. ranges from 1 (highest priority) to MAX_INT (lowest priority). Note that\n // using priorities has a slight impact on performance, so do not use it if not required.\n\n delay: number; // An amount of milliseconds to wait until this job can be processed. Note that for accurate delays, both\n // server and clients should have their clocks synchronized. [optional].\n\n attempts: number; // The total number of attempts to try the job until it completes.\n\n repeat: RepeatOpts; // Repeat job according to a cron specification, see below for details.\n\n backoff: number | BackoffOpts; // Backoff setting for automatic retries if the job fails, default strategy: `fixed`.\n // Needs `attempts` to be set.\n\n lifo: boolean; // if true, adds the job to the right of the queue instead of the left (default false)\n timeout: number; // The number of milliseconds after which the job should fail with a timeout error [optional]\n\n jobId: number | string; // Override the job ID - by default, the job ID is a unique\n // integer, but you can use this setting to override it.\n // If you use this option, it is up to you to ensure the\n // jobId is unique. If you attempt to add a job with an id that\n // already exists, it will not be added (see caveat below about repeatable jobs).\n\n removeOnComplete: boolean | number | KeepJobs; // If true, removes the job when it successfully\n // completes. A number specified the amount of jobs to keep. Default behavior is to keep the job in the completed set.\n // See KeepJobs if using that interface instead.\n\n removeOnFail: boolean | number | KeepJobs; // If true, removes the job when it fails after all attempts. A number specified the amount of jobs to keep, see KeepJobs if using that interface instead.\n // Default behavior is to keep the job in the failed set.\n stackTraceLimit: number; // Limits the amount of stack trace lines that will be recorded in the stacktrace.\n}\n```\n\n#### KeepJobs Options\n```typescript\n/**\n * KeepJobs\n *\n * Specify which jobs to keep after finishing. If both age and count are\n * specified, then the jobs kept will be the ones that satisfies both\n * properties.\n */\nexport interface KeepJobs {\n /**\n * Maximum age in *seconds* for job to be kept.\n */\n age?: number;\n\n /**\n * Maximum count of jobs to be kept.\n */\n count?: number;\n}\n```\n\n---\n\n#### Timeout Implementation\n\nIt is important to note that jobs are _not_ proactively stopped after the given `timeout`. The job is marked as failed\nand the job's promise is rejected, but Bull has no way to stop the processor function externally.\n\nIf you need a job to stop processing after it times out, here are a couple suggestions:\n - Have the job itself periodically check `job.getStatus()`, and exit if the status becomes `'failed'`\n - Implement the job as a _cancelable promise_. If the processor's promise has a `cancel()` method, it will\n be called when a job times out, and the job can respond accordingly. (Note: currently this only works for\n native Promises, see [#2203](https://github.com/OptimalBits/bull/issues/2203)\n - If you have a way to externally stop a job, add a listener for the `failed` event and do so there.\n\n#### Repeated Job Details\n```typescript\ninterface RepeatOpts {\n cron?: string; // Cron string\n tz?: string; // Timezone\n startDate?: Date | string | number; // Start date when the repeat job should start repeating (only with cron).\n endDate?: Date | string | number; // End date when the repeat job should stop repeating.\n limit?: number; // Number of times the job should repeat at max.\n every?: number; // Repeat every millis (cron setting cannot be used together with this setting.)\n count?: number; // The start value for the repeat iteration count.\n readonly key: string; // The key for the repeatable job metadata in Redis.\n}\n```\n\nAdding a job with the `repeat` option set will actually do two things immediately: create a Repeatable Job configuration,\nand schedule a regular delayed job for the job's first run. This first run will be scheduled \"on the hour\", that is if you create\na job that repeats every 15 minutes at 4:07, the job will first run at 4:15, then 4:30, and so on. If `startDate` is set, the job\nwill not run before `startDate`, but will still run \"on the hour\". In the previous example, if `startDate` was set for some day at\n6:05, the same day, the first job would run on that day at 6:15.\n\nThe cron expression uses the [cron-parser](https://github.com/harrisiirak/cron-parser) library, see their docs for more details.\n\nThe Repeatable Job configuration is not a job, so it will not show up in methods like `getJobs()`. To manage Repeatable Job\nconfigurations, use [`getRepeatableJobs()`](#queuegetrepeatablejobs) and similar. This also means repeated jobs do **not**\nparticipate in evaluating `jobId` uniqueness - that is, a non-repeatable job can have the same `jobId` as a Repeatable Job\nconfiguration, and two Repeatable Job configurations can have the same `jobId` as long as they have different repeat options.\n\nThat is, the following code will result in three jobs being created (one immediate and two delayed):\n```ts\nawait queue.add({}, { jobId: 'example', repeat: { every: 5 * 1000 } })\nawait queue.add({}, { jobId: 'example', repeat: { every: 5 * 1000 } }) // Will not be created, same repeat configuration\nawait queue.add({}, { jobId: 'example', repeat: { every: 10 * 1000 } }) // Will be created, different repeat configuration\nawait queue.add({}, { jobId: 'example' }) // Will be created, no regular job with this id\nawait queue.add({}, { jobId: 'example' }) // Will not be created, conflicts with previous regular job\n```\n\n#### Backoff Options\n```typescript\ninterface BackoffOpts {\n type: string; // Backoff type, which can be either `fixed` or `exponential`. A custom backoff strategy can also be specified in `backoffStrategies` on the queue settings.\n delay: number; // Backoff delay, in milliseconds.\n options?: any; // Options for custom strategies\n}\n```\n\n---\n\n### Queue#addBulk\n\n```ts\naddBulk(jobs: { name?: string, data: object, opts?: JobOpts }[]): Promise\n```\n\nCreates array of jobs and adds them to the queue. They follow the same signature as [Queue#add](#queueadd).\n\n---\n\n### Queue#pause\n\n```ts\npause(isLocal?: boolean, doNotWaitActive?: boolean): Promise\n```\n\nReturns a promise that resolves when the queue is paused. A paused queue will not process new jobs until resumed, but current jobs being processed will continue until they are finalized. The pause can be either global or local. If global, all workers in all queue instances for a given queue will be paused. If local, just this worker will stop processing new jobs after the current lock expires. This can be useful to stop a worker from taking new jobs prior to shutting down.\n\nIf `doNotWaitActive` is `true`, `pause` will _not_ wait for any active jobs to finish before resolving. Otherwise, `pause` _will_ wait for active jobs to finish. See [Queue#whenCurrentJobsFinished](#queuewhencurrentjobsfinished) for more information.\n\nPausing a queue that is already paused does nothing.\n\n---\n\n### Queue#isPaused\n\n```ts\nisPaused(isLocal?: boolean): Promise\n```\n\nChecks if the queue is paused. Pass true if you need to know if this particular instance is paused.\n\n---\n\n### Queue#resume\n\n```ts\nresume(isLocal?: boolean): Promise\n```\n\nReturns a promise that resolves when the queue is resumed after being paused. The resume can be either local or global. If global, all workers in all queue instances for a given queue will be resumed. If local, only this worker will be resumed. Note that resuming a queue globally will _not_ resume workers that have been paused locally; for those, `resume(true)` must be called directly on their instances.\n\nResuming a queue that is not paused does nothing.\n\n---\n\n### Queue#whenCurrentJobsFinished\n\n```ts\nwhenCurrentJobsFinished(): Promise\n```\n\nReturns a promise that resolves when all jobs currently being processed by this worker have finished.\n\n---\n\n### Queue#count\n\n```ts\ncount(): Promise\n```\n\nReturns a promise that returns the number of jobs in the queue, waiting or delayed. Since there may be other processes adding or processing jobs, this value may be true only for a very small amount of time.\n\n---\n\n### Queue#removeJobs\n\n```ts\nremoveJobs(pattern: string): Promise\n```\n\nRemoves all the jobs which jobId matches the given pattern. The pattern must follow redis glob-style pattern [syntax](https://redis.io/commands/keys)\n\nExample:\n\n```js\nmyQueue.removeJobs('?oo*').then(function () {\n console.log('done removing jobs');\n});\n```\n\nWill remove jobs with ids such as: \"boo\", \"foofighter\", etc.\n\nNote: This method does not affect Repeatable Job configurations, instead use [`removeRepeatable()`](#queueremoverepeatable) or [`removeRepeatableByKey()`](#queueremoverepeatablebykey)\n\n---\n\n### Queue#empty\n\n```ts\nempty(): Promise\n```\n\nDrains a queue deleting all the *input* lists and associated jobs.\n\nNote: This function only removes the jobs that are *waiting* to be processed by the queue or *delayed*.\nJobs in other states (active, failed, completed) and Repeatable Job configurations will remain, and\nrepeatable jobs will continue to be created on schedule.\n\nTo remove other job statuses, use [`clean()`](#queueclean), and to remove everything including Repeatable Job\nconfigurations, use [`obliterate()`](#queueobliterate).\n\n---\n\n### Queue#close\n\n```ts\nclose(doNotWaitJobs?: boolean): Promise\n```\n\nCloses the underlying Redis client. Use this to perform a graceful shutdown.\n\n```js\nconst Queue = require('bull');\nconst queue = Queue('example');\n\nconst after100 = _.after(100, function () {\n queue.close().then(function () {\n console.log('done');\n });\n});\n\nqueue.on('completed', after100);\n```\n\n`close` can be called from anywhere, with one caveat: if called\nfrom within a job handler the queue won't close until _after_\nthe job has been processed, so the following won't work:\n\n```js\nqueue.process(function (job, jobDone) {\n handle(job);\n queue.close().then(jobDone);\n});\n```\n\nInstead, do this:\n\n```js\nqueue.process(function (job, jobDone) {\n handle(job);\n queue.close();\n jobDone();\n});\n```\n\nOr this:\n\n```js\nqueue.process(function (job) {\n queue.close();\n return handle(job).then(...);\n});\n```\n\n---\n\n### Queue#getJob\n\n```ts\ngetJob(jobId: string): Promise\n```\n\nReturns a promise that will return the job instance associated with the `jobId`\nparameter. If the specified job cannot be located, the promise will be resolved to `null`.\n\nNote: This method does not return Repeatable Job configurations, to do so see [`getRepeatableJobs()`](#queuegetrepeatablejobs)\n\n---\n\n### Queue#getJobs\n\n```ts\ngetJobs(types: JobStatus[], start?: number, end?: number, asc?: boolean): Promise\n```\n\nReturns a promise that will return an array of job instances of the given job statuses. Optional parameters for range and ordering are provided.\n\nNote: The `start` and `end` options are applied **per job statuses**. For example, if there are 10 jobs in state `completed` and 10 jobs in state `active`, `getJobs(['completed', 'active'], 0, 4)` will yield an array with 10 entries, representing the first 5 completed jobs (0 - 4) and the first 5 active jobs (0 - 4).\n\nThis method does not return Repeatable Job configurations, to do so see [`getRepeatableJobs()`](#queuegetrepeatablejobs)\n\n---\n\n### Queue#getJobLogs\n\n```ts\ngetJobLogs(jobId: string, start?: number, end?: number): Promise<{\n logs: string[],\n count: number\n}>\n```\n\nReturns a object with the logs according to the start and end arguments. The returned count\nvalue is the total amount of logs, useful for implementing pagination.\n\n---\n\n### Queue#getRepeatableJobs\n\n```ts\ngetRepeatableJobs(start?: number, end?: number, asc?: boolean): Promise<{\n key: string,\n name: string,\n id: number | string,\n endDate: Date,\n tz: string,\n cron: string,\n every: number,\n next: number\n }[]>\n```\n\nReturns a promise that will return an array of Repeatable Job configurations. Optional parameters for range and ordering are provided.\n\n---\n\n### Queue#removeRepeatable\n\n```ts\nremoveRepeatable(name?: string, repeat: RepeatOpts): Promise\n```\n\nRemoves a given Repeatable Job configuration. The RepeatOpts needs to be the same as the ones used\nfor the job when it was added.\n\n---\n\n---\n\n### Queue#removeRepeatableByKey\n\n```ts\nremoveRepeatableByKey(key: string): Promise\n```\n\nRemoves a given Repeatable Job configuration by its key so that no more repeatable jobs will be processed for this\nparticular configuration.\n\nThere are currently two ways to get the \"key\" of a repeatable job.\n\nWhen first creating the job, `queue.add()` will return a job object with the key for that job, which you can store for later use:\n```ts\nconst job = await queue.add('remove', { example: 'data' }, { repeat: { every: 1000 } });\n// store job.opts.repeat.key somewhere...\nconst repeatableKey = job.opts.repeat.key;\n\n// ...then later...\nawait queue.removeRepeatableByKey(repeatableKey);\n```\n\nOtherwise, you can list all repeatable jobs with [`getRepeatableJobs()`](#queuegetrepeatablejobs), find the job you want to remove in the list, and use the key there to remove it:\n```ts\nawait queue.add('remove', { example: 'data' }, { jobId: 'findMe', repeat: { every: 1000 } })\n\n// ... then later ...\nconst repeatableJobs = await queue.getRepeatableJobs()\nconst foundJob = repeatableJobs.find(job => job.id === 'findMe')\nawait queue.removeRepeatableByKey(foundJob.key)\n```\n---\n\n### Queue#getJobCounts\n\n```ts\ngetJobCounts() : Promise\n```\n\nReturns a promise that will return the job counts for the given queue.\n\n```typescript\n{\n interface JobCounts {\n waiting: number,\n active: number,\n completed: number,\n failed: number,\n delayed: number\n }\n}\n```\n\n---\n\n### Queue#getCompletedCount\n\n```ts\ngetCompletedCount() : Promise\n```\n\nReturns a promise that will return the completed job counts for the given queue.\n\n---\n\n### Queue#getFailedCount\n\n```ts\ngetFailedCount() : Promise\n```\n\nReturns a promise that will return the failed job counts for the given queue.\n\n---\n\n### Queue#getDelayedCount\n\n```ts\ngetDelayedCount() : Promise\n```\n\nReturns a promise that will return the delayed job counts for the given queue.\n\n---\n\n### Queue#getActiveCount\n\n```ts\ngetActiveCount() : Promise\n```\n\nReturns a promise that will return the active job counts for the given queue.\n\n---\n\n### Queue#getWaitingCount\n\n```ts\ngetWaitingCount() : Promise\n```\n\nReturns a promise that will return the waiting job counts for the given queue.\n\n---\n\n### Queue#getPausedCount\n\n*DEPRECATED* Since only the queue can be paused, getWaitingCount gives the same\nresult.\n\n```ts\ngetPausedCount() : Promise\n```\n\nReturns a promise that will return the paused job counts for the given queue.\n\n---\n\n### Getters\n\nThe following methods are used to get the jobs that are in certain states.\n\nThe GetterOpts can be used for configure some aspects from the getters.\n\n```ts\ninterface GetterOpts\n excludeData: boolean; // Exclude the data field of the jobs.\n```\n\n### Queue#getWaiting\n\n```ts\ngetWaiting(start?: number, end?: number, opts?: GetterOpts) : Promise>\n```\n\nReturns a promise that will return an array with the waiting jobs between start and end.\n\n---\n\n### Queue#getActive\n\n```ts\ngetActive(start?: number, end?: number, opts?: GetterOpts) : Promise>\n```\n\nReturns a promise that will return an array with the active jobs between start and end.\n\n---\n\n### Queue#getDelayed\n\n```ts\ngetDelayed(start?: number, end?: number, opts?: GetterOpts) : Promise>\n```\n\nReturns a promise that will return an array with the delayed jobs between start and end.\n\n---\n\n### Queue#getCompleted\n\n```ts\ngetCompleted(start?: number, end?: number, opts?: GetterOpts) : Promise>\n```\n\nReturns a promise that will return an array with the completed jobs between start and end.\n\n---\n\n### Queue#getFailed\n\n```ts\ngetFailed(start?: number, end?: number, opts?: GetterOpts) : Promise>\n```\n\nReturns a promise that will return an array with the failed jobs between start and end.\n\n---\n\n### Queue#getWorkers\n\n```ts\ngetWorkers() : Promise>\n```\n\nReturns a promise that will resolve to an array workers currently listening or processing jobs.\nThe object includes the same fields as [Redis CLIENT LIST](https://redis.io/commands/client-list) command.\n\n---\n\n### Queue#getMetrics\n\n```ts\ngetMetrics(type: 'completed' | 'failed', start = 0, end = -1) : Promise<{\n meta: {\n count: number;\n prevTS: number;\n prevCount: number;\n };\n data: number[];\n count: number;\n}>\n```\n\nReturns a promise that resolves to a Metrics object.\n\n---\n\n### Queue#clean\n\n```ts\nclean(grace: number, status?: string, limit?: number): Promise\n```\n\nTells the queue remove jobs of a specific type created outside of a grace period.\n\n#### Example\n\n```js\nqueue.on('cleaned', function (jobs, type) {\n console.log('Cleaned %s %s jobs', jobs.length, type);\n});\n\n//cleans all jobs that completed over 5 seconds ago.\nawait queue.clean(5000);\n//clean all jobs that failed over 10 seconds ago.\nawait queue.clean(10000, 'failed');\n```\n\n### Queue#obliterate\n\n```ts\nobliterate(ops?: { force: boolean}): Promise\n```\n\nCompletely removes a queue with all its data.\nIn order to obliterate a queue there cannot be active jobs, but this\nbehaviour can be overrided with the \"force\" option.\n\nNote: since this operation can be quite long in duration depending on how\nmany jobs there are in the queue, it is not performed atomically, instead\nis performed iterativelly. However the queue is always paused during this process,\nif the queue gets unpaused during the obliteration by another script, the call\nwill fail with the removed items it managed to remove until the failure.\n\n#### Example\n\n```js\n// Removes everything but only if there are no active jobs\nawait queue.obliterate();\n\nawait queue.obliterate({ force: true });\n```\n\n---\n\n## Job\n\nA job includes all data needed to perform its execution, as well as the progress method needed to update its progress.\n\nThe most important property for the user is `Job#data` that includes the object that was passed to [`Queue#add`](#queueadd), and that is normally used to perform the job.\n\nOther useful job properties:\n* `job.attemptsMade`: number of failed attempts.\n* `job.finishedOn`: Unix Timestamp, when job is completed or finally failed after all attempts.\n\n### Job#progress\n\n```ts\nprogress(progress?: number | object): Promise\n```\n\nUpdates a job progress if called with an argument.\nReturn a promise resolving to the current job's progress if called without argument.\n\n#### Arguments\n\n```js\n progress: number; Job progress number or any serializable object representing progress or similar.\n```\n\n---\n\n### Job#log\n\n```ts\nlog(row: string): Promise\n```\n\nAdds a log row to this job specific job. Logs can be retrieved using [Queue#getJobLogs](#queuegetjoblogs).\n\n---\n\n### Job#getState\n\n```ts\ngetState(): Promise\n```\n\nReturns a promise resolving to the current job's status (completed, failed, delayed etc.). Possible returns are: completed, failed, delayed, active, waiting, paused, stuck or null.\n\nPlease take note that the implementation of this method is not very efficient, nor is it atomic. If your queue does have a very large quantity of jobs, you may want to avoid using this method.\n\n---\n\n### Job#update\n\n```ts\nupdate(data: object): Promise\n```\n\nUpdated a job data field with the give data object.\n\n---\n\n### Job#remove\n\n```ts\nremove(): Promise\n```\n\nRemoves a job from the queue and from any lists it may be included in.\n\n---\n\n### Job#retry\n\n```ts\nretry(): Promise\n```\n\nRe-run a job that has failed. Returns a promise that resolves when the job is scheduled for retry.\n\n---\n\n### Job#discard\n\n```ts\ndiscard(): Promise\n```\n\nEnsure this job is never ran again even if `attemptsMade` is less than `job.attempts`.\n\n---\n\n### Job#promote\n\n```ts\npromote(): Promise\n```\n\nPromotes a job that is currently \"delayed\" to the \"waiting\" state and executed as soon as possible.\n\n---\n\n### Job#finished\n\n```ts\nfinished(): Promise\n```\n\nReturns a promise that resolves or rejects when the job completes or fails.\n\n---\n\n### Job#moveToCompleted\n\n```ts\nmoveToCompleted(returnValue: any, ignoreLock: boolean, notFetch?: boolean): Promise\n```\n\nMoves a job to the `completed` queue. Pulls a job from 'waiting' to 'active' and returns a tuple containing the next jobs data and id. If no job is in the `waiting` queue, returns null. Set `notFetch` to true to avoid prefetching the next job in the queue.\n\n---\n\n### Job#moveToFailed\n\n```ts\nmoveToFailed(errorInfo:{ message: string; }, ignoreLock?:boolean): Promise\n```\n\nMoves a job to the `failed` queue. Pulls a job from 'waiting' to 'active' and returns a tuple containing the next jobs data and id. If no job is in the `waiting` queue, returns null.\n\n---\n\n### Job#lockKey\n\n```ts\nlockKey(): string\n```\n\nReturn a unique key representing a lock for this Job.\n\n---\n\n### Job#releaseLock\n\n```ts\nreleaseLock(): Promise\n```\n\nReleases the lock on the job. Only locks owned by the queue instance can be released.\n\n---\n\n### Job#takeLock\n\n```ts\ntakeLock(): Promise\n```\n\nTakes a lock for this job so that no other queue worker can process it at the same time.\n\n---\n\n### Job#extendLock\n\n```ts\nextendLock(duration: number): Promise\n```\n\nExtend the lock for this job. The `duration` parameter specifies the lock duration in milliseconds. It will return '1' on success and '0' on failure.\n\n---\n## Events\n\nA queue emits also some useful events:\n\n```js\n.on('error', function (error) {\n // An error occured.\n})\n\n.on('waiting', function (jobId) {\n // A Job is waiting to be processed as soon as a worker is idling.\n});\n\n.on('active', function (job, jobPromise) {\n // A job has started. You can use `jobPromise.cancel()`` to abort it.\n})\n\n.on('stalled', function (job) {\n // A job has been marked as stalled. This is useful for debugging job\n // workers that crash or pause the event loop.\n})\n\n.on('lock-extension-failed', function (job, err) {\n // A job failed to extend lock. This will be useful to debug redis\n // connection issues and jobs getting restarted because workers\n // are not able to extend locks.\n});\n\n.on('progress', function (job, progress) {\n // A job's progress was updated!\n})\n\n.on('completed', function (job, result) {\n // A job successfully completed with a `result`.\n})\n\n.on('failed', function (job, err) {\n // A job failed with reason `err`!\n})\n\n.on('paused', function () {\n // The queue has been paused.\n})\n\n.on('resumed', function (job) {\n // The queue has been resumed.\n})\n\n.on('cleaned', function (jobs, type) {\n // Old jobs have been cleaned from the queue. `jobs` is an array of cleaned\n // jobs, and `type` is the type of jobs cleaned.\n});\n\n.on('drained', function () {\n // Emitted every time the queue has processed all the waiting jobs (even if there can be some delayed jobs not yet processed)\n});\n\n.on('removed', function (job) {\n // A job successfully removed.\n});\n\n```\n\n### Global events\n\nEvents are local by default — in other words, they only fire on the listeners that are registered on the given worker. If you need to listen to events globally, for example from other servers across redis, just prefix the event with `'global:'`:\n\n```js\n// Will listen locally, just to this queue...\nqueue.on('completed', listener):\n\n// Will listen globally, to instances of this queue...\nqueue.on('global:completed', listener);\n```\n\nWhen working with global events whose local counterparts pass a `Job` instance to the event listener callback, notice that global events pass the **job's ID** instead.\n\nIf you need to access the `Job` instance in a global listener, use [Queue#getJob](#queuegetjob) to retrieve it. However, remember that if `removeOnComplete` is enabled when adding the job, the job will no longer be available after completion. Should you need to both access the job and remove it after completion, you can use [Job#remove](#jobremove) to remove it in the listener.\n\n```js\n// Local events pass the job instance...\nqueue.on('progress', function (job, progress) {\n console.log(`Job ${job.id} is ${progress * 100}% ready!`);\n});\n\nqueue.on('completed', function (job, result) {\n console.log(`Job ${job.id} completed! Result: ${result}`);\n job.remove();\n});\n\n// ...whereas global events only pass the job ID:\nqueue.on('global:progress', function (jobId, progress) {\n console.log(`Job ${jobId} is ${progress * 100}% ready!`);\n});\n\nqueue.on('global:completed', function (jobId, result) {\n console.log(`Job ${jobId} completed! Result: ${result}`);\n queue.getJob(jobId).then(function (job) {\n job.remove();\n });\n});\n```\n" }, { "path": "commandTransform.js", "content": "'use strict';\nconst path = require('path');\nconst fs = require('fs');\nconst { argv } = require('process');\n\nconst readFile = fs.promises.readFile;\nconst writeFile = fs.promises.writeFile;\nconst readdir = fs.promises.readdir;\n\nconst loadScripts = async (readDir, writeDir) => {\n const normalizedDir = path.normalize(readDir);\n\n const files = await readdir(normalizedDir);\n\n const luaFiles = files.filter(file => path.extname(file) === '.lua');\n const writeFilenamePath = path.normalize(writeDir);\n\n if (!fs.existsSync(writeFilenamePath)) {\n fs.mkdirSync(writeFilenamePath);\n }\n\n let indexContent = \"'use strict';\\nmodule.exports = {\\n\";\n\n if (luaFiles.length === 0) {\n /**\n * To prevent unclarified runtime error \"updateDelayset is not a function\n * @see https://github.com/OptimalBits/bull/issues/920\n */\n throw new Error('No .lua files found!');\n }\n\n for (let i = 0; i < luaFiles.length; i++) {\n const completedFilename = path.join(normalizedDir, luaFiles[i]);\n const longName = path.basename(luaFiles[i], '.lua');\n indexContent += ` [\"${longName}\"]: require('./${longName}'),\\n`;\n\n await loadCommand(completedFilename, longName, writeFilenamePath);\n }\n indexContent += `}\\n`;\n\n await writeFile(path.join(writeFilenamePath, 'index.js'), indexContent);\n};\n\nconst loadCommand = async (filename, longName, writeFilenamePath) => {\n const filenamePath = path.resolve(filename);\n\n const content = (await readFile(filenamePath)).toString();\n\n const [name, num] = longName.split('-');\n const numberOfKeys = num && parseInt(num, 10);\n\n const newContent = `'use strict';\nconst content = \\`${content}\\`;\nmodule.exports = {\n name: '${name}',\n content,${\n numberOfKeys\n ? `\n keys: ${numberOfKeys},`\n : ''\n }\n};\n`;\n await writeFile(path.join(writeFilenamePath, longName + '.js'), newContent);\n};\n\nloadScripts(argv[2], argv[3]);\n" }, { "path": "commitlint.config.js", "content": "'use strict';\n\nmodule.exports = { extends: ['@commitlint/config-conventional'] };\n" }, { "path": "docker-compose.yml", "content": "version: '3.2'\nservices:\n redis:\n image: redis:6.2-alpine\n container_name: cache\n ports:\n - 6379:6379\n" }, { "path": "docs/README.md", "content": "
\n
\n \n
\n
\n\n# What is Bull?\n\nBull is a Node library that implements a fast and robust queue system based on [redis](https://redis.io).\n\nAlthough it is possible to implement queues directly using Redis commands, this library provides an API that takes care of all the low-level details and enriches Redis basic functionality so that more complex use-cases can be handled easily.\n\nIf you are new to queues you may wonder why they are needed after all. Queues can solve many different problems in an elegant way, from smoothing out processing peaks to creating robust communication channels between microservices or offloading heavy work from one server to many smaller workers, etc.\n\n# Getting Started\n\nBull is a public npm package and can be installed using either npm or yarn:\n\n```bash\n$ npm install bull --save\n```\n\nor\n\n```bash\n$ yarn add bull\n```\n\nIn order to work with Bull, you also need to have a Redis server running. For local development, you can easily install\nit using [docker](https://hub.docker.com/_/redis/).\n\nBull will by default try to connect to a Redis server running on `localhost:6379`\n\n# Simple Queues\n\nA queue is simply created by instantiating a Bull instance:\n\n```js\nconst myFirstQueue = new Bull('my-first-queue');\n```\n\nA queue instance can normally have 3 main different roles: A job producer, a job consumer or/and an events listener.\n\nAlthough one given instance can be used for the 3 roles, normally the producer and consumer are divided into several instances. A given queue, always referred to by its instantiation name ( `my-first-queue` in the example above ), can have many producers, many consumers, and many listeners. An important aspect is that producers can add jobs to a queue even if there are no consumers available at that moment: queues provide asynchronous communication, which is one of the features that makes them so powerful.\n\nConversely, you can have one or more workers consuming jobs from the queue, which will consume the jobs in a given order: FIFO (the default), LIFO or according to priorities.\n\nTalking about workers, they can run in the same or different processes, in the same machine, or in a cluster. Redis will act as a common point, and as long as a consumer or producer can connect to Redis, they will be able to co-operate in processing the jobs.\n\n## Producers\n\nA job producer is simply some Node program that adds jobs to a queue, like this:\n\n```js\nconst myFirstQueue = new Bull('my-first-queue');\n\nconst job = await myFirstQueue.add({\n foo: 'bar'\n});\n```\n\nAs you can see a job is just a javascript object. This object needs to be serializable, more concrete it should be possible to JSON stringify it since that is how it is going to be stored in Redis.\n\nIt is also possible to provide an options object after the job's data, but we will cover that later on.\n\n## Consumers\n\nA consumer or worker (we will use these two terms interchangeably in this guide), is nothing more than a Node program\nthat defines a process function like so:\n\n```js\nconst myFirstQueue = new Bull('my-first-queue');\n\nmyFirstQueue.process(async (job) => {\n return doSomething(job.data);\n});\n```\n\nThe `process` function will be called every time the worker is idling and there are jobs to process in the queue. Since\nthe consumer does not need to be online when the jobs are added, the queue could have many jobs already waiting in it. So then the process will be kept busy processing jobs one by one until all of them are done.\n\nIn the example above we define the process function as `async`, which is the highly recommended way to define them.\nIf your Node runtime does not support async/await, then you can just return a promise at the end of the process\nfunction for a similar result.\n\nThe value returned by your process function will be stored in the jobs object and can be accessed later on, for example\nin a listener for the `completed` event.\n\nSometimes you need to provide a job's _progress_ information to an external listener, this can be easily accomplished\nby using the `progress` method on the job object:\n\n```js\nmyFirstQueue.process(async (job) => {\n let progress = 0;\n for (i = 0; i < 100; i++) {\n await doSomething(job.data);\n progress += 10;\n job.progress(progress);\n }\n});\n```\n\n## Listeners\n\nFinally, you can just listen to events that happen in the queue. Listeners can be local, meaning that they only will\nreceive notifications produced in the _given queue instance_, or global, meaning that they listen to _all_ the events\nfor a given queue. So you can attach a listener to any instance, even instances that are acting as consumers or producers. But note that a local event will never fire if the queue is not a consumer or producer, you will need to use global events in that\ncase.\n\n```js\nconst myFirstQueue = new Bull('my-first-queue');\n\n// Define a local completed event\nmyFirstQueue.on('completed', (job, result) => {\n console.log(`Job completed with result ${result}`);\n})\n```\n\n## A Job's Lifecycle\n\nIn order to use the full potential of Bull queues, it is important to understand the lifecycle of a job.\nFrom the moment a producer calls the `add` method on a queue instance, a job enters a lifecycle where it will\nbe in different states, until its completion or failure (although technically a failed job could be retried and get a new lifecycle).\n\n![Diagram showing job statuses](job-lifecycle.png)\n\nWhen a job is added to a queue it can be in one of two states, it can either be in the \"wait\" status, which is, in fact, a waiting list, where all jobs must enter before they can be processed, or it can be in a \"delayed\" status: a delayed status implies that the job is waiting for some timeout or to be promoted for being processed. However, a delayed job will not be processed directly, instead, it will be placed at the beginning of the waiting list and processed as soon as a worker is idle.\n\nThe next state for a job is the \"active\" state. The active state is represented by a set, and are jobs that are currently being\nprocessed, i.e. they are running in the `process` function explained in the previous chapter. A job can be in the active state for an unlimited amount of time until the process is completed or an exception is thrown so that the job will end in\neither the \"completed\" or the \"failed\" status.\n\n## Stalled jobs\n\nIn Bull, we defined the concept of stalled jobs. A stalled job is a job that is being processed but where Bull suspects that\nthe process function has hanged. This happens when the process function is processing a job and is keeping the CPU so busy that\nthe worker is not able to tell the queue that it is still working on the job.\n\nWhen a job stalls, depending on the job settings the job can be retried by another idle worker or it can just move to the failed status.\n\nStalled jobs can be avoided by either making sure that the process function does not keep the Node event loop busy for too long (we are talking several seconds with Bull default options), or by using a separate [sandboxed processor](#sandboxed-processors).\n\n# Events\n\nA Queue in Bull generates a handful of events that are useful in many use cases.\nEvents can be local for a given queue instance (a worker), for example, if a job is completed in a given worker, a local event will be emitted just for that instance. However, it is possible to listen to all events, by prefixing ```global:``` to the local event name. Then we can listen to all the events produced by all the workers of a given queue.\n\nA local complete event:\n\n```js\nqueue.on('completed', job => {\n console.log(`Job with id ${job.id} has been completed`);\n})\n```\n\nWhereas the global version of the event can be listened to with:\n\n```js\nqueue.on('global:completed', jobId => {\n console.log(`Job with id ${jobId} has been completed`);\n})\n```\n\nNote that signatures of global events are slightly different than their local counterpart, in the example above it is only sent the job id not a complete instance of the job itself, this is done for performance reasons.\n\nThe list of available events can be found in the [reference](https://github.com/OptimalBits/bull/blob/master/REFERENCE.md#eventsk).\n\n# Queue Options\n\nA queue can be instantiated with some useful options, for instance, you can specify the location and password of your Redis server,\nas well as some other useful settings. All these settings are described in Bull's [reference](https://github.com/OptimalBits/bull/blob/master/REFERENCE.md#queue) and we will not repeat them here. However, we will go through some use cases.\n\n## Rate Limiter\n\nIt is possible to create queues that limit the number of jobs processed in a unit of time. The limiter is defined per queue, independently of the number of workers, so you can scale horizontally and still limit the rate of processing easily:\n\n```js\n// Limit queue to max 1000 jobs per 5000 milliseconds.\nconst myRateLimitedQueue = new Queue('rateLimited', {\n limiter: {\n max: 1000,\n duration: 5000\n }\n});\n```\n\nWhen a queue hits the rate limit, requested jobs will join the `delayed` queue.\n\n## Named jobs\n\nIt is possible to give names to jobs. This does not change any of the mechanics of the queue but can be used for clearer code and\nbetter visualization in UI tools:\n\n```js\n// Jobs producer\nconst myJob = await transcoderQueue.add('image', { input: 'myimagefile' });\nconst myJob = await transcoderQueue.add('audio', { input: 'myaudiofile' });\nconst myJob = await transcoderQueue.add('video', { input: 'myvideofile' });\n```\n\n```js\n// Worker\ntranscoderQueue.process('image', processImage);\ntranscoderQueue.process('audio', processAudio);\ntranscoderQueue.process('video', processVideo);\n```\n\nJust keep in mind that every queue instance is required to provide a processor for *every* named job or you will get an exception.\n\n## Sandboxed Processors\n\nAs explained above, when defining a process function, it is also possible to provide a concurrency setting. This setting allows the worker to process several\njobs in parallel. The jobs are still processed in the same Node process,\nand if the jobs are very IO intensive they will be handled just fine.\n\nSometimes jobs are more CPU intensive which could lock the Node event loop\nfor too long and Bull could decide the job has been stalled. To avoid this situation, it is possible to run the process functions in separate Node processes. In this case, the concurrency parameter will decide the maximum number of concurrent processes that are allowed to run.\n\nWe call these kinds of processes \"sandboxed\" processes, and they also have the property that if they crash they will not affect any other process, and a new\nprocess will be spawned automatically to replace it.\n\n\n# Job types\n\nThe default job type in Bull is \"FIFO\" (first in first out), meaning that the jobs are processed in the same order they are coming into the\nqueue. Sometimes it is useful to process jobs in a different order.\n\n## LIFO\n\nLifo (last in first out) means that jobs are added to the beginning of the queue and therefore will be processed as soon as the worker is idle.\n\n```js\nconst myJob = await myqueue.add({ foo: 'bar' }, { lifo: true });\n```\n\n## Delayed\n\nIt is also possible to add jobs to the queue that are delayed a certain amount of time before they will be processed. Note that the delay parameter means the _minimum_ amount of time the job will wait before being processed. When the delay time has passed the job will be moved to the beginning of the queue and be processed as soon as a worker is idle.\n\n```js\n// Delayed 5 seconds\nconst myJob = await myqueue.add({ foo: 'bar' }, { delay: 5000 });\n```\n\n## Prioritized\n\nJobs can be added to a queue with a priority value. Jobs with higher priority will be processed before jobs with lower priority. The highest priority is 1, and the larger the integer you use, the lower the priority of the job. Keep in mind that priority queues are a bit slower than a standard queue (currently insertion time O(n), n being the number of jobs currently waiting in the queue, instead of O(1) for standard queues).\n\n```js\nconst myJob = await myqueue.add({ foo: 'bar' }, { priority: 3 });\n```\n\n## Repeatable\n\nRepeatable jobs are special jobs that repeat themselves indefinitely or until a given maximum date or the number of repetitions has been reached, according to a cron specification or a time interval.\n\n```js\n// Repeat every 10 seconds for 100 times.\nconst myJob = await myqueue.add(\n { foo: 'bar' },\n {\n repeat: {\n every: 10000,\n limit: 100\n }\n }\n);\n\n// Repeat payment job once every day at 3:15 (am)\npaymentsQueue.add(paymentsData, { repeat: { cron: '15 3 * * *' } });\n```\n\nThere are some important considerations regarding repeatable jobs:\n\n- Bull is smart enough not to add the same repeatable job if the repeat options are the same. (CAUTION: A job id is part of the repeat options since: https://github.com/OptimalBits/bull/pull/603, therefore passing job ids will allow jobs with the same cron to be inserted in the queue)\n- If there are no workers running, repeatable jobs will not accumulate the next time a worker is online.\n- Repeatable jobs can be removed using the [removeRepeatable](https://github.com/OptimalBits/bull/blob/master/REFERENCE.md#queueremoverepeatable) method.\n" }, { "path": "docs/_config.yml", "content": "theme: jekyll-theme-minimal\ntitle: Welcome to Bull's Guide\n" }, { "path": "generateRawScripts.js", "content": "'use strict';\nconst { ScriptLoader } = require('./lib/commands');\nconst path = require('path');\nconst fs = require('fs');\nconst { promisify } = require('util');\n\nconst writeFile = promisify(fs.writeFile);\n\nclass RawScriptLoader extends ScriptLoader {\n /**\n * Transpile lua scripts in one file, specifying an specific directory to be saved\n * @param pathname - the path to the directory containing the scripts\n * @param writeDir - the path to the directory where scripts will be saved\n */\n async transpileScripts(pathname, writeDir) {\n const writeFilenamePath = path.normalize(writeDir);\n\n if (!fs.existsSync(writeFilenamePath)) {\n fs.mkdirSync(writeFilenamePath);\n }\n\n const paths = new Set();\n if (!paths.has(pathname)) {\n paths.add(pathname);\n const scripts = await this.loadScripts(pathname);\n for (const command of scripts) {\n const {\n name,\n options: { numberOfKeys, lua }\n } = command;\n await writeFile(\n path.join(writeFilenamePath, `${name}-${numberOfKeys}.lua`),\n lua\n );\n }\n }\n }\n}\n\nconst scriptLoader = new RawScriptLoader();\n\nscriptLoader.transpileScripts(\n path.join(__dirname, './lib/commands'),\n path.join(__dirname, './rawScripts')\n);\n" }, { "path": "index.d.ts", "content": "// Type definitions for bull 3.15\n// Project: https://github.com/OptimalBits/bull\n// Definitions by: Bruno Grieder \n// Cameron Crothers \n// Marshall Cottrell \n// Weeco \n// Oleg Repin \n// David Koblas \n// Bond Akinmade \n// Wuha Team \n// Alec Brunelle \n// Dan Manastireanu \n// Kjell-Morten Bratsberg Thorsen \n// Christian D. \n// Silas Rech \n// DoYoung Ha \n// Borys Kupar \n// Remko Klein \n// Levi Bostian \n// Todd Dukart \n// Mix \n// Definitions: https://github.com/DefinitelyTyped/DefinitelyTyped\n// TypeScript Version: 2.8\n\nimport * as Redis from 'ioredis';\nimport { EventEmitter } from 'events';\n\n/**\n * This is the Queue constructor.\n * It creates a new Queue that is persisted in Redis.\n * Everytime the same queue is instantiated it tries to process all the old jobs that may exist from a previous unfinished session.\n */\ndeclare const Bull: {\n /* tslint:disable:no-unnecessary-generics unified-signatures */\n (queueName: string, opts?: Bull.QueueOptions): Bull.Queue;\n (\n queueName: string,\n url: string,\n opts?: Bull.QueueOptions\n ): Bull.Queue;\n new (queueName: string, opts?: Bull.QueueOptions): Bull.Queue;\n new (\n queueName: string,\n url: string,\n opts?: Bull.QueueOptions\n ): Bull.Queue;\n /* tslint:enable:no-unnecessary-generics unified-signatures */\n};\n\ndeclare namespace Bull {\n interface RateLimiter {\n /** Max numbers of jobs processed */\n max: number;\n /** Per duration in milliseconds */\n duration: number;\n /** When jobs get rate limited, they stay in the waiting queue and are not moved to the delayed queue */\n bounceBack?: boolean | undefined;\n /** Groups jobs with the specified key from the data object passed to the Queue#add ex. \"network.handle\" */\n groupKey?: string | undefined;\n }\n\n interface QueueOptions {\n /**\n * Options passed into the `ioredis` constructor's `options` parameter.\n * `connectionName` is overwritten with `Queue.clientName()`. other properties are copied\n */\n redis?: Redis.RedisOptions | string | undefined;\n\n /**\n * When specified, the `Queue` will use this function to create new `ioredis` client connections.\n * This is useful if you want to re-use connections or connect to a Redis cluster.\n */\n createClient?(\n type: 'client' | 'subscriber' | 'bclient',\n redisOpts?: Redis.RedisOptions\n ): Redis.Redis | Redis.Cluster;\n\n /**\n * Prefix to use for all redis keys\n */\n prefix?: string | undefined;\n\n settings?: AdvancedSettings | undefined;\n\n limiter?: RateLimiter | undefined;\n\n defaultJobOptions?: JobOptions | undefined;\n\n metrics?: MetricsOpts; // Configure metrics\n }\n\n interface MetricsOpts {\n maxDataPoints?: number; // Max number of data points to collect, granularity is fixed at one minute.\n }\n\n interface AdvancedSettings {\n /**\n * Key expiration time for job locks\n */\n lockDuration?: number | undefined;\n\n /**\n * Interval in milliseconds on which to acquire the job lock.\n */\n lockRenewTime?: number | undefined;\n\n /**\n * How often check for stalled jobs (use 0 for never checking)\n */\n stalledInterval?: number | undefined;\n\n /**\n * Max amount of times a stalled job will be re-processed\n */\n maxStalledCount?: number | undefined;\n\n /**\n * Poll interval for delayed jobs and added jobs\n */\n guardInterval?: number | undefined;\n\n /**\n * Delay before processing next job in case of internal error\n */\n retryProcessDelay?: number | undefined;\n\n /**\n * Define a custom backoff strategy\n */\n backoffStrategies?:\n | {\n [key: string]: (\n attemptsMade: number,\n err: Error,\n strategyOptions?: any\n ) => number;\n }\n | undefined;\n\n /**\n * A timeout for when the queue is in `drained` state (empty waiting for jobs).\n * It is used when calling `queue.getNextJob()`, which will pass it to `.brpoplpush` on the Redis client.\n */\n drainDelay?: number | undefined;\n }\n\n type DoneCallback = (error?: Error | null, value?: any) => void;\n\n type JobId = number | string;\n\n type ProcessCallbackFunction = (job: Job, done: DoneCallback) => void;\n type ProcessPromiseFunction = (job: Job) => Promise;\n\n interface Job {\n id: JobId;\n\n /**\n * The custom data passed when the job was created\n */\n data: T;\n\n /**\n * Options of the job\n */\n opts: JobOptions;\n\n /**\n * How many attempts where made to run this job\n */\n attemptsMade: number;\n\n /**\n * When this job was started (unix milliseconds)\n */\n processedOn?: number | undefined;\n\n /**\n * When this job was completed (unix milliseconds)\n */\n finishedOn?: number | undefined;\n\n /**\n * Which queue this job was part of\n */\n queue: Queue;\n\n timestamp: number;\n\n /**\n * The named processor name\n */\n name: string;\n\n /**\n * The stacktrace for any errors\n */\n stacktrace: string[];\n\n returnvalue: any;\n\n failedReason?: string | undefined;\n\n /**\n * Get progress on a job\n */\n progress(): any;\n\n /**\n * Report progress on a job\n */\n progress(value: any): Promise;\n\n /**\n * Logs one row of log data.\n *\n * @param row String with log data to be logged.\n */\n log(row: string): Promise;\n\n /**\n * Returns a promise resolving to a boolean which, if true, current job's state is completed\n */\n isCompleted(): Promise;\n\n /**\n * Returns a promise resolving to a boolean which, if true, current job's state is failed\n */\n isFailed(): Promise;\n\n /**\n * Returns a promise resolving to a boolean which, if true, current job's state is delayed\n */\n isDelayed(): Promise;\n\n /**\n * Returns a promise resolving to a boolean which, if true, current job's state is active\n */\n isActive(): Promise;\n\n /**\n * Returns a promise resolving to a boolean which, if true, current job's state is wait\n */\n isWaiting(): Promise;\n\n /**\n * Returns a promise resolving to a boolean which, if true, current job's state is paused\n */\n isPaused(): Promise;\n\n /**\n * Returns a promise resolving to a boolean which, if true, current job's state is stuck\n */\n isStuck(): Promise;\n\n /**\n * Returns a promise resolving to the current job's status.\n * Please take note that the implementation of this method is not very efficient, nor is\n * it atomic. If your queue does have a very large quantity of jobs, you may want to\n * avoid using this method.\n */\n getState(): Promise;\n\n /**\n * Update a specific job's data. Promise resolves when the job has been updated.\n */\n update(data: T): Promise;\n\n /**\n * Removes a job from the queue and from any lists it may be included in.\n * The returned promise resolves when the job has been removed.\n */\n remove(): Promise;\n\n /**\n * Re-run a job that has failed. The returned promise resolves when the job\n * has been scheduled for retry.\n */\n retry(): Promise;\n\n /**\n * Ensure this job is never ran again even if attemptsMade is less than job.attempts.\n */\n discard(): Promise;\n\n /**\n * Returns a promise that resolves to the returned data when the job has been finished.\n */\n finished(): Promise;\n\n /**\n * Moves a job to the `completed` queue. Pulls a job from 'waiting' to 'active'\n * and returns a tuple containing the next jobs data and id. If no job is in the `waiting` queue, returns null.\n */\n moveToCompleted(\n returnValue?: string,\n ignoreLock?: boolean,\n notFetch?: boolean\n ): Promise<[any, JobId] | null>;\n\n /**\n * Moves a job to the `failed` queue. Pulls a job from 'waiting' to 'active'\n * and returns a tuple containing the next jobs data and id. If no job is in the `waiting` queue, returns null.\n */\n moveToFailed(\n errorInfo: { message: string },\n ignoreLock?: boolean\n ): Promise<[any, JobId] | null>;\n\n /**\n * Promotes a job that is currently \"delayed\" to the \"waiting\" state and executed as soon as possible.\n */\n promote(): Promise;\n\n /**\n * Return a unique key representing a lock for this Job\n */\n lockKey(): string;\n\n /**\n * Releases the lock on the job. Only locks owned by the queue instance can be released.\n */\n releaseLock(): Promise;\n\n /**\n * Takes a lock for this job so that no other queue worker can process it at the same time.\n */\n takeLock(): Promise;\n\n /**\n * Extend the lock for this job.\n *\n * @param duration lock duration in milliseconds\n */\n extendLock(duration: number): Promise\n\n /**\n * Get job properties as Json Object\n */\n toJSON(): {\n id: JobId;\n name: string;\n data: T;\n opts: JobOptions;\n progress: number;\n delay: number;\n timestamp: number;\n attemptsMade: number;\n failedReason: any;\n stacktrace: string[] | null;\n returnvalue: any;\n finishedOn: number | null;\n processedOn: number | null;\n };\n }\n\n type JobStatus =\n | 'completed'\n | 'waiting'\n | 'active'\n | 'delayed'\n | 'failed'\n | 'paused';\n type JobStatusClean =\n | 'completed'\n | 'wait'\n | 'active'\n | 'delayed'\n | 'failed'\n | 'paused';\n\n interface BackoffOptions {\n /**\n * Backoff type, which can be either `fixed` or `exponential`\n */\n type: string;\n\n /**\n * Backoff delay, in milliseconds\n */\n delay?: number | undefined;\n\n /**\n * Options for custom strategies\n */\n options?: any;\n }\n\n interface RepeatOptions {\n /**\n * Timezone\n */\n tz?: string | undefined;\n\n /**\n * End date when the repeat job should stop repeating\n */\n endDate?: Date | string | number | undefined;\n\n /**\n * Number of times the job should repeat at max.\n */\n limit?: number | undefined;\n\n /**\n * The start value for the repeat iteration count.\n */\n count?: number | undefined;\n }\n\n interface CronRepeatOptions extends RepeatOptions {\n /**\n * Cron pattern specifying when the job should execute\n */\n cron: string;\n\n /**\n * Start date when the repeat job should start repeating (only with cron).\n */\n startDate?: Date | string | number | undefined;\n }\n\n interface EveryRepeatOptions extends RepeatOptions {\n /**\n * Repeat every millis (cron setting cannot be used together with this setting.)\n */\n every: number;\n }\n\n interface DebounceOptions {\n /**\n * ttl in milliseconds\n */\n ttl?: number;\n /**\n * Identifier\n */\n id: string;\n }\n\n interface JobOptions {\n /**\n * Debounce options.\n */\n debounce?: DebounceOptions;\n\n /**\n * Optional priority value. ranges from 1 (highest priority) to MAX_INT (lowest priority).\n * Note that using priorities has a slight impact on performance, so do not use it if not required\n */\n priority?: number | undefined;\n\n /**\n * An amount of miliseconds to wait until this job can be processed.\n * Note that for accurate delays, both server and clients should have their clocks synchronized. [optional]\n */\n delay?: number | undefined;\n\n /**\n * The total number of attempts to try the job until it completes\n */\n attempts?: number | undefined;\n\n /**\n * Repeat job according to a cron specification\n */\n repeat?:\n | ((CronRepeatOptions | EveryRepeatOptions) & {\n /**\n * The key for the repeatable job metadata in Redis.\n */\n readonly key?: string;\n })\n | undefined;\n\n /**\n * Backoff setting for automatic retries if the job fails\n */\n backoff?: number | BackoffOptions | undefined;\n\n /**\n * A boolean which, if true, adds the job to the right\n * of the queue instead of the left (default false)\n */\n lifo?: boolean | undefined;\n\n /**\n * The number of milliseconds after which the job should be fail with a timeout error\n */\n timeout?: number | undefined;\n\n /**\n * Override the job ID - by default, the job ID is a unique\n * integer, but you can use this setting to override it.\n * If you use this option, it is up to you to ensure the\n * jobId is unique. If you attempt to add a job with an id that\n * already exists, it will not be added.\n */\n jobId?: JobId | undefined;\n\n /**\n * A boolean which, if true, removes the job when it successfully completes.\n * When a number, it specifies the amount of jobs to keep.\n * Default behavior is to keep the job in the completed set.\n * See KeepJobsOptions if using that interface instead.\n */\n removeOnComplete?: boolean | number | KeepJobsOptions | undefined;\n\n /**\n * A boolean which, if true, removes the job when it fails after all attempts.\n * When a number, it specifies the amount of jobs to keep.\n * Default behavior is to keep the job in the failed set.\n * See KeepJobsOptions if using that interface instead.\n */\n removeOnFail?: boolean | number | KeepJobsOptions | undefined;\n\n /**\n * Limits the amount of stack trace lines that will be recorded in the stacktrace.\n */\n stackTraceLimit?: number | undefined;\n\n /**\n * Prevents JSON data from being parsed.\n */\n preventParsingData?: boolean | undefined;\n }\n\n /**\n * Specify which jobs to keep after finishing processing this job.\n * If both age and count are specified, then the jobs kept will be the ones that satisfies both properties.\n */\n interface KeepJobsOptions {\n /**\n * Maximum age in *seconds* for job to be kept.\n */\n age?: number | undefined;\n\n /**\n * Maximum count of jobs to be kept.\n */\n count?: number | undefined;\n }\n\n interface JobCounts {\n active: number;\n completed: number;\n failed: number;\n delayed: number;\n waiting: number;\n }\n\n interface JobInformation {\n key: string;\n name: string;\n id?: string | undefined;\n endDate?: number | undefined;\n tz?: string | undefined;\n cron: string;\n every: number;\n next: number;\n }\n\n interface Queue extends EventEmitter {\n /**\n * The name of the queue\n */\n name: string;\n\n /**\n * Queue client (used to add jobs, pause queues, etc);\n */\n client: Redis.Redis;\n\n /**\n * Returns a promise that resolves when Redis is connected and the queue is ready to accept jobs.\n * This replaces the `ready` event emitted on Queue in previous verisons.\n */\n isReady(): Promise;\n\n /* tslint:disable:unified-signatures */\n\n /**\n * Defines a processing function for the jobs placed into a given Queue.\n *\n * The callback is called everytime a job is placed in the queue.\n * It is passed an instance of the job as first argument.\n *\n * If the callback signature contains the second optional done argument,\n * the callback will be passed a done callback to be called after the job has been completed.\n * The done callback can be called with an Error instance, to signal that the job did not complete successfully,\n * or with a result as second argument (e.g.: done(null, result);) when the job is successful.\n * Errors will be passed as a second argument to the \"failed\" event; results, as a second argument to the \"completed\" event.\n *\n * If, however, the callback signature does not contain the done argument,\n * a promise must be returned to signal job completion.\n * If the promise is rejected, the error will be passed as a second argument to the \"failed\" event.\n * If it is resolved, its value will be the \"completed\" event's second argument.\n */\n process(callback: ProcessCallbackFunction): Promise;\n process(callback: ProcessPromiseFunction): Promise;\n process(callback: string): Promise;\n\n /**\n * Defines a processing function for the jobs placed into a given Queue.\n *\n * The callback is called everytime a job is placed in the queue.\n * It is passed an instance of the job as first argument.\n *\n * If the callback signature contains the second optional done argument,\n * the callback will be passed a done callback to be called after the job has been completed.\n * The done callback can be called with an Error instance, to signal that the job did not complete successfully,\n * or with a result as second argument (e.g.: done(null, result);) when the job is successful.\n * Errors will be passed as a second argument to the \"failed\" event; results, as a second argument to the \"completed\" event.\n *\n * If, however, the callback signature does not contain the done argument,\n * a promise must be returned to signal job completion.\n * If the promise is rejected, the error will be passed as a second argument to the \"failed\" event.\n * If it is resolved, its value will be the \"completed\" event's second argument.\n *\n * @param concurrency Bull will then call your handler in parallel respecting this maximum value.\n */\n process(\n concurrency: number,\n callback: ProcessCallbackFunction\n ): Promise;\n process(\n concurrency: number,\n callback: ProcessPromiseFunction\n ): Promise;\n process(concurrency: number, callback: string): Promise;\n\n /**\n * Defines a processing function for the jobs placed into a given Queue.\n *\n * The callback is called everytime a job is placed in the queue.\n * It is passed an instance of the job as first argument.\n *\n * If the callback signature contains the second optional done argument,\n * the callback will be passed a done callback to be called after the job has been completed.\n * The done callback can be called with an Error instance, to signal that the job did not complete successfully,\n * or with a result as second argument (e.g.: done(null, result);) when the job is successful.\n * Errors will be passed as a second argument to the \"failed\" event; results, as a second argument to the \"completed\" event.\n *\n * If, however, the callback signature does not contain the done argument,\n * a promise must be returned to signal job completion.\n * If the promise is rejected, the error will be passed as a second argument to the \"failed\" event.\n * If it is resolved, its value will be the \"completed\" event's second argument.\n *\n * @param name Bull will only call the handler if the job name matches\n */\n process(name: string, callback: ProcessCallbackFunction): Promise;\n process(name: string, callback: ProcessPromiseFunction): Promise;\n process(name: string, callback: string): Promise;\n\n /**\n * Defines a processing function for the jobs placed into a given Queue.\n *\n * The callback is called everytime a job is placed in the queue.\n * It is passed an instance of the job as first argument.\n *\n * If the callback signature contains the second optional done argument,\n * the callback will be passed a done callback to be called after the job has been completed.\n * The done callback can be called with an Error instance, to signal that the job did not complete successfully,\n * or with a result as second argument (e.g.: done(null, result);) when the job is successful.\n * Errors will be passed as a second argument to the \"failed\" event; results, as a second argument to the \"completed\" event.\n *\n * If, however, the callback signature does not contain the done argument,\n * a promise must be returned to signal job completion.\n * If the promise is rejected, the error will be passed as a second argument to the \"failed\" event.\n * If it is resolved, its value will be the \"completed\" event's second argument.\n *\n * @param name Bull will only call the handler if the job name matches\n * @param concurrency Bull will then call your handler in parallel respecting this maximum value.\n */\n process(\n name: string,\n concurrency: number,\n callback: ProcessCallbackFunction\n ): Promise;\n process(\n name: string,\n concurrency: number,\n callback: ProcessPromiseFunction\n ): Promise;\n process(name: string, concurrency: number, callback: string): Promise;\n\n /* tslint:enable:unified-signatures */\n\n /**\n * Creates a new job and adds it to the queue.\n * If the queue is empty the job will be executed directly,\n * otherwise it will be placed in the queue and executed as soon as possible.\n */\n add(data: T, opts?: JobOptions): Promise>;\n\n /**\n * Creates a new named job and adds it to the queue.\n * If the queue is empty the job will be executed directly,\n * otherwise it will be placed in the queue and executed as soon as possible.\n */\n add(name: string, data: T, opts?: JobOptions): Promise>;\n\n /**\n * Adds an array of jobs to the queue.\n * If the queue is empty the jobs will be executed directly,\n * otherwise they will be placed in the queue and executed as soon as possible.\n * 'repeat' option is not supported in addBulk https://github.com/OptimalBits/bull/issues/1731\n */\n addBulk(\n jobs: Array<{\n name?: string | undefined;\n data: T;\n opts?: Omit | undefined;\n }>\n ): Promise>>;\n\n /**\n * Returns a promise that resolves when the queue is paused.\n *\n * A paused queue will not process new jobs until resumed, but current jobs being processed will continue until\n * they are finalized. The pause can be either global or local. If global, all workers in all queue instances\n * for a given queue will be paused. If local, just this worker will stop processing new jobs after the current\n * lock expires. This can be useful to stop a worker from taking new jobs prior to shutting down.\n *\n * If doNotWaitActive is true, pause will not wait for any active jobs to finish before resolving. Otherwise, pause\n * will wait for active jobs to finish. See Queue#whenCurrentJobsFinished for more information.\n *\n * Pausing a queue that is already paused does nothing.\n */\n pause(isLocal?: boolean, doNotWaitActive?: boolean): Promise;\n\n /**\n * Returns a promise that resolves when the queue is resumed after being paused.\n *\n * The resume can be either local or global. If global, all workers in all queue instances for a given queue\n * will be resumed. If local, only this worker will be resumed. Note that resuming a queue globally will not\n * resume workers that have been paused locally; for those, resume(true) must be called directly on their\n * instances.\n *\n * Resuming a queue that is not paused does nothing.\n */\n resume(isLocal?: boolean): Promise;\n\n /**\n * Returns a promise that resolves with a boolean if queue is paused\n */\n isPaused(isLocal?: boolean): Promise;\n\n /**\n * Returns a promise that returns the number of jobs in the queue, waiting or paused.\n * Since there may be other processes adding or processing jobs, this value may be true only for a very small amount of time.\n */\n count(): Promise;\n\n /**\n * Empties a queue deleting all the input lists and associated jobs.\n */\n empty(): Promise;\n\n /**\n * Closes the underlying redis client. Use this to perform a graceful shutdown.\n *\n * `close` can be called from anywhere, with one caveat:\n * if called from within a job handler the queue won't close until after the job has been processed\n */\n close(doNotWaitJobs?: boolean): Promise;\n\n /**\n * Returns the number of jobs per priority.\n */\n getCountsPerPriority(priorities: number[]): Promise<{\n [index: string]: number;\n }>;\n\n /**\n * Returns a promise that will return the job instance associated with the jobId parameter.\n * If the specified job cannot be located, the promise callback parameter will be set to null.\n */\n getJob(jobId: JobId): Promise | null>;\n\n /**\n * Returns a promise that will return an array with the waiting jobs between start and end.\n */\n getWaiting(start?: number, end?: number): Promise>>;\n\n /**\n * Returns a promise that will return an array with the active jobs between start and end.\n */\n getActive(start?: number, end?: number): Promise>>;\n\n /**\n * Returns a promise that will return an array with the delayed jobs between start and end.\n */\n getDelayed(start?: number, end?: number): Promise>>;\n\n /**\n * Returns a promise that will return an array with the completed jobs between start and end.\n */\n getCompleted(start?: number, end?: number): Promise>>;\n\n /**\n * Returns a promise that will return an array with the failed jobs between start and end.\n */\n getFailed(start?: number, end?: number): Promise>>;\n\n /**\n * Returns JobInformation of repeatable jobs (ordered descending). Provide a start and/or an end\n * index to limit the number of results. Start defaults to 0, end to -1 and asc to false.\n */\n getRepeatableJobs(\n start?: number,\n end?: number,\n asc?: boolean\n ): Promise;\n\n /**\n * ???\n */\n nextRepeatableJob(\n name: string,\n data: any,\n opts: JobOptions\n ): Promise>;\n\n /**\n * Removes a given repeatable job. The RepeatOptions and JobId needs to be the same as the ones\n * used for the job when it was added.\n */\n removeRepeatable(\n repeat: (CronRepeatOptions | EveryRepeatOptions) & {\n jobId?: JobId | undefined;\n }\n ): Promise;\n\n /**\n * Removes a given repeatable job. The RepeatOptions and JobId needs to be the same as the ones\n * used for the job when it was added.\n *\n * name: The name of the to be removed job\n */\n removeRepeatable(\n name: string,\n repeat: (CronRepeatOptions | EveryRepeatOptions) & {\n jobId?: JobId | undefined;\n }\n ): Promise;\n\n /**\n * Removes a given repeatable job by key.\n */\n removeRepeatableByKey(key: string): Promise;\n\n /**\n * Returns a promise that will return an array of job instances of the given job statuses.\n * Optional parameters for range and ordering are provided.\n */\n getJobs(\n types: JobStatus[],\n start?: number,\n end?: number,\n asc?: boolean\n ): Promise>>;\n\n /**\n * Returns a promise that resolves to a Metrics object.\n */\n getMetrics(type: 'completed' | 'failed', start?: number, end?: number): Promise<{\n meta: {\n count: number;\n prevTS: number;\n prevCount: number;\n };\n data: number[];\n count: number;\n }>\n\n /**\n * Returns a promise that resolves to the next job in queue.\n */\n getNextJob(): Promise | undefined>;\n\n /**\n * Returns a object with the logs according to the start and end arguments. The returned count\n * value is the total amount of logs, useful for implementing pagination.\n */\n getJobLogs(\n jobId: JobId,\n start?: number,\n end?: number\n ): Promise<{ logs: string[]; count: number }>;\n\n /**\n * Returns a promise that resolves with the job counts for the given queue.\n */\n getJobCounts(): Promise;\n\n /**\n * Returns a promise that resolves with the job counts for the given queue of the given job statuses.\n */\n getJobCountByTypes(types: JobStatus[] | JobStatus): Promise;\n\n /**\n * Returns a promise that resolves with the quantity of completed jobs.\n */\n getCompletedCount(): Promise;\n\n /**\n * Returns a promise that resolves with the quantity of failed jobs.\n */\n getFailedCount(): Promise;\n\n /**\n * Returns a promise that resolves with the quantity of delayed jobs.\n */\n getDelayedCount(): Promise;\n\n /**\n * Returns a promise that resolves with the quantity of waiting jobs.\n */\n getWaitingCount(): Promise;\n\n /**\n * Returns a promise that resolves with the quantity of paused jobs.\n */\n getPausedCount(): Promise;\n\n /**\n * Returns a promise that resolves with the quantity of active jobs.\n */\n getActiveCount(): Promise;\n\n /**\n * Returns a promise that resolves to the quantity of repeatable jobs.\n */\n getRepeatableCount(): Promise;\n\n /**\n * Tells the queue remove all jobs created outside of a grace period in milliseconds.\n * You can clean the jobs with the following states: completed, wait (typo for waiting), active, delayed, and failed.\n * @param grace Grace period in milliseconds.\n * @param status Status of the job to clean. Values are completed, wait, active, delayed, and failed. Defaults to completed.\n * @param limit Maximum amount of jobs to clean per call. If not provided will clean all matching jobs.\n */\n clean(\n grace: number,\n status?: JobStatusClean,\n limit?: number\n ): Promise>>;\n\n /**\n * Returns a promise that resolves to a Metrics object.\n * @param type Job metric type either 'completed' or 'failed'\n * @param start Start point of the metrics, where 0 is the newest point to be returned.\n * @param end End point of the metrics, where -1 is the oldest point to be returned.\n * @returns - Returns an object with queue metrics.\n */\n getMetrics(\n type: 'completed' | 'failed',\n start?: number,\n end?: number\n ): Promise<{\n meta: {\n count: number;\n prevTS: number;\n prevCount: number;\n };\n data: number[];\n count: number;\n }>;\n\n /**\n * Returns a promise that marks the start of a transaction block.\n */\n multi(): Redis.Pipeline;\n\n /**\n * Returns the queue specific key.\n */\n toKey(queueType: string): string;\n\n /**\n * Completely destroys the queue and all of its contents irreversibly.\n * @param ops.force Obliterate the queue even if there are active jobs\n */\n obliterate(ops?: { force: boolean }): Promise;\n\n /**\n * Listens to queue events\n */\n on(event: string, callback: (...args: any[]) => void): this;\n\n /**\n * An error occured\n */\n on(event: 'error', callback: ErrorEventCallback): this;\n\n /**\n * A Job is waiting to be processed as soon as a worker is idling.\n */\n on(event: 'waiting', callback: WaitingEventCallback): this;\n\n /**\n * A job has started. You can use `jobPromise.cancel()` to abort it\n */\n on(event: 'active', callback: ActiveEventCallback): this;\n\n /**\n * A job has been marked as stalled.\n * This is useful for debugging job workers that crash or pause the event loop.\n */\n on(event: 'stalled', callback: StalledEventCallback): this;\n\n /**\n * A job's progress was updated\n */\n on(event: 'progress', callback: ProgressEventCallback): this;\n\n /**\n * A job successfully completed with a `result`\n */\n on(event: 'completed', callback: CompletedEventCallback): this;\n\n /**\n * A job failed with `err` as the reason\n */\n on(event: 'failed', callback: FailedEventCallback): this;\n\n /**\n * The queue has been paused\n */\n on(event: 'paused', callback: EventCallback): this;\n\n /**\n * The queue has been resumed\n */\n on(event: 'resumed', callback: EventCallback): this; // tslint:disable-line unified-signatures\n\n /**\n * A job successfully removed.\n */\n on(event: 'removed', callback: RemovedEventCallback): this;\n\n /**\n * Old jobs have been cleaned from the queue.\n * `jobs` is an array of jobs that were removed, and `type` is the type of those jobs.\n *\n * @see Queue#clean() for details\n */\n on(event: 'cleaned', callback: CleanedEventCallback): this;\n\n /**\n * Emitted every time the queue has processed all the waiting jobs\n * (even if there can be some delayed jobs not yet processed)\n */\n on(event: 'drained', callback: EventCallback): this; // tslint:disable-line unified-signatures\n\n /**\n * Array of Redis clients the queue uses\n */\n clients: Redis.Redis[];\n\n /**\n * Set clientName to Redis.client\n */\n setWorkerName(): Promise;\n\n /**\n * Returns array of workers that are currently working on this queue.\n */\n getWorkers(): Promise<{ [index: string]: string }[]>;\n\n /**\n * Returns Queue name in base64 encoded format\n */\n base64Name(): string;\n\n /**\n * Returns Queue name with keyPrefix (default: 'bull')\n */\n clientName(): string;\n\n /**\n * Returns Redis clients array which belongs to current Queue from string with all redis clients\n *\n * @param list String with all redis clients\n */\n parseClientList(list: string): { [index: string]: string }[][];\n\n /**\n * Returns a promise that resolves when active jobs are finished\n */\n whenCurrentJobsFinished(): Promise;\n\n /**\n * Removes all the jobs which jobId matches the given pattern. The pattern must follow redis glob-style pattern\n * (syntax)[redis.io/commands/keys]\n */\n removeJobs(pattern: string): Promise;\n }\n\n type EventCallback = () => void;\n\n type ErrorEventCallback = (error: Error) => void;\n\n interface JobPromise {\n /**\n * Abort this job\n */\n cancel(): void;\n }\n\n type ActiveEventCallback = (\n job: Job,\n jobPromise?: JobPromise\n ) => void;\n\n type StalledEventCallback = (job: Job) => void;\n\n type ProgressEventCallback = (job: Job, progress: any) => void;\n\n type CompletedEventCallback = (job: Job, result: any) => void;\n\n type FailedEventCallback = (job: Job, error: Error) => void;\n\n type CleanedEventCallback = (\n jobs: Array>,\n status: JobStatusClean\n ) => void;\n\n type RemovedEventCallback = (job: Job) => void;\n\n type WaitingEventCallback = (jobId: JobId) => void;\n}\n\nexport = Bull;\n" }, { "path": "index.js", "content": "'use strict';\n\nmodule.exports = require('./lib/queue');\nmodule.exports.Job = require('./lib/job');\nmodule.exports.utils = require('./lib/utils');\n" }, { "path": "lib/backoffs.js", "content": "'use strict';\n\nconst _ = require('lodash');\n\nconst builtinStrategies = {\n fixed(delay) {\n return function() {\n return delay;\n };\n },\n\n exponential(delay) {\n return function(attemptsMade) {\n return Math.round((Math.pow(2, attemptsMade) - 1) * delay);\n };\n }\n};\n\nfunction lookupStrategy(backoff, customStrategies) {\n if (backoff.type in customStrategies) {\n return customStrategies[backoff.type];\n } else if (backoff.type in builtinStrategies) {\n return builtinStrategies[backoff.type](backoff.delay);\n } else {\n throw new Error(\n 'Unknown backoff strategy ' +\n backoff.type +\n '. If a custom backoff strategy is used, specify it when the queue is created.'\n );\n }\n}\n\nmodule.exports = {\n normalize(backoff) {\n if (_.isFinite(backoff)) {\n return {\n type: 'fixed',\n delay: backoff\n };\n } else if (backoff) {\n return backoff;\n }\n },\n\n calculate(backoff, attemptsMade, customStrategies, err, strategyOptions) {\n if (backoff) {\n const strategy = lookupStrategy(\n backoff,\n customStrategies,\n strategyOptions\n );\n\n return strategy(attemptsMade, err, strategyOptions);\n }\n }\n};\n" }, { "path": "lib/commands/addJob-6.lua", "content": "--[[\n Adds a job to the queue by doing the following:\n - Increases the job counter if needed.\n - Creates a new job key with the job data.\n\n - if delayed:\n - computes timestamp.\n - adds to delayed zset.\n - Emits a global event 'delayed' if the job is delayed.\n - if not delayed\n - Adds the jobId to the wait/paused list in one of three ways:\n - LIFO\n - FIFO\n - prioritized.\n - Adds the job to the \"added\" list so that workers gets notified.\n\n Input:\n KEYS[1] 'wait',\n KEYS[2] 'paused'\n KEYS[3] 'meta-paused'\n KEYS[4] 'id'\n KEYS[5] 'delayed'\n KEYS[6] 'priority'\n\n ARGV[1] key prefix,\n ARGV[2] custom id (will not generate one automatically)\n ARGV[3] name\n ARGV[4] data (json stringified job data)\n ARGV[5] opts (json stringified job opts)\n ARGV[6] timestamp\n ARGV[7] delay\n ARGV[8] delayedTimestamp\n ARGV[9] priority\n ARGV[10] LIFO\n ARGV[11] token\n ARGV[12] debounce key\n ARGV[13] debounceId\n ARGV[14] debounceTtl\n]]\nlocal jobId\nlocal jobIdKey\nlocal rcall = redis.call\n\n-- Includes\n--- @include \"includes/addJobWithPriority\"\n--- @include \"includes/debounceJob\"\n--- @include \"includes/getTargetQueueList\"\n\nlocal jobCounter = rcall(\"INCR\", KEYS[4])\n\nif ARGV[2] == \"\" then\n jobId = jobCounter\n jobIdKey = ARGV[1] .. jobId\nelse\n jobId = ARGV[2]\n jobIdKey = ARGV[1] .. jobId\n if rcall(\"EXISTS\", jobIdKey) == 1 then\n rcall(\"PUBLISH\", ARGV[1] .. \"duplicated@\" .. ARGV[11], jobId)\n return jobId .. \"\" -- convert to string\n end\nend\n\nlocal debounceKey = ARGV[12]\n\nlocal opts = cmsgpack.unpack(ARGV[5])\n\nlocal debouncedJobId = debounceJob(ARGV[1], ARGV[13], ARGV[14],\n jobId, debounceKey, ARGV[11])\nif debouncedJobId then\n return debouncedJobId\nend\n\nlocal debounceId = ARGV[13]\n\nlocal optionalValues = {}\n\nif debounceId ~= \"\" then\n table.insert(optionalValues, \"deid\")\n table.insert(optionalValues, debounceId)\nend\n\n -- Store the job.\nrcall(\"HMSET\", jobIdKey, \"name\", ARGV[3], \"data\", ARGV[4], \"opts\", opts, \"timestamp\",\n ARGV[6], \"delay\", ARGV[7], \"priority\", ARGV[9], unpack(optionalValues))\n\n-- Check if job is delayed\nlocal delayedTimestamp = tonumber(ARGV[8])\nif(delayedTimestamp ~= 0) then\n local timestamp = delayedTimestamp * 0x1000 + bit.band(jobCounter, 0xfff)\n rcall(\"ZADD\", KEYS[5], timestamp, jobId)\n rcall(\"PUBLISH\", KEYS[5], delayedTimestamp)\nelse\n local target\n\n -- Whe check for the meta-paused key to decide if we are paused or not\n -- (since an empty list and !EXISTS are not really the same)\n local target, paused = getTargetQueueList(KEYS[3], KEYS[1], KEYS[2])\n\n -- Standard or priority add\n local priority = tonumber(ARGV[9])\n if priority == 0 then\n -- LIFO or FIFO\n rcall(ARGV[10], target, jobId)\n else\n addJobWithPriority(KEYS[6], priority, jobId, target)\n end\n\n -- Emit waiting event (wait..ing@token)\n rcall(\"PUBLISH\", KEYS[1] .. \"ing@\" .. ARGV[11], jobId)\nend\n\nreturn jobId .. \"\" -- convert to string\n" }, { "path": "lib/commands/addLog-2.lua", "content": "--[[\n Add job log\n\n Input:\n KEYS[1] job id key\n KEYS[2] job logs key\n\n ARGV[1] id\n ARGV[2] log\n ARGV[3] keepLogs\n\n Output:\n -1 - Missing job.\n]]\nlocal rcall = redis.call\n\nif rcall(\"EXISTS\", KEYS[1]) == 1 then -- // Make sure job exists\n local logCount = rcall(\"RPUSH\", KEYS[2], ARGV[2])\n\n if ARGV[3] ~= '' then\n local keepLogs = tonumber(ARGV[3])\n rcall(\"LTRIM\", KEYS[2], -keepLogs, -1)\n\n return math.min(keepLogs, logCount)\n end\n\n return logCount\nelse\n return -1\nend\n" }, { "path": "lib/commands/cleanJobsInSet-3.lua", "content": "--[[\n Remove jobs from the specific set.\n\n Input:\n KEYS[1] set key,\n KEYS[2] priority key\n KEYS[3] rate limiter key\n\n ARGV[1] prefix key\n ARGV[2] maxTimestamp\n ARGV[3] limit the number of jobs to be removed. 0 is unlimited\n ARGV[4] set name, can be any of 'wait', 'active', 'paused', 'delayed', 'completed', or 'failed'\n]]\n\nlocal setKey = KEYS[1]\nlocal priorityKey = KEYS[2]\nlocal rateLimiterKey = KEYS[3]\n\nlocal prefixKey = ARGV[1]\nlocal maxTimestamp = ARGV[2]\nlocal limitStr = ARGV[3]\nlocal setName = ARGV[4]\n\nlocal isList = false\nlocal rcall = redis.call\n\n-- Includes\n--- @include \"includes/removeDebounceKey\"\n\nif setName == \"wait\" or setName == \"active\" or setName == \"paused\" then\n isList = true\nend\n\n-- We use ZRANGEBYSCORE to make the case where we're deleting a limited number\n-- of items in a sorted set only run a single iteration. If we simply used\n-- ZRANGE, we may take a long time traversing through jobs that are within the\n-- grace period.\nlocal function shouldUseZRangeByScore(isList, limit)\n return not isList and limit > 0\nend\n\nlocal function getJobs(setKey, isList, rangeStart, rangeEnd, maxTimestamp, limit)\n if isList then\n return rcall(\"LRANGE\", setKey, rangeStart, rangeEnd)\n elseif shouldUseZRangeByScore(isList, limit) then\n return rcall(\"ZRANGEBYSCORE\", setKey, 0, maxTimestamp, \"LIMIT\", 0, limit)\n else\n return rcall(\"ZRANGE\", setKey, rangeStart, rangeEnd)\n end\nend\n\nlocal limit = tonumber(limitStr)\nlocal rangeStart = 0\nlocal rangeEnd = -1\n\n-- If we're only deleting _n_ items, avoid retrieving all items\n-- for faster performance\n--\n-- Start from the tail of the list, since that's where oldest elements\n-- are generally added for FIFO lists\nif limit > 0 then\n rangeStart = -1 - limit + 1\n rangeEnd = -1\nend\n\nlocal jobIds = getJobs(setKey, isList, rangeStart, rangeEnd, maxTimestamp, limit)\nlocal deleted = {}\nlocal deletedCount = 0\nlocal jobTS\n\n-- Run this loop:\n-- - Once, if limit is -1 or 0\n-- - As many times as needed if limit is positive\nwhile ((limit <= 0 or deletedCount < limit) and next(jobIds, nil) ~= nil) do\n local jobIdsLen = #jobIds\n for i, jobId in ipairs(jobIds) do\n if limit > 0 and deletedCount >= limit then\n break\n end\n\n local jobKey = prefixKey .. jobId\n if (rcall(\"EXISTS\", jobKey .. \":lock\") == 0) then\n -- Find the right timestamp of the job to compare to maxTimestamp:\n -- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed\n -- * processedOn represents when the job was last attempted, but it doesn't get populated until the job is first tried\n -- * timestamp is the original job submission time\n -- Fetch all three of these (in that order) and use the first one that is set so that we'll leave jobs that have been active within the grace period:\n for _, ts in ipairs(rcall(\"HMGET\", jobKey, \"finishedOn\", \"processedOn\", \"timestamp\")) do\n if (ts) then\n jobTS = ts\n break\n end\n end\n if (not jobTS or jobTS < maxTimestamp) then\n if isList then\n -- Job ids can't be the empty string. Use the empty string as a\n -- deletion marker. The actual deletion will occur at the end of the\n -- script.\n rcall(\"LSET\", setKey, rangeEnd - jobIdsLen + i, \"\")\n else\n rcall(\"ZREM\", setKey, jobId)\n end\n rcall(\"ZREM\", priorityKey, jobId)\n\n if setName ~= \"completed\" and setName ~= \"failed\" then\n removeDebounceKey(prefixKey, jobKey)\n end\n\n rcall(\"DEL\", jobKey)\n rcall(\"DEL\", jobKey .. \":logs\")\n\n -- delete keys related to rate limiter\n -- NOTE: this code is unncessary for other sets than wait, paused and delayed.\n local limiterIndexTable = rateLimiterKey .. \":index\"\n local limitedSetKey = rcall(\"HGET\", limiterIndexTable, jobId)\n\n if limitedSetKey then\n rcall(\"SREM\", limitedSetKey, jobId)\n rcall(\"HDEL\", limiterIndexTable, jobId)\n end\n\n deletedCount = deletedCount + 1\n table.insert(deleted, jobId)\n end\n end\n end\n\n -- If we didn't have a limit or used the single-iteration ZRANGEBYSCORE\n -- function, return immediately. We should have deleted all the jobs we can\n if limit <= 0 or shouldUseZRangeByScore(isList, limit) then\n break\n end\n\n if deletedCount < limit then\n -- We didn't delete enough. Look for more to delete\n rangeStart = rangeStart - limit\n rangeEnd = rangeEnd - limit\n jobIds = getJobs(setKey, isList, rangeStart, rangeEnd, maxTimestamp, limit)\n end\nend\n\nif isList then\n rcall(\"LREM\", setKey, 0, \"\")\nend\n\nreturn deleted\n" }, { "path": "lib/commands/extendLock-2.lua", "content": "--[[\n Extend lock and removes the job from the stalled set.\n\n Input:\n KEYS[1] 'lock',\n KEYS[2] 'stalled'\n\n ARGV[1] token\n ARGV[2] lock duration in milliseconds\n ARGV[3] jobid\n\n Output:\n \"1\" if lock extended succesfully.\n]]\nlocal rcall = redis.call\nif rcall(\"GET\", KEYS[1]) == ARGV[1] then\n if rcall(\"SET\", KEYS[1], ARGV[1], \"PX\", ARGV[2]) then\n rcall(\"SREM\", KEYS[2], ARGV[3])\n return 1\n end\nend\nreturn 0\n" }, { "path": "lib/commands/getCountsPerPriority-4.lua", "content": "--[[\n Get counts per provided states\n\n Input:\n KEYS[1] wait key\n KEYS[2] paused key\n KEYS[3] meta-paused key\n KEYS[4] priority key\n\n ARGV[1...] priorities\n]]\nlocal rcall = redis.call\nlocal results = {}\nlocal prioritizedKey = KEYS[4]\n\n-- Includes\n--- @include \"includes/getTargetQueueList\"\n\nfor i = 1, #ARGV do\n local priority = tonumber(ARGV[i])\n if priority == 0 then\n local target = getTargetQueueList(KEYS[3], KEYS[1], KEYS[2])\n local count = rcall(\"LLEN\", target) - rcall(\"ZCARD\", prioritizedKey)\n if count < 0 then\n -- considering when last waiting job is moved to active before\n -- removing priority reference\n results[#results+1] = 0\n else\n results[#results+1] = count\n end\n else\n results[#results+1] = rcall(\"ZCOUNT\", prioritizedKey,\n priority, priority)\n end\nend\n\nreturn results\n" }, { "path": "lib/commands/includes/addJobWithPriority.lua", "content": "--[[\n Function to add job considering priority.\n]]\n\nlocal function addJobWithPriority(priorityKey, priority, jobId, targetKey)\n rcall(\"ZADD\", priorityKey, priority, jobId)\n local count = rcall(\"ZCOUNT\", priorityKey, 0, priority)\n\n local len = rcall(\"LLEN\", targetKey)\n local id = rcall(\"LINDEX\", targetKey, len - (count - 1))\n if id then\n rcall(\"LINSERT\", targetKey, \"BEFORE\", id, jobId)\n else\n rcall(\"RPUSH\", targetKey, jobId)\n end\nend\n" }, { "path": "lib/commands/includes/batches.lua", "content": "--[[\n Function to loop in batches.\n Just a bit of warning, some commands as ZREM\n could receive a maximum of 7000 parameters per call.\n]]\n\nlocal function batches(n, batchSize)\n local i = 0\n\n return function()\n local from = i * batchSize + 1\n i = i + 1\n if (from <= n) then\n local to = math.min(from + batchSize - 1, n)\n return from, to\n end\n end\nend\n" }, { "path": "lib/commands/includes/collectMetrics.lua", "content": "--[[\n Functions to collect metrics based on a current and previous count of jobs.\n Granualarity is fixed at 1 minute.\n]]\n\n-- Includes\n--- @include \"batches\"\n\nlocal function collectMetrics(metaKey, dataPointsList, maxDataPoints, timestamp)\n -- Increment current count\n local count = rcall(\"HINCRBY\", metaKey, \"count\", 1) - 1\n\n -- Compute how many data points we need to add to the list, N.\n local prevTS = rcall(\"HGET\", metaKey, \"prevTS\")\n\n if not prevTS then\n -- If prevTS is nil, set it to the current timestamp\n rcall(\"HSET\", metaKey, \"prevTS\", timestamp, \"prevCount\", 0)\n return\n end\n\n local N = math.min(math.floor(timestamp / 60000) - math.floor(prevTS / 60000), tonumber(maxDataPoints))\n\n if N > 0 then\n local delta = count - rcall(\"HGET\", metaKey, \"prevCount\")\n -- If N > 1, add N-1 zeros to the list\n if N > 1 then\n local points = {}\n points[1] = delta\n for i = 2, N do points[i] = 0 end\n\n for from, to in batches(#points, 7000) do\n rcall(\"LPUSH\", dataPointsList, unpack(points, from, to))\n end\n else\n -- LPUSH delta to the list\n rcall(\"LPUSH\", dataPointsList, delta)\n end\n\n -- LTRIM to keep list to its max size\n rcall(\"LTRIM\", dataPointsList, 0, maxDataPoints - 1)\n\n -- update prev count with current count\n rcall(\"HSET\", metaKey, \"prevCount\", count, \"prevTS\", timestamp)\n end\nend\n" }, { "path": "lib/commands/includes/debounceJob.lua", "content": "--[[\n Function to debounce a job.\n]]\n\nlocal function debounceJob(prefixKey, debounceId, ttl, jobId, debounceKey, token)\n if debounceId ~= \"\" then\n local debounceKeyExists\n if ttl ~= \"\" then\n debounceKeyExists = not rcall('SET', debounceKey, jobId, 'PX', ttl, 'NX')\n else\n debounceKeyExists = not rcall('SET', debounceKey, jobId, 'NX')\n end\n if debounceKeyExists then\n local currentDebounceJobId = rcall('GET', debounceKey)\n rcall(\"PUBLISH\", prefixKey .. \"debounced@\" .. token, currentDebounceJobId)\n\n return currentDebounceJobId\n end\n end\nend" }, { "path": "lib/commands/includes/getTargetQueueList.lua", "content": "--[[\n Function to check for the meta.paused key to decide if we are paused or not\n (since an empty list and !EXISTS are not really the same).\n]]\n\nlocal function getTargetQueueList(queueMetaKey, waitKey, pausedKey)\n if rcall(\"EXISTS\", queueMetaKey) ~= 1 then\n return waitKey, false\n else\n return pausedKey, true\n end\nend\n" }, { "path": "lib/commands/includes/removeDebounceKey.lua", "content": "\n--[[\n Function to remove debounce key.\n]]\n\nlocal function removeDebounceKey(prefixKey, jobKey)\n local debounceId = rcall(\"HGET\", jobKey, \"deid\")\n if debounceId then\n local debounceKey = prefixKey .. \"de:\" .. debounceId\n rcall(\"DEL\", debounceKey)\n end\nend\n" }, { "path": "lib/commands/includes/removeDebounceKeyIfNeeded.lua", "content": "--[[\n Function to remove debounce key if needed.\n]]\n\nlocal function removeDebounceKeyIfNeeded(prefixKey, debounceId)\n if debounceId then\n local debounceKey = prefixKey .. \"de:\" .. debounceId\n local pttl = rcall(\"PTTL\", debounceKey)\n\n if pttl == 0 or pttl == -1 then\n rcall(\"DEL\", debounceKey)\n end\n end\nend\n" }, { "path": "lib/commands/includes/removeLock.lua", "content": "local function removeLock(jobKey, stalledKey, token, jobId)\n if token ~= \"0\" then\n local lockKey = jobKey .. ':lock'\n local lockToken = rcall(\"GET\", lockKey)\n if lockToken == token then\n rcall(\"DEL\", lockKey)\n rcall(\"SREM\", stalledKey, jobId)\n else\n if lockToken then\n -- Lock exists but token does not match\n return -6\n else\n -- Lock is missing completely\n return -2\n end\n end\n end\n return 0\nend\n" }, { "path": "lib/commands/index.js", "content": "'use strict';\nconst { ScriptLoader } = require('./script-loader');\n\nconst scriptLoader = new ScriptLoader();\n\nmodule.exports = {\n ScriptLoader,\n scriptLoader\n};\n" }, { "path": "lib/commands/isFinished-2.lua", "content": "--[[\n Checks if a job is finished (.i.e. is in the completed or failed set)\n\n Input: \n KEYS[1] completed key\n KEYS[2] failed key\n\n ARGV[1] job id\n Output:\n 0 - not finished.\n 1 - completed.\n 2 - failed.\n]]\nif redis.call(\"ZSCORE\", KEYS[1], ARGV[1]) ~= false then\n return 1\nend\n\nif redis.call(\"ZSCORE\", KEYS[2], ARGV[1]) ~= false then\n return 2\nend\n\nreturn redis.call(\"ZSCORE\", KEYS[2], ARGV[1])\n" }, { "path": "lib/commands/isJobInList-1.lua", "content": "--[[\n Checks if job is in a given list.\n\n Input:\n KEYS[1]\n ARGV[1]\n\n Output:\n 1 if element found in the list.\n]]\nlocal function item_in_list (list, item)\n for _, v in pairs(list) do\n if v == item then\n return 1\n end\n end\n return nil\nend\nlocal items = redis.call(\"LRANGE\", KEYS[1] , 0, -1)\nreturn item_in_list(items, ARGV[1])\n" }, { "path": "lib/commands/moveStalledJobsToWait-7.lua", "content": "--[[\n Move stalled jobs to wait.\n\n Input:\n KEYS[1] 'stalled' (SET)\n KEYS[2] 'wait', (LIST)\n KEYS[3] 'active', (LIST)\n KEYS[4] 'failed', (ZSET)\n KEYS[5] 'stalled-check', (KEY)\n\n KEYS[6] 'meta-paused', (KEY)\n KEYS[7] 'paused', (LIST)\n\n ARGV[1] Max stalled job count\n ARGV[2] queue.toKey('')\n ARGV[3] timestamp\n ARGV[4] max check time\n\n Events:\n 'stalled' with stalled job id.\n]]\n\nlocal rcall = redis.call\n\n-- Includes\n--- @include \"includes/batches\"\n--- @include \"includes/getTargetQueueList\"\n--- @include \"includes/removeDebounceKeyIfNeeded\"\n\nlocal function removeJob(jobId, baseKey)\n local jobKey = baseKey .. jobId\n rcall(\"DEL\", jobKey, jobKey .. ':logs')\nend\n\nlocal function removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)\n local start = timestamp - maxAge * 1000\n local jobIds = rcall(\"ZREVRANGEBYSCORE\", targetSet, start, \"-inf\")\n for i, jobId in ipairs(jobIds) do\n removeJob(jobId, prefix)\n end\n rcall(\"ZREMRANGEBYSCORE\", targetSet, \"-inf\", start)\nend\n\nlocal function removeJobsByMaxCount(maxCount, targetSet, prefix)\n local start = maxCount\n local jobIds = rcall(\"ZREVRANGE\", targetSet, start, -1)\n for i, jobId in ipairs(jobIds) do\n removeJob(jobId, prefix)\n end\n rcall(\"ZREMRANGEBYRANK\", targetSet, 0, -(maxCount + 1))\nend\n\n-- Check if we need to check for stalled jobs now.\nif rcall(\"EXISTS\", KEYS[5]) == 1 then\n return {{}, {}}\nend\n\nrcall(\"SET\", KEYS[5], ARGV[3], \"PX\", ARGV[4])\n\n-- Move all stalled jobs to wait\nlocal stalling = rcall('SMEMBERS', KEYS[1])\nlocal stalled = {}\nlocal failed = {}\nif(#stalling > 0) then\n rcall('DEL', KEYS[1])\n\n local MAX_STALLED_JOB_COUNT = tonumber(ARGV[1])\n\n -- Remove from active list\n for i, jobId in ipairs(stalling) do\n local jobKey = ARGV[2] .. jobId\n\n -- Check that the lock is also missing, then we can handle this job as really stalled.\n if(rcall(\"EXISTS\", jobKey .. \":lock\") == 0) then\n -- Remove from the active queue.\n local removed = rcall(\"LREM\", KEYS[3], 1, jobId)\n\n if(removed > 0) then\n -- If this job has been stalled too many times, such as if it crashes the worker, then fail it.\n local stalledCount = rcall(\"HINCRBY\", jobKey, \"stalledCounter\", 1)\n if(stalledCount > MAX_STALLED_JOB_COUNT) then\n local jobAttributes = rcall(\"HMGET\", jobKey, \"opts\", \"deid\")\n local opts = cjson.decode(jobAttributes[1])\n local removeOnFailType = type(opts[\"removeOnFail\"])\n rcall(\"ZADD\", KEYS[4], ARGV[3], jobId)\n rcall(\"HMSET\", jobKey, \"failedReason\", \"job stalled more than allowable limit\",\n \"finishedOn\", ARGV[3])\n removeDebounceKeyIfNeeded(ARGV[2], jobAttributes[2])\n rcall(\"PUBLISH\", KEYS[4], '{\"jobId\":\"' .. jobId .. '\", \"val\": \"job stalled more than maxStalledCount\"}')\n\n if removeOnFailType == \"number\" then\n removeJobsByMaxCount(opts[\"removeOnFail\"],\n KEYS[4], ARGV[2])\n elseif removeOnFailType == \"boolean\" then\n if opts[\"removeOnFail\"] then\n removeJob(jobId, ARGV[2])\n rcall(\"ZREM\", KEYS[4], jobId)\n end\n elseif removeOnFailType ~= \"nil\" then\n local maxAge = opts[\"removeOnFail\"][\"age\"]\n local maxCount = opts[\"removeOnFail\"][\"count\"]\n\n if maxAge ~= nil then\n removeJobsByMaxAge(ARGV[3], maxAge,\n KEYS[4], ARGV[2])\n end\n\n if maxCount ~= nil and maxCount > 0 then\n removeJobsByMaxCount(maxCount, KEYS[4],\n ARGV[2])\n end\n end\n\n table.insert(failed, jobId)\n else\n local target = getTargetQueueList(KEYS[6], KEYS[2], KEYS[7])\n\n -- Move the job back to the wait queue, to immediately be picked up by a waiting worker.\n rcall(\"RPUSH\", target, jobId)\n rcall('PUBLISH', KEYS[1] .. '@', jobId)\n table.insert(stalled, jobId)\n end\n end\n end\n end\nend\n\n-- Mark potentially stalled jobs\nlocal active = rcall('LRANGE', KEYS[3], 0, -1)\n\nif (#active > 0) then\n for from, to in batches(#active, 7000) do\n rcall('SADD', KEYS[1], unpack(active, from, to))\n end\nend\n\nreturn {failed, stalled}\n" }, { "path": "lib/commands/moveToActive-8.lua", "content": "--[[\n Move next job to be processed to active, lock it and fetch its data. The job\n may be delayed, in that case we need to move it to the delayed set instead.\n\n This operation guarantees that the worker owns the job during the locks\n expiration time. The worker is responsible of keeping the lock fresh\n so that no other worker picks this job again.\n\n Input:\n KEYS[1] wait key\n KEYS[2] active key\n KEYS[3] priority key\n KEYS[4] active event key\n KEYS[5] stalled key\n\n -- Rate limiting\n KEYS[6] rate limiter key\n KEYS[7] delayed key\n\n --\n KEYS[8] drained key\n\n ARGV[1] key prefix\n ARGV[2] lock token\n ARGV[3] lock duration in milliseconds\n ARGV[4] timestamp\n ARGV[5] optional jobid\n\n ARGV[6] optional jobs per time unit (rate limiter)\n ARGV[7] optional time unit (rate limiter)\n ARGV[8] optional do not do anything with job if rate limit hit\n ARGV[9] optional rate limit by key\n]]\n\nlocal rcall = redis.call\n\nlocal rateLimit = function(jobId, maxJobs)\n local rateLimiterKey = KEYS[6];\n local limiterIndexTable = rateLimiterKey .. \":index\"\n\n -- Rate limit by group?\n if(ARGV[9]) then\n local group = string.match(jobId, \"[^:]+$\")\n if group ~= nil then\n rateLimiterKey = rateLimiterKey .. \":\" .. group\n end\n end\n\n -- -- key for storing rate limited jobs\n -- When a job has been previously rate limited it should be part of this set\n -- if the job is back here means that the delay time for this job has passed and now we should\n -- be able to process it again.\n local limitedSetKey = rateLimiterKey .. \":limited\"\n local delay = 0\n\n -- -- Check if job was already limited\n local isLimited = rcall(\"SISMEMBER\", limitedSetKey, jobId);\n\n if isLimited == 1 then\n -- Remove from limited zset since we are going to try to process it\n rcall(\"SREM\", limitedSetKey, jobId)\n rcall(\"HDEL\", limiterIndexTable, jobId)\n else\n -- If not, check if there are any limited jobs\n -- If the job has not been rate limited, we should check if there are any other rate limited jobs, because if that\n -- is the case we do not want to process this job, just calculate a delay for it and put it to \"sleep\".\n local numLimitedJobs = rcall(\"SCARD\", limitedSetKey)\n\n if numLimitedJobs > 0 then\n -- Note, add some slack to compensate for drift.\n delay = ((numLimitedJobs * ARGV[7] * 1.1) / maxJobs) + tonumber(rcall(\"PTTL\", rateLimiterKey))\n end\n end\n\n local jobCounter = tonumber(rcall(\"GET\", rateLimiterKey))\n if(jobCounter == nil) then\n jobCounter = 0\n end\n -- check if rate limit hit\n if (delay == 0) and (jobCounter >= maxJobs) then\n -- Seems like there are no current rated limited jobs, but the jobCounter has exceeded the number of jobs for this unit of time so we need to rate limit this job.\n local exceedingJobs = jobCounter - maxJobs\n delay = tonumber(rcall(\"PTTL\", rateLimiterKey)) + ((exceedingJobs) * ARGV[7]) / maxJobs\n end\n\n if delay > 0 then\n local bounceBack = ARGV[8]\n if bounceBack == 'false' then\n local timestamp = delay + tonumber(ARGV[4])\n -- put job into delayed queue\n rcall(\"ZADD\", KEYS[7], timestamp * 0x1000 + bit.band(jobCounter, 0xfff), jobId)\n rcall(\"PUBLISH\", KEYS[7], timestamp)\n rcall(\"SADD\", limitedSetKey, jobId)\n\n -- store index so that we can delete rate limited data\n rcall(\"HSET\", limiterIndexTable, jobId, limitedSetKey)\n\n end\n\n -- remove from active queue\n rcall(\"LREM\", KEYS[2], 1, jobId)\n return true\n else\n -- false indicates not rate limited\n -- increment jobCounter only when a job is not rate limited\n if (jobCounter == 0) then\n rcall(\"PSETEX\", rateLimiterKey, ARGV[7], 1)\n else\n rcall(\"INCR\", rateLimiterKey)\n end\n return false\n end\nend\n\nlocal jobId = ARGV[5]\n\nif jobId ~= '' then\n -- clean stalled key\n rcall(\"SREM\", KEYS[5], jobId)\nelse\n -- move from wait to active\n jobId = rcall(\"RPOPLPUSH\", KEYS[1], KEYS[2])\nend\n\nif jobId then\n -- Check if we need to perform rate limiting.\n local maxJobs = tonumber(ARGV[6])\n\n if maxJobs then\n if rateLimit(jobId, maxJobs) then\n return\n end\n end\n\n -- get a lock\n local jobKey = ARGV[1] .. jobId\n local lockKey = jobKey .. ':lock'\n rcall(\"SET\", lockKey, ARGV[2], \"PX\", ARGV[3])\n\n -- remove from priority\n rcall(\"ZREM\", KEYS[3], jobId)\n rcall(\"PUBLISH\", KEYS[4], jobId)\n rcall(\"HSET\", jobKey, \"processedOn\", ARGV[4])\n\n return {rcall(\"HGETALL\", jobKey), jobId} -- get job data\nelse\n rcall(\"PUBLISH\", KEYS[8], \"\")\nend\n\n" }, { "path": "lib/commands/moveToDelayed-4.lua", "content": "--[[\n Moves job from active to delayed set.\n\n Input:\n KEYS[1] active key\n KEYS[2] delayed key\n KEYS[3] job key\n KEYS[4] stalled key\n\n ARGV[1] delayedTimestamp\n ARGV[2] the id of the job\n ARGV[3] queue token\n\n Output:\n 0 - OK\n -1 - Missing job.\n -2 - Job is locked.\n\n Events:\n - delayed key.\n]]\nlocal rcall = redis.call\n\n-- Includes\n--- @include \"includes/removeLock\"\n\nif rcall(\"EXISTS\", KEYS[3]) == 1 then\n local errorCode = removeLock(KEYS[3], KEYS[4], ARGV[3], ARGV[2])\n if errorCode < 0 then\n return errorCode\n end\n\n local numRemovedElements = rcall(\"LREM\", KEYS[1], -1, ARGV[2])\n if numRemovedElements < 1 then return -3 end\n\n local score = tonumber(ARGV[1])\n rcall(\"ZADD\", KEYS[2], score, ARGV[2])\n rcall(\"PUBLISH\", KEYS[2], (score / 0x1000))\n\n return 0\nelse\n return -1\nend\n" }, { "path": "lib/commands/moveToFinished-9.lua", "content": "--[[\n Move job from active to a finished status (completed or failed)\n A job can only be moved to completed if it was active.\n The job must be locked before it can be moved to a finished status,\n and the lock must be released in this script.\n\n Input:\n KEYS[1] active key\n KEYS[2] completed/failed key\n KEYS[3] jobId key\n\n KEYS[4] wait key\n KEYS[5] priority key\n KEYS[6] active event key\n\n KEYS[7] delayed key\n KEYS[8] stalled key\n\n KEYS[9] metrics key\n\n ARGV[1] jobId\n ARGV[2] timestamp\n ARGV[3] msg property\n ARGV[4] return value / failed reason\n ARGV[5] token\n ARGV[6] shouldRemove\n ARGV[7] event data (? maybe just send jobid).\n ARGV[8] should fetch next job\n ARGV[9] base key\n ARGV[10] lock token\n ARGV[11] lock duration in milliseconds\n ARGV[12] maxMetricsSize\n\n Output:\n 0 OK\n -1 Missing key.\n -2 Missing lock.\n -3 - Job not in active set.\n\n Events:\n 'completed/failed'\n]]\nlocal rcall = redis.call\n\n-- Includes\n--- @include \"includes/collectMetrics\"\n--- @include \"includes/removeLock\"\n--- @include \"includes/removeDebounceKeyIfNeeded\"\n\nif rcall(\"EXISTS\", KEYS[3]) == 1 then -- // Make sure job exists\n local errorCode = removeLock(KEYS[3], KEYS[8], ARGV[5], ARGV[1])\n if errorCode < 0 then\n return errorCode\n end\n\n -- Remove from active list (if not active we shall return error)\n local numRemovedElements = rcall(\"LREM\", KEYS[1], -1, ARGV[1])\n\n if numRemovedElements < 1 then return -3 end\n\n local debounceId = rcall(\"HGET\", KEYS[3], \"deid\")\n removeDebounceKeyIfNeeded(ARGV[9], debounceId)\n\n -- Remove job?\n local keepJobs = cmsgpack.unpack(ARGV[6])\n local maxCount = keepJobs['count']\n local maxAge = keepJobs['age']\n local targetSet = KEYS[2]\n local timestamp = ARGV[2]\n\n if maxCount ~= 0 then\n\n -- Add to complete/failed set\n rcall(\"ZADD\", targetSet, timestamp, ARGV[1])\n rcall(\"HMSET\", KEYS[3], ARGV[3], ARGV[4], \"finishedOn\", timestamp) -- \"returnvalue\" / \"failedReason\" and \"finishedOn\"\n\n local function removeJobs(jobIds)\n for i, jobId in ipairs(jobIds) do\n local jobKey = ARGV[9] .. jobId\n local jobLogKey = jobKey .. ':logs'\n rcall(\"DEL\", jobKey, jobLogKey)\n end\n end\n\n -- Remove old jobs?\n if maxAge ~= nil then\n local start = timestamp - maxAge * 1000\n local jobIds = rcall(\"ZREVRANGEBYSCORE\", targetSet, start, \"-inf\")\n removeJobs(jobIds)\n rcall(\"ZREMRANGEBYSCORE\", targetSet, \"-inf\", start)\n end\n\n if maxCount ~= nil and maxCount > 0 then\n local start = maxCount\n local jobIds = rcall(\"ZREVRANGE\", targetSet, start, -1)\n removeJobs(jobIds)\n rcall(\"ZREMRANGEBYRANK\", targetSet, 0, -(maxCount + 1));\n end\n else\n local jobLogKey = KEYS[3] .. ':logs'\n rcall(\"DEL\", KEYS[3], jobLogKey)\n end\n\n -- Collect metrics\n if ARGV[12] ~= \"\" then\n collectMetrics(KEYS[9], KEYS[9]..':data', ARGV[12], timestamp)\n end\n\n rcall(\"PUBLISH\", targetSet, ARGV[7])\n\n -- Try to get next job to avoid an extra roundtrip if the queue is not closing, \n -- and not rate limited.\n if (ARGV[8] == \"1\") then\n -- move from wait to active \n local jobId = rcall(\"RPOPLPUSH\", KEYS[4], KEYS[1])\n if jobId then\n local jobKey = ARGV[9] .. jobId\n local lockKey = jobKey .. ':lock'\n\n -- get a lock\n rcall(\"SET\", lockKey, ARGV[11], \"PX\", ARGV[10])\n\n rcall(\"ZREM\", KEYS[5], jobId) -- remove from priority\n rcall(\"PUBLISH\", KEYS[6], jobId)\n rcall(\"HSET\", jobKey, \"processedOn\", ARGV[2])\n\n return {rcall(\"HGETALL\", jobKey), jobId} -- get job data\n end\n end\n\n return 0\nelse\n return -1\nend\n" }, { "path": "lib/commands/obliterate-2.lua", "content": "--[[\n Completely obliterates a queue and all of its contents\n Input:\n\n KEYS[1] meta-paused\n KEYS[2] base\n \n ARGV[1] count\n ARGV[2] force\n]] \n-- This command completely destroys a queue including all of its jobs, current or past \n-- leaving no trace of its existence. Since this script needs to iterate to find all the job\n-- keys, consider that this call may be slow for very large queues.\n-- The queue needs to be \"paused\" or it will return an error\n-- If the queue has currently active jobs then the script by default will return error,\n-- however this behaviour can be overrided using the 'force' option.\nlocal maxCount = tonumber(ARGV[1])\nlocal baseKey = KEYS[2]\n\nlocal rcall = redis.call\n\n-- Includes\n--- @include \"includes/removeDebounceKey\"\n\nlocal function getListItems(keyName, max)\n return rcall('LRANGE', keyName, 0, max - 1)\nend\n\nlocal function getZSetItems(keyName, max)\n return rcall('ZRANGE', keyName, 0, max - 1)\nend\n\nlocal function removeJobs(baseKey, keys)\n for i, key in ipairs(keys) do\n local jobKey = baseKey .. key\n rcall(\"DEL\", jobKey, jobKey .. ':logs')\n removeDebounceKey(baseKey, jobKey)\n end\n maxCount = maxCount - #keys\nend\n\nlocal function removeListJobs(keyName, max)\n local jobs = getListItems(keyName, max)\n removeJobs(baseKey, jobs)\n rcall(\"LTRIM\", keyName, #jobs, -1)\nend\n\nlocal function removeZSetJobs(keyName, max)\n local jobs = getZSetItems(keyName, max)\n removeJobs(baseKey, jobs)\n if (#jobs > 0) then rcall(\"ZREM\", keyName, unpack(jobs)) end\nend\n\nlocal function removeLockKeys(keys)\n for i, key in ipairs(keys) do rcall(\"DEL\", baseKey .. key .. ':lock') end\nend\n\n-- 1) Check if paused, if not return with error.\nif rcall(\"EXISTS\", KEYS[1]) ~= 1 then\n return -1 -- Error, NotPaused\nend\n\n-- 2) Check if there are active jobs, if there are and not \"force\" return error.\nlocal activeKey = baseKey .. 'active'\nlocal activeJobs = getListItems(activeKey, maxCount)\nif (#activeJobs > 0) then\n if (ARGV[2] == \"\") then\n return -2 -- Error, ExistsActiveJobs\n end\nend\n\nremoveLockKeys(activeJobs)\nremoveJobs(baseKey, activeJobs)\nrcall(\"LTRIM\", activeKey, #activeJobs, -1)\nif (maxCount <= 0) then return 1 end\n\nlocal waitKey = baseKey .. 'paused'\nremoveListJobs(waitKey, maxCount)\nif (maxCount <= 0) then return 1 end\n\nlocal delayedKey = baseKey .. 'delayed'\nremoveZSetJobs(delayedKey, maxCount)\nif (maxCount <= 0) then return 1 end\n\nlocal completedKey = baseKey .. 'completed'\nremoveZSetJobs(completedKey, maxCount)\nif (maxCount <= 0) then return 1 end\n\nlocal failedKey = baseKey .. 'failed'\nremoveZSetJobs(failedKey, maxCount)\nif (maxCount <= 0) then return 1 end\n\nif (maxCount > 0) then\n rcall(\"DEL\", baseKey .. 'priority')\n rcall(\"DEL\", baseKey .. 'stalled-check')\n rcall(\"DEL\", baseKey .. 'stalled')\n rcall(\"DEL\", baseKey .. 'meta-paused')\n rcall(\"DEL\", baseKey .. 'meta')\n rcall(\"DEL\", baseKey .. 'id')\n rcall(\"DEL\", baseKey .. 'repeat')\n rcall(\"DEL\", baseKey .. 'metrics:completed')\n rcall(\"DEL\", baseKey .. 'metrics:completed:data')\n rcall(\"DEL\", baseKey .. 'metrics:failed')\n rcall(\"DEL\", baseKey .. 'metrics:failed:data')\n return 0\nelse\n return 1\nend\n" }, { "path": "lib/commands/pause-5.lua", "content": "--[[\n Pauses or resumes a queue globably.\n\n Input:\n KEYS[1] 'wait' or 'paused''\n KEYS[2] 'paused' or 'wait'\n KEYS[3] 'meta-paused'\n KEYS[4] 'paused' o 'resumed' event.\n KEYS[5] 'meta' this key is only used in BullMQ and above.\n\n ARGV[1] 'paused' or 'resumed'\n\n Event:\n publish paused or resumed event.\n]]\nlocal rcall = redis.call\n\nif rcall(\"EXISTS\", KEYS[1]) == 1 then\n rcall(\"RENAME\", KEYS[1], KEYS[2])\nend\n\nif ARGV[1] == \"paused\" then\n rcall(\"SET\", KEYS[3], 1)\n\n -- for forwards compatibility\n rcall(\"HSET\", KEYS[5], \"paused\", 1)\nelse\n rcall(\"DEL\", KEYS[3])\n\n -- for forwards compatibility\n rcall(\"HDEL\", KEYS[5], \"paused\")\n\nend\n\nrcall(\"PUBLISH\", KEYS[4], ARGV[1])\n" }, { "path": "lib/commands/promote-5.lua", "content": "--[[\n Promotes a job that is currently \"delayed\" to the \"waiting\" state\n\n Input:\n KEYS[1] 'delayed'\n KEYS[2] 'wait'\n KEYS[3] 'paused'\n KEYS[4] 'meta-paused'\n KEYS[5] 'priority'\n\n ARGV[1] queue.toKey('')\n ARGV[2] jobId\n ARGV[3] queue token\n\n Events:\n 'waiting'\n]]\nlocal rcall = redis.call;\nlocal jobId = ARGV[2]\n\n-- Includes\n--- @include \"includes/addJobWithPriority\"\n--- @include \"includes/getTargetQueueList\"\n\nif rcall(\"ZREM\", KEYS[1], jobId) == 1 then\n local priority = tonumber(rcall(\"HGET\", ARGV[1] .. jobId, \"priority\")) or 0\n\n local target = getTargetQueueList(KEYS[4], KEYS[2], KEYS[3])\n\n if priority == 0 then\n -- LIFO or FIFO\n rcall(\"LPUSH\", target, jobId)\n else\n addJobWithPriority(KEYS[5], priority, jobId, target)\n end\n\n -- Emit waiting event (wait..ing@token)\n rcall(\"PUBLISH\", KEYS[2] .. \"ing@\" .. ARGV[3], jobId)\n\n rcall(\"HSET\", ARGV[1] .. jobId, \"delay\", 0)\n\n return 0\nelse\n return -1\nend\n" }, { "path": "lib/commands/releaseLock-1.lua", "content": "--[[\n Release lock\n\n Input:\n KEYS[1] 'lock',\n \n ARGV[1] token\n ARGV[2] lock duration in milliseconds\n \n Output:\n \"OK\" if lock extented succesfully.\n]]\nlocal rcall = redis.call\n\nif rcall(\"GET\", KEYS[1]) == ARGV[1] then\n return rcall(\"DEL\", KEYS[1])\nelse\n return 0\nend\n" }, { "path": "lib/commands/removeJob-11.lua", "content": "--[[\n Remove a job from all the queues it may be in as well as all its data.\n In order to be able to remove a job, it must be unlocked.\n\n Input:\n KEYS[1] 'active',\n KEYS[2] 'wait',\n KEYS[3] 'delayed',\n KEYS[4] 'paused',\n KEYS[5] 'completed',\n KEYS[6] 'failed',\n KEYS[7] 'priority',\n KEYS[8] jobId key\n KEYS[9] job logs\n KEYS[10] rate limiter index table\n KEYS[11] prefix key\n\n ARGV[1] jobId\n ARGV[2] lock token\n\n Events:\n 'removed'\n]]\n\n-- TODO PUBLISH global event 'removed'\n\nlocal rcall = redis.call\n\n-- Includes\n--- @include \"includes/removeDebounceKey\"\n\nlocal lockKey = KEYS[8] .. ':lock'\nlocal lock = rcall(\"GET\", lockKey)\nif not lock then -- or (lock == ARGV[2])) then\n local jobId = ARGV[1]\n rcall(\"LREM\", KEYS[1], 0, jobId)\n rcall(\"LREM\", KEYS[2], 0, jobId)\n rcall(\"ZREM\", KEYS[3], jobId)\n rcall(\"LREM\", KEYS[4], 0, jobId)\n rcall(\"ZREM\", KEYS[5], jobId)\n rcall(\"ZREM\", KEYS[6], jobId)\n rcall(\"ZREM\", KEYS[7], jobId)\n\n removeDebounceKey(KEYS[11], KEYS[8])\n\n rcall(\"DEL\", KEYS[8])\n rcall(\"DEL\", KEYS[9])\n\n -- delete keys related to rate limiter\n local limiterIndexTable = KEYS[10] .. \":index\"\n local limitedSetKey = rcall(\"HGET\", limiterIndexTable, jobId)\n if limitedSetKey then\n rcall(\"SREM\", limitedSetKey, jobId)\n rcall(\"HDEL\", limiterIndexTable, jobId)\n end\n return 1\nelse\n return 0\nend\n" }, { "path": "lib/commands/removeJobs-8.lua", "content": "--[[\n Remove all jobs matching a given pattern from all the queues they may be in as well as all its data.\n In order to be able to remove any job, they must be unlocked.\n\n Input:\n KEYS[1] 'active',\n KEYS[2] 'wait',\n KEYS[3] 'delayed',\n KEYS[4] 'paused',\n KEYS[5] 'completed',\n KEYS[6] 'failed',\n KEYS[7] 'priority',\n KEYS[8] 'rate-limiter'\n\n ARGV[1] prefix\n ARGV[2] pattern\n ARGV[3] cursor\n\n Events:\n 'removed'\n]]\n\n-- TODO PUBLISH global events 'removed'\n\nlocal rcall = redis.call\nlocal result = rcall(\"SCAN\", ARGV[3], \"MATCH\", ARGV[1] .. ARGV[2])\nlocal cursor = result[1];\nlocal jobKeys = result[2];\nlocal removed = {}\n\nlocal prefixLen = string.len(ARGV[1]) + 1\nfor i, jobKey in ipairs(jobKeys) do\n local keyTypeResp = rcall(\"TYPE\", jobKey)\n if keyTypeResp[\"ok\"] == \"hash\" then\n local jobId = string.sub(jobKey, prefixLen)\n local lockKey = jobKey .. ':lock'\n local lock = redis.call(\"GET\", lockKey)\n if not lock then\n rcall(\"LREM\", KEYS[1], 0, jobId)\n rcall(\"LREM\", KEYS[2], 0, jobId)\n rcall(\"ZREM\", KEYS[3], jobId)\n rcall(\"LREM\", KEYS[4], 0, jobId)\n rcall(\"ZREM\", KEYS[5], jobId)\n rcall(\"ZREM\", KEYS[6], jobId)\n rcall(\"ZREM\", KEYS[7], jobId)\n rcall(\"DEL\", jobKey)\n rcall(\"DEL\", jobKey .. ':logs')\n\n -- delete keys related to rate limiter\n local limiterIndexTable = KEYS[8] .. \":index\"\n local limitedSetKey = rcall(\"HGET\", limiterIndexTable, jobId)\n\n if limitedSetKey then\n rcall(\"SREM\", limitedSetKey, jobId)\n rcall(\"HDEL\", limiterIndexTable, jobId)\n end\n table.insert(removed, jobId)\n end\n end\nend\nreturn {cursor, removed}\n" }, { "path": "lib/commands/removeRepeatable-2.lua", "content": "\n--[[\n Removes a repeatable job\n Input:\n KEYS[1] repeat jobs key\n KEYS[2] delayed jobs key\n\n ARGV[1] repeat job id\n ARGV[2] repeat job key\n ARGV[3] queue key\n]]\nlocal millis = redis.call(\"ZSCORE\", KEYS[1], ARGV[2])\n\nif(millis) then\n -- Delete next programmed job.\n local repeatJobId = ARGV[1] .. millis\n if(redis.call(\"ZREM\", KEYS[2], repeatJobId) == 1) then\n redis.call(\"DEL\", ARGV[3] .. repeatJobId)\n end\nend\n\nredis.call(\"ZREM\", KEYS[1], ARGV[2]);\n" }, { "path": "lib/commands/reprocessJob-6.lua", "content": "--[[\n Attempts to reprocess a job\n\n Input:\n KEYS[1] job key\n KEYS[2] job lock key\n KEYS[3] job state\n KEYS[4] wait key\n KEYS[5] meta-pause\n KEYS[6] paused key\n\n ARGV[1] job.id,\n ARGV[2] (job.opts.lifo ? 'R' : 'L') + 'PUSH'\n ARGV[3] token\n ARGV[4] timestamp\n\n Output:\n 1 means the operation was a success\n 0 means the job does not exist\n -1 means the job is currently locked and can't be retried.\n -2 means the job was not found in the expected set.\n\n\n]]\nlocal rcall = redis.call;\nif (rcall(\"EXISTS\", KEYS[1]) == 1) then\n if (rcall(\"EXISTS\", KEYS[2]) == 0) then\n rcall(\"HDEL\", KEYS[1], \"finishedOn\", \"processedOn\", \"failedReason\")\n rcall(\"HSET\", KEYS[1], \"retriedOn\", ARGV[4])\n\n if (rcall(\"ZREM\", KEYS[3], ARGV[1]) == 1) then\n local target\n if rcall(\"EXISTS\", KEYS[5]) ~= 1 then\n target = KEYS[4]\n else\n target = KEYS[6]\n end\n\n rcall(ARGV[2], target, ARGV[1])\n\n -- Emit waiting event (wait..ing@token)\n rcall(\"PUBLISH\", KEYS[4] .. \"ing@\" .. ARGV[3], ARGV[1])\n return 1\n else\n return -2\n end\n else\n return -1\n end\nelse\n return 0\nend\n" }, { "path": "lib/commands/retryJob-7.lua", "content": "--[[\n Retries a failed job by moving it back to the wait queue.\n\n Input:\n KEYS[1] 'active',\n KEYS[2] 'wait'\n KEYS[3] jobId key\n KEYS[4] 'meta-paused'\n KEYS[5] 'paused'\n KEYS[6] stalled key\n KEYS[7] 'priority'\n\n ARGV[1] pushCmd\n ARGV[2] jobId\n ARGV[3] token\n\n Events:\n 'prefix:added'\n\n Output:\n 0 - OK\n -1 - Missing key\n -2 - Job Not locked\n -3 - Job not in active set\n]]\nlocal rcall = redis.call\n\n-- Includes\n--- @include \"includes/addJobWithPriority\"\n--- @include \"includes/getTargetQueueList\"\n--- @include \"includes/removeLock\"\n\nif rcall(\"EXISTS\", KEYS[3]) == 1 then\n local errorCode = removeLock(KEYS[3], KEYS[6], ARGV[3], ARGV[2])\n if errorCode < 0 then\n return errorCode\n end\n\n local numRemovedElements = rcall(\"LREM\", KEYS[1], -1, ARGV[2])\n if numRemovedElements < 1 then return -3 end\n\n local target = getTargetQueueList(KEYS[4], KEYS[2], KEYS[5])\n\n local priority = tonumber(rcall(\"HGET\", KEYS[3], \"priority\")) or 0\n\n if priority == 0 then\n -- LIFO or FIFO\n rcall(ARGV[1], target, ARGV[2])\n else\n addJobWithPriority(KEYS[7], priority, ARGV[2], target)\n end\n\n return 0\nelse\n return -1\nend\n" }, { "path": "lib/commands/retryJobs-5.lua", "content": "--[[\n Attempts to retry all failed jobs\n\n Input:\n KEYS[1] base key\n KEYS[2] failed state key\n KEYS[3] wait state key\n KEYS[4] 'meta-paused'\n KEYS[5] 'paused'\n\n ARGV[1] count\n\n Output:\n 1 means the operation is not completed\n 0 means the operation is completed\n]]\nlocal baseKey = KEYS[1]\nlocal maxCount = tonumber(ARGV[1])\n\nlocal rcall = redis.call;\n\n-- Includes\n--- @include \"includes/batches\"\n\nlocal function getZSetItems(keyName, max)\n return rcall('ZRANGE', keyName, 0, max - 1)\nend\n\nlocal jobs = getZSetItems(KEYS[2], maxCount)\n\nif (#jobs > 0) then\n for i, key in ipairs(jobs) do\n local jobKey = baseKey .. key\n rcall(\"HDEL\", jobKey, \"finishedOn\", \"processedOn\", \"failedReason\")\n end\n\n local target\n if rcall(\"EXISTS\", KEYS[4]) ~= 1 then\n target = KEYS[3]\n else\n target = KEYS[5]\n end\n\n for from, to in batches(#jobs, 7000) do\n rcall(\"ZREM\", KEYS[2], unpack(jobs, from, to))\n rcall(\"LPUSH\", target, unpack(jobs, from, to))\n end\nend\n\nmaxCount = maxCount - #jobs\n\nif (maxCount <= 0) then return 1 end\n\nreturn 0\n" }, { "path": "lib/commands/saveStacktrace-1.lua", "content": "--[[\n Save stacktrace and failedReason.\n\n Input:\n KEYS[1] job key\n\n ARGV[1] stacktrace\n ARGV[2] failedReason\n ARGV[3] attemptsMade\n\n Output:\n 0 - OK\n -1 - Missing key\n]]\nlocal rcall = redis.call\n\nif rcall(\"EXISTS\", KEYS[1]) == 1 then\n rcall(\"HMSET\", KEYS[1], \"stacktrace\", ARGV[1], \"failedReason\", ARGV[2],\n \"attemptsMade\", ARGV[3])\n\n return 0\nelse\n return -1\nend\n" }, { "path": "lib/commands/script-loader.js", "content": "'use strict';\nconst { createHash } = require('crypto');\nconst path = require('path');\nconst fs = require('fs');\nconst { promisify } = require('util');\n\nconst readFile = promisify(fs.readFile);\nconst readdir = promisify(fs.readdir);\n\nconst GlobOptions = { dot: true, silent: false };\nconst IncludeRegex = /^[-]{2,3}[ \\t]*@include[ \\t]+([\"'])(.+?)\\1[; \\t\\n]*$/m;\nconst EmptyLineRegex = /^\\s*[\\r\\n]/gm;\n\nclass ScriptLoaderError extends Error {\n /**\n * The include stack\n */\n\n constructor(message, path, stack = [], line, position = 0) {\n super(message);\n // Ensure the name of this error is the same as the class name\n this.name = this.constructor.name;\n Error.captureStackTrace(this, this.constructor);\n this.includes = stack;\n this.line = line ? line : 0;\n this.position = position;\n }\n}\n\nconst isPossiblyMappedPath = path => path && ['~', '<'].includes(path[0]);\n\n/**\n * Lua script loader with include support\n */\nclass ScriptLoader {\n constructor() {\n this.pathMapper = new Map();\n this.clientScripts = new WeakMap();\n /**\n * Cache commands by dir\n */\n this.commandCache = new Map();\n this.rootPath = getPkgJsonDir();\n this.pathMapper.set('~', this.rootPath);\n this.pathMapper.set('rootDir', this.rootPath);\n this.pathMapper.set('base', __dirname);\n }\n\n /**\n * Add a script path mapping. Allows includes of the form \"/utils.lua\" where `includes` is a user\n * defined path\n * @param name - the name of the mapping. Note: do not include angle brackets\n * @param mappedPath - if a relative path is passed, it's relative to the *caller* of this function.\n * Mapped paths are also accepted, e.g. \"~/server/scripts/lua\" or \"/includes\"\n */\n addPathMapping(name, mappedPath) {\n let resolved;\n\n if (isPossiblyMappedPath(mappedPath)) {\n resolved = this.resolvePath(mappedPath);\n } else {\n const caller = getCallerFile();\n const callerPath = path.dirname(caller);\n resolved = path.normalize(path.resolve(callerPath, mappedPath));\n }\n\n const last = resolved.length - 1;\n if (resolved[last] === path.sep) {\n resolved = resolved.substr(0, last);\n }\n\n this.pathMapper.set(name, resolved);\n }\n\n /**\n * Resolve the script path considering path mappings\n * @param scriptName - the name of the script\n * @param stack - the include stack, for nicer errors\n */\n resolvePath(scriptName, stack = []) {\n const first = scriptName[0];\n if (first === '~') {\n scriptName = path.join(this.rootPath, scriptName.substr(2));\n } else if (first === '<') {\n const p = scriptName.indexOf('>');\n if (p > 0) {\n const name = scriptName.substring(1, p);\n const mappedPath = this.pathMapper.get(name);\n if (!mappedPath) {\n throw new ScriptLoaderError(\n `No path mapping found for \"${name}\"`,\n scriptName,\n stack\n );\n }\n scriptName = path.join(mappedPath, scriptName.substring(p + 1));\n }\n }\n\n return path.normalize(scriptName);\n }\n\n /**\n * Recursively collect all scripts included in a file\n * @param file - the parent file\n * @param cache - a cache for file metadata to increase efficiency. Since a file can be included\n * multiple times, we make sure to load it only once.\n * @param stack - internal stack to prevent circular references\n */\n async resolveDependencies(file, cache, isInclude = false, stack = []) {\n cache = cache ? cache : new Map();\n\n if (stack.includes(file.path)) {\n throw new ScriptLoaderError(\n `circular reference: \"${file.path}\"`,\n file.path,\n stack\n );\n }\n stack.push(file.path);\n\n function findPos(content, match) {\n const pos = content.indexOf(match);\n const arr = content.slice(0, pos).split('\\n');\n return {\n line: arr.length,\n column: arr[arr.length - 1].length + match.indexOf('@include') + 1\n };\n }\n\n function raiseError(msg, match) {\n const pos = findPos(file.content, match);\n throw new ScriptLoaderError(msg, file.path, stack, pos.line, pos.column);\n }\n // eslint-disable-next-line node/no-unpublished-require\n const minimatch = require('minimatch');\n\n if (!minimatch) {\n console.warn('Install minimatch as dev-dependency');\n }\n\n const Minimatch = minimatch.Minimatch || class Empty {};\n // eslint-disable-next-line node/no-unpublished-require\n const fg = require('fast-glob');\n\n if (!fg) {\n console.warn('Install fast-glob as dev-dependency');\n }\n\n const nonOp = () => {\n return [''];\n };\n const glob = fg ? fg.glob : nonOp;\n\n const hasMagic = pattern => {\n if (!Array.isArray(pattern)) {\n pattern = [pattern];\n }\n for (const p of pattern) {\n if (new Minimatch(p, GlobOptions).hasMagic()) {\n return true;\n }\n }\n return false;\n };\n\n const hasFilenamePattern = path => hasMagic(path);\n\n async function getFilenamesByPattern(pattern) {\n return glob(pattern, { dot: true });\n }\n\n let res;\n let content = file.content;\n\n while ((res = IncludeRegex.exec(content)) !== null) {\n const [match, , reference] = res;\n\n const includeFilename = isPossiblyMappedPath(reference)\n ? // mapped paths imply absolute reference\n this.resolvePath(ensureExt(reference), stack)\n : // include path is relative to the file being processed\n path.resolve(path.dirname(file.path), ensureExt(reference));\n\n let includePaths;\n\n if (hasFilenamePattern(includeFilename)) {\n const filesMatched = await getFilenamesByPattern(includeFilename);\n includePaths = filesMatched.map(x => path.resolve(x));\n } else {\n includePaths = [includeFilename];\n }\n\n includePaths = includePaths.filter(file => path.extname(file) === '.lua');\n\n if (includePaths.length === 0) {\n raiseError(`include not found: \"${reference}\"`, match);\n }\n\n const tokens = [];\n\n for (let i = 0; i < includePaths.length; i++) {\n const includePath = includePaths[i];\n\n const hasInclude = file.includes.find(x => x.path === includePath);\n\n if (hasInclude) {\n /**\n * We have something like\n * --- \\@include \"a\"\n * ...\n * --- \\@include \"a\"\n */\n raiseError(\n `file \"${reference}\" already included in \"${file.path}\"`,\n match\n );\n }\n\n let includeMetadata = cache.get(includePath);\n let token;\n\n if (!includeMetadata) {\n const { name, numberOfKeys } = splitFilename(includePath);\n let childContent = '';\n try {\n const buf = await readFile(includePath, { flag: 'r' });\n childContent = buf.toString();\n } catch (err) {\n if (err.code === 'ENOENT') {\n raiseError(`include not found: \"${reference}\"`, match);\n } else {\n throw err;\n }\n }\n // this represents a normalized version of the path to make replacement easy\n token = getPathHash(includePath);\n includeMetadata = {\n name,\n numberOfKeys,\n path: includePath,\n content: childContent,\n token,\n includes: []\n };\n cache.set(includePath, includeMetadata);\n } else {\n token = includeMetadata.token;\n }\n\n tokens.push(token);\n\n file.includes.push(includeMetadata);\n await this.resolveDependencies(includeMetadata, cache, true, stack);\n }\n\n // Replace @includes with normalized path hashes\n const substitution = tokens.join('\\n');\n content = content.replace(match, substitution);\n }\n\n file.content = content;\n\n if (isInclude) {\n cache.set(file.path, file);\n } else {\n cache.set(file.name, file);\n }\n\n stack.pop();\n }\n\n /**\n * Parse a (top-level) lua script\n * @param filename - the full path to the script\n * @param content - the content of the script\n * @param cache - cache\n */\n async parseScript(filename, content, cache) {\n const { name, numberOfKeys } = splitFilename(filename);\n const meta = cache ? cache.get(name) : undefined;\n if (meta && meta.content === content) {\n return meta;\n }\n const fileInfo = {\n path: filename,\n token: getPathHash(filename),\n content,\n name,\n numberOfKeys,\n includes: []\n };\n\n await this.resolveDependencies(fileInfo, cache);\n return fileInfo;\n }\n\n /**\n * Construct the final version of a file by interpolating its includes in dependency order.\n * @param file - the file whose content we want to construct\n * @param processed - a cache to keep track of which includes have already been processed\n */\n interpolate(file, processed) {\n processed = processed || new Set();\n let content = file.content;\n file.includes.forEach(child => {\n const emitted = processed ? processed.has(child.path) : undefined;\n const fragment = this.interpolate(child, processed);\n const replacement = emitted ? '' : fragment;\n\n if (!replacement) {\n content = replaceAll(content, child.token, '');\n } else {\n // replace the first instance with the dependency\n content = content.replace(child.token, replacement);\n // remove the rest\n content = replaceAll(content, child.token, '');\n }\n\n if (processed) {\n processed.add(child.path);\n }\n });\n\n return content;\n }\n\n async loadCommand(filename, cache) {\n filename = path.resolve(filename);\n\n const { name: scriptName } = splitFilename(filename);\n let script = cache ? cache.get(scriptName) : undefined;\n if (!script) {\n const content = (await readFile(filename)).toString();\n script = await this.parseScript(filename, content, cache);\n }\n\n const lua = removeEmptyLines(this.interpolate(script));\n const { name, numberOfKeys } = script;\n\n return {\n name,\n options: { numberOfKeys: numberOfKeys, lua }\n };\n }\n\n /**\n * Load redis lua scripts.\n * The name of the script must have the following format:\n *\n * cmdName-numKeys.lua\n *\n * cmdName must be in camel case format.\n *\n * For example:\n * moveToFinish-3.lua\n *\n */\n async loadScripts(dir, cache) {\n dir = path.normalize(dir || __dirname);\n\n let commands = this.commandCache.get(dir);\n if (commands) {\n return commands;\n }\n\n const files = await readdir(dir);\n\n const luaFiles = files.filter(file => path.extname(file) === '.lua');\n\n if (luaFiles.length === 0) {\n /**\n * To prevent unclarified runtime error \"updateDelayset is not a function\n * @see https://github.com/OptimalBits/bull/issues/920\n */\n throw new ScriptLoaderError('No .lua files found!', dir, []);\n }\n\n commands = [];\n cache = cache ? cache : new Map();\n\n for (let i = 0; i < luaFiles.length; i++) {\n const file = path.join(dir, luaFiles[i]);\n\n const command = await this.loadCommand(file, cache);\n commands.push(command);\n }\n\n this.commandCache.set(dir, commands);\n\n return commands;\n }\n\n /**\n * Attach all lua scripts in a given directory to a client instance\n * @param client - redis client to attach script to\n * @param pathname - the path to the directory containing the scripts\n */\n async load(client, pathname, cache) {\n let paths = this.clientScripts.get(client);\n if (!paths) {\n paths = new Set();\n this.clientScripts.set(client, paths);\n }\n if (!paths.has(pathname)) {\n paths.add(pathname);\n const scripts = await this.loadScripts(\n pathname,\n cache ? cache : new Map()\n );\n scripts.forEach(command => {\n // Only define the command if not already defined\n if (!client[command.name]) {\n client.defineCommand(command.name, command.options);\n }\n });\n }\n }\n\n /**\n * Clears the command cache\n */\n clearCache() {\n this.commandCache.clear();\n }\n}\n\nfunction ensureExt(filename, ext = 'lua') {\n const foundExt = path.extname(filename);\n if (foundExt && foundExt !== '.') {\n return filename;\n }\n if (ext && ext[0] !== '.') {\n ext = `.${ext}`;\n }\n return `${filename}${ext}`;\n}\n\nfunction splitFilename(filePath) {\n const longName = path.basename(filePath, '.lua');\n const [name, num] = longName.split('-');\n const numberOfKeys = num ? parseInt(num, 10) : undefined;\n return { name, numberOfKeys };\n}\n\n// Determine the project root\n// https://stackoverflow.com/a/18721515\nfunction getPkgJsonDir() {\n for (const modPath of module.paths || []) {\n try {\n const prospectivePkgJsonDir = path.dirname(modPath);\n fs.accessSync(modPath, fs.constants.F_OK);\n return prospectivePkgJsonDir;\n // eslint-disable-next-line no-empty\n } catch (e) {}\n }\n return '';\n}\n\n// https://stackoverflow.com/a/66842927\n// some dark magic here :-)\n// this version is preferred to the simpler version because of\n// https://github.com/facebook/jest/issues/5303 -\n// tldr: dont assume you're the only one with the doing something like this\nfunction getCallerFile() {\n const originalFunc = Error.prepareStackTrace;\n\n let callerFile = '';\n try {\n Error.prepareStackTrace = (_, stack) => stack;\n\n const sites = new Error().stack;\n const shiftResponse = sites.shift();\n const currentFile = shiftResponse ? shiftResponse.getFileName() : undefined;\n\n while (sites.length) {\n const newShiftResponse = sites.shift();\n callerFile = newShiftResponse ? newShiftResponse.getFileName() : '';\n\n if (currentFile !== callerFile) {\n break;\n }\n }\n // eslint-disable-next-line no-empty\n } catch (e) {\n } finally {\n Error.prepareStackTrace = originalFunc;\n }\n\n return callerFile;\n}\n\nfunction sha1(data) {\n return createHash('sha1')\n .update(data)\n .digest('hex');\n}\n\nfunction getPathHash(normalizedPath) {\n return `@@${sha1(normalizedPath)}`;\n}\n\nfunction replaceAll(str, find, replace) {\n return str.replace(new RegExp(find, 'g'), replace);\n}\n\nfunction removeEmptyLines(str) {\n return str.replace(EmptyLineRegex, '');\n}\n\nmodule.exports = {\n ScriptLoaderError,\n ScriptLoader\n};\n" }, { "path": "lib/commands/takeLock-1.lua", "content": "--[[\n Takes a lock\n\n Input:\n KEYS[1] 'lock',\n \n ARGV[1] token\n ARGV[2] lock duration in milliseconds\n \n Output:\n \"OK\" if lock taken successfully.\n]]\nif redis.call(\"SET\", KEYS[1], ARGV[1], \"NX\", \"PX\", ARGV[2]) then\n return 1\nelse\n return 0\nend\n" }, { "path": "lib/commands/updateData-1.lua", "content": "--[[\n Update job data\n\n Input:\n KEYS[1] Job id key\n\n ARGV[1] data\n\n Output:\n 0 - OK\n -1 - Missing job.\n]]\nlocal rcall = redis.call\n\nif rcall(\"EXISTS\",KEYS[1]) == 1 then -- // Make sure job exists\n rcall(\"HSET\", KEYS[1], \"data\", ARGV[1])\n return 0\nelse\n return -1\nend\n" }, { "path": "lib/commands/updateDelaySet-6.lua", "content": "--[[\n Updates the delay set, by picking a delayed job that should\n be processed now.\n\n Input:\n KEYS[1] 'delayed'\n KEYS[2] 'active'\n KEYS[3] 'wait'\n KEYS[4] 'priority'\n\n KEYS[5] 'paused'\n KEYS[6] 'meta-paused'\n\n ARGV[1] queue.toKey('')\n ARGV[2] delayed timestamp\n ARGV[3] queue token\n\n Events:\n 'removed'\n]]\nlocal rcall = redis.call;\n\n-- Includes\n--- @include \"includes/addJobWithPriority\"\n--- @include \"includes/getTargetQueueList\"\n\n-- Try to get as much as 1000 jobs at once\nlocal jobs = rcall(\"ZRANGEBYSCORE\", KEYS[1], 0, tonumber(ARGV[2]) * 0x1000, \"LIMIT\", 0, 1000)\n\nif(#jobs > 0) then\n rcall(\"ZREM\", KEYS[1], unpack(jobs))\n\n -- check if we need to use push in paused instead of waiting\n local target = getTargetQueueList(KEYS[6], KEYS[3], KEYS[5])\n\n for _, jobId in ipairs(jobs) do\n -- Is this really needed?\n rcall(\"LREM\", KEYS[2], 0, jobId)\n\n local priority = tonumber(rcall(\"HGET\", ARGV[1] .. jobId, \"priority\")) or 0\n \n if priority == 0 then\n -- LIFO or FIFO\n rcall(\"LPUSH\", target, jobId)\n else\n addJobWithPriority(KEYS[4], priority, jobId, target)\n end\n \n -- Emit waiting event (wait..ing@token)\n rcall(\"PUBLISH\", KEYS[3] .. \"ing@\" .. ARGV[3], jobId)\n rcall(\"HSET\", ARGV[1] .. jobId, \"delay\", 0)\n end\nend\n\nlocal nextTimestamp = rcall(\"ZRANGE\", KEYS[1], 0, 0, \"WITHSCORES\")[2]\nif(nextTimestamp ~= nil) then\n rcall(\"PUBLISH\", KEYS[1], nextTimestamp / 0x1000)\nend\nreturn nextTimestamp\n" }, { "path": "lib/commands/updateProgress-2.lua", "content": "--[[\n Update job progress\n\n Input:\n KEYS[1] Job id key\n KEYS[2] progress event key\n \n ARGV[1] progress\n ARGV[2] event data\n\n Event:\n progress(jobId, progress)\n]]\nlocal rcall = redis.call\nif rcall(\"EXISTS\", KEYS[1]) == 1 then -- // Make sure job exists\n rcall(\"HSET\", KEYS[1], \"progress\", ARGV[1])\n rcall(\"PUBLISH\", KEYS[2], ARGV[2])\n return 0\nelse\n return -1\nend\n" }, { "path": "lib/errors.js", "content": "'use strict';\n\nmodule.exports.Messages = {\n RETRY_JOB_NOT_EXIST: \"Couldn't retry job: The job doesn't exist\",\n RETRY_JOB_IS_LOCKED: \"Couldn't retry job: The job is locked\",\n RETRY_JOB_NOT_FAILED:\n \"Couldn't retry job: The job has been already retried or has not failed\",\n MISSING_REDIS_OPTS: `Using a redis instance with enableReadyCheck or maxRetriesPerRequest for bclient/subscriber is not permitted.\n see https://github.com/OptimalBits/bull/issues/1873\n `\n};\n" }, { "path": "lib/getters.js", "content": "'use strict';\n\nconst _ = require('lodash');\nconst Job = require('./job');\nconst scripts = require('./scripts');\n\nmodule.exports = function(Queue) {\n Queue.prototype.getJob = async function(jobId) {\n await this.isReady();\n return Job.fromId(this, jobId);\n };\n\n Queue.prototype.getCountsPerPriority = async function(priorities) {\n const uniquePriorities = [...new Set(priorities)];\n const responses = await scripts.getCountsPerPriority(\n this,\n uniquePriorities\n );\n\n const counts = {};\n responses.forEach((res, index) => {\n counts[`${uniquePriorities[index]}`] = res || 0;\n });\n\n return counts;\n };\n\n Queue.prototype._commandByType = function(types, count, callback) {\n return _.map(types, type => {\n type = type === 'waiting' ? 'wait' : type; // alias\n\n const key = this.toKey(type);\n\n switch (type) {\n case 'completed':\n case 'failed':\n case 'delayed':\n case 'repeat':\n return callback(key, count ? 'zcard' : 'zrange');\n case 'active':\n case 'wait':\n case 'paused':\n return callback(key, count ? 'llen' : 'lrange');\n }\n });\n };\n\n /**\n Returns the number of jobs waiting to be processed.\n */\n Queue.prototype.count = function() {\n return this.getJobCountByTypes('wait', 'paused', 'delayed');\n };\n\n // Job counts by type\n // Queue#getJobCountByTypes('completed') => completed count\n // Queue#getJobCountByTypes('completed,failed') => completed + failed count\n // Queue#getJobCountByTypes('completed', 'failed') => completed + failed count\n // Queue#getJobCountByTypes('completed,waiting', 'failed') => completed + waiting + failed count\n Queue.prototype.getJobCountByTypes = function() {\n return this.getJobCounts.apply(this, arguments).then(result => {\n return _.chain(result)\n .values()\n .sum()\n .value();\n });\n };\n\n /**\n * Returns the job counts for each type specified or every list/set in the queue by default.\n *\n */\n Queue.prototype.getJobCounts = function() {\n const types = parseTypeArg(arguments);\n const multi = this.multi();\n\n this._commandByType(types, true, (key, command) => {\n multi[command](key);\n });\n\n return multi.exec().then(res => {\n const counts = {};\n res.forEach((res, index) => {\n counts[types[index]] = res[1] || 0;\n });\n return counts;\n });\n };\n\n Queue.prototype.getCompletedCount = function() {\n return this.getJobCountByTypes('completed');\n };\n\n Queue.prototype.getFailedCount = function() {\n return this.getJobCountByTypes('failed');\n };\n\n Queue.prototype.getDelayedCount = function() {\n return this.getJobCountByTypes('delayed');\n };\n\n Queue.prototype.getActiveCount = function() {\n return this.getJobCountByTypes('active');\n };\n\n Queue.prototype.getWaitingCount = function() {\n return this.getJobCountByTypes('wait', 'paused');\n };\n\n /**\n *\n * @returns the potential stalled jobs. Only useful for tests.\n */\n Queue.prototype.getStalledCount = function() {\n const key = this.toKey('stalled');\n return this.client.scard(key);\n };\n\n // TO BE DEPRECATED --->\n Queue.prototype.getPausedCount = function() {\n return this.getJobCountByTypes('paused');\n };\n // <-----\n\n Queue.prototype.getWaiting = function(start, end, opts) {\n return this.getJobs(['wait', 'paused'], start, end, true, opts);\n };\n\n Queue.prototype.getActive = function(start, end, opts) {\n return this.getJobs('active', start, end, true, opts);\n };\n\n Queue.prototype.getDelayed = function(start, end, opts) {\n return this.getJobs('delayed', start, end, true, opts);\n };\n\n Queue.prototype.getCompleted = function(start, end, opts) {\n return this.getJobs('completed', start, end, false, opts);\n };\n\n Queue.prototype.getFailed = function(start, end, opts) {\n return this.getJobs('failed', start, end, false, opts);\n };\n\n Queue.prototype.getRanges = function(types, start, end, asc) {\n start = _.isUndefined(start) ? 0 : start;\n end = _.isUndefined(end) ? -1 : end;\n\n const multi = this.multi();\n const multiCommands = [];\n\n this._commandByType(parseTypeArg(types), false, (key, command) => {\n switch (command) {\n case 'lrange':\n if (asc) {\n multiCommands.push('lrange');\n multi.lrange(key, -(end + 1), -(start + 1));\n } else {\n multi.lrange(key, start, end);\n }\n break;\n case 'zrange':\n multiCommands.push('zrange');\n if (asc) {\n multi.zrange(key, start, end);\n } else {\n multi.zrevrange(key, start, end);\n }\n break;\n }\n });\n\n return multi.exec().then(responses => {\n let results = [];\n\n responses.forEach((response, index) => {\n const result = response[1] || [];\n\n if (asc && multiCommands[index] === 'lrange') {\n results = results.concat(result.reverse());\n } else {\n results = results.concat(result);\n }\n });\n return results;\n });\n };\n\n Queue.prototype.getJobs = function(types, start, end, asc, opts) {\n return this.getRanges(types, start, end, asc).then(jobIds => {\n return Promise.all(jobIds.map(jobId => this.getJobFromId(jobId, opts)));\n });\n };\n\n Queue.prototype.getJobLogs = function(jobId, start, end, asc = true) {\n start = _.isUndefined(start) ? 0 : start;\n end = _.isUndefined(end) ? -1 : end;\n\n const multi = this.multi();\n\n const logsKey = this.toKey(jobId + ':logs');\n if (asc) {\n multi.lrange(logsKey, start, end);\n } else {\n multi.lrange(logsKey, -(end + 1), -(start + 1));\n }\n multi.llen(logsKey);\n return multi.exec().then(result => {\n if (!asc) {\n result[0][1].reverse();\n }\n return {\n logs: result[0][1],\n count: result[1][1]\n };\n });\n };\n\n /**\n * Get queue metrics related to the queue.\n *\n * This method returns the gathered metrics for the queue.\n * The metrics are represented as an array of job counts\n * per unit of time (1 minute).\n *\n * @param start - Start point of the metrics, where 0\n * is the newest point to be returned.\n * @param end - End poinf of the metrics, where -1 is the\n * oldest point to be returned.\n *\n * @returns - Returns an object with queue metrics.\n */\n Queue.prototype.getMetrics = async function(type, start = 0, end = -1) {\n const metricsKey = this.toKey(`metrics:${type}`);\n const dataKey = `${metricsKey}:data`;\n\n const multi = this.multi();\n multi.hmget(metricsKey, 'count', 'prevTS', 'prevCount');\n multi.lrange(dataKey, start, end);\n multi.llen(dataKey);\n\n const [hmget, range, len] = await multi.exec();\n const [err, [count, prevTS, prevCount]] = hmget;\n const [err2, data] = range;\n const [err3, numPoints] = len;\n if (err || err2) {\n throw err || err2 || err3;\n }\n\n return {\n meta: {\n count: parseInt(count || '0', 10),\n prevTS: parseInt(prevTS || '0', 10),\n prevCount: parseInt(prevCount || '0', 10)\n },\n data,\n count: numPoints\n };\n };\n};\n\nfunction parseTypeArg(args) {\n const types = _.chain([])\n .concat(args)\n .join(',')\n .split(/\\s*,\\s*/g)\n .compact()\n .value();\n\n return types.length\n ? types\n : ['waiting', 'active', 'completed', 'failed', 'delayed', 'paused'];\n}\n" }, { "path": "lib/job.js", "content": "'use strict';\n\nconst _ = require('lodash');\nconst utils = require('./utils');\nconst scripts = require('./scripts');\nconst debuglog = require('util').debuglog('bull');\nconst errors = require('./errors');\nconst backoffs = require('./backoffs');\n\nconst FINISHED_WATCHDOG = 5000;\nconst DEFAULT_JOB_NAME = '__default__';\n\n/**\ninterface JobOptions\n{\n priority: Priority;\n attempts: number;\n delay: number;\n}\n*/\n\nconst jobFields = [\n 'opts',\n 'name',\n 'id',\n 'progress',\n 'delay',\n 'timestamp',\n 'finishedOn',\n 'processedOn',\n 'retriedOn',\n 'failedReason',\n 'attemptsMade',\n 'stacktrace',\n 'returnvalue'\n];\n\n// queue: Queue, data: {}, opts: JobOptions\nconst Job = function(queue, name, data, opts) {\n if (typeof name !== 'string') {\n opts = data;\n data = name;\n name = DEFAULT_JOB_NAME;\n }\n\n // defaults\n this.opts = setDefaultOpts(opts);\n\n this.name = name;\n this.queue = queue;\n this.data = data;\n this._progress = 0;\n this.delay = this.opts.delay < 0 ? 0 : this.opts.delay;\n this.timestamp = this.opts.timestamp;\n this.stacktrace = [];\n this.returnvalue = null;\n this.attemptsMade = 0;\n\n this.toKey = _.bind(queue.toKey, queue);\n this.debounceId = this.opts.debounce ? this.opts.debounce.id : undefined;\n};\n\nfunction setDefaultOpts(opts) {\n const _opts = Object.assign({}, opts);\n\n _opts.attempts = typeof _opts.attempts == 'undefined' ? 1 : _opts.attempts;\n _opts.delay = typeof _opts.delay == 'undefined' ? 0 : Number(_opts.delay);\n _opts.timestamp =\n typeof _opts.timestamp == 'undefined' ? Date.now() : _opts.timestamp;\n\n _opts.attempts = parseInt(_opts.attempts);\n _opts.backoff = backoffs.normalize(_opts.backoff);\n\n return _opts;\n}\n\nJob.DEFAULT_JOB_NAME = DEFAULT_JOB_NAME;\n\nfunction addJob(queue, client, job) {\n const opts = job.opts;\n\n const jobData = job.toData();\n return scripts.addJob(client, queue, jobData, {\n lifo: opts.lifo,\n customJobId: opts.jobId,\n priority: opts.priority,\n debounce: opts.debounce\n });\n}\n\nJob.create = function(queue, name, data, opts) {\n const job = new Job(queue, name, data, opts);\n\n return queue\n .isReady()\n .then(() => {\n return addJob(queue, queue.client, job);\n })\n .then(jobId => {\n job.id = jobId;\n debuglog('Job added', jobId);\n return job;\n });\n};\n\nJob.createBulk = function(queue, jobs) {\n jobs = jobs.map(job => new Job(queue, job.name, job.data, job.opts));\n\n return queue\n .isReady()\n .then(() => {\n const multi = queue.client.multi();\n\n for (const job of jobs) {\n addJob(queue, multi, job);\n }\n\n return multi.exec();\n })\n .then(res => {\n res.forEach((res, index) => {\n jobs[index].id = res[1];\n debuglog('Job added', res[1]);\n });\n\n return jobs;\n });\n};\n\nJob.fromId = async function(queue, jobId, opts) {\n // jobId can be undefined if moveJob returns undefined\n if (!jobId) {\n return Promise.resolve();\n }\n\n const jobKey = queue.toKey(jobId);\n let rawJob;\n\n if (opts && opts.excludeData) {\n rawJob = _.zipObject(\n jobFields,\n await queue.client.hmget(jobKey, jobFields)\n );\n } else {\n rawJob = await queue.client.hgetall(jobKey);\n }\n return _.isEmpty(rawJob) ? null : Job.fromJSON(queue, rawJob, jobId);\n};\n\nJob.remove = async function(queue, pattern) {\n await queue.isReady();\n const removed = await scripts.removeWithPattern(queue, pattern);\n removed.forEach(jobId => queue.emit('removed', jobId));\n};\n\nJob.prototype.progress = function(progress) {\n if (_.isUndefined(progress)) {\n return this._progress;\n }\n this._progress = progress;\n return scripts.updateProgress(this, progress);\n};\n\nJob.prototype.update = async function(data) {\n this.data = data;\n const code = await scripts.updateData(this, data);\n\n if (code < 0) {\n throw scripts.finishedErrors(code, this.id, 'updateData');\n }\n};\n\nJob.prototype.toJSON = function() {\n const opts = Object.assign({}, this.opts);\n return {\n id: this.id,\n name: this.name,\n data: this.data || {},\n opts: opts,\n progress: this._progress,\n delay: this.delay, // Move to opts\n timestamp: this.timestamp,\n attemptsMade: this.attemptsMade,\n failedReason: this.failedReason,\n stacktrace: this.stacktrace || null,\n returnvalue: this.returnvalue || null,\n debounceId: this.debounceId || null,\n finishedOn: this.finishedOn || null,\n processedOn: this.processedOn || null\n };\n};\n\nJob.prototype.toData = function() {\n const json = this.toJSON();\n\n json.data = JSON.stringify(json.data);\n json.opts = JSON.stringify(json.opts);\n json.stacktrace = JSON.stringify(json.stacktrace);\n json.failedReason = JSON.stringify(json.failedReason);\n json.returnvalue = JSON.stringify(json.returnvalue);\n\n return json;\n};\n\n/**\n Return a unique key representing a lock for this Job\n*/\nJob.prototype.lockKey = function() {\n return this.toKey(this.id) + ':lock';\n};\n\n/**\n Takes a lock for this job so that no other queue worker can process it at the\n same time.\n*/\nJob.prototype.takeLock = function() {\n return scripts.takeLock(this.queue, this).then(lock => {\n return lock || false;\n });\n};\n\n/**\n Releases the lock. Only locks owned by the queue instance can be released.\n*/\nJob.prototype.releaseLock = function() {\n return scripts.releaseLock(this.queue, this.id).then(unlocked => {\n if (unlocked != 1) {\n throw new Error('Could not release lock for job ' + this.id);\n }\n });\n};\n\n/**\n * Extend the lock for this job.\n *\n * @param duration lock duration in milliseconds\n */\nJob.prototype.extendLock = function(duration) {\n return scripts.extendLock(this.queue, this.id, duration);\n};\n\n/**\n * Moves a job to the completed queue.\n * Returned job to be used with Queue.prototype.nextJobFromJobData.\n * @param returnValue {string} The jobs success message.\n * @param ignoreLock {boolean} True when wanting to ignore the redis lock on this job.\n * @param notFetch {boolean} True when should not fetch next job from queue.\n * @returns {Promise} Returns the jobData of the next job in the waiting queue.\n */\nJob.prototype.moveToCompleted = function(\n returnValue,\n ignoreLock,\n notFetch = false\n) {\n return this.queue.isReady().then(() => {\n this.returnvalue = returnValue || 0;\n\n returnValue = utils.tryCatch(JSON.stringify, JSON, [returnValue]);\n if (returnValue === utils.errorObject) {\n const err = utils.errorObject.value;\n return Promise.reject(err);\n }\n this.finishedOn = Date.now();\n\n return scripts.moveToCompleted(\n this,\n returnValue,\n this.opts.removeOnComplete,\n ignoreLock,\n notFetch\n );\n });\n};\n\nJob.prototype.discard = function() {\n this._discarded = true;\n};\n\n/**\n * Moves a job to the failed queue.\n * @param err {string} The jobs error message.\n * @param ignoreLock {boolean} True when wanting to ignore the redis lock on this job.\n * @returns void\n */\nJob.prototype.moveToFailed = async function(err, ignoreLock) {\n err = err || { message: 'Unknown reason' };\n\n this.failedReason = err.message;\n\n await this.queue.isReady();\n\n let command;\n const multi = this.queue.client.multi();\n this._saveAttempt(multi, err);\n\n // Check if an automatic retry should be performed\n let moveToFailed = false;\n if (this.attemptsMade < this.opts.attempts && !this._discarded) {\n // Check if backoff is needed\n const delay = await backoffs.calculate(\n this.opts.backoff,\n this.attemptsMade,\n this.queue.settings.backoffStrategies,\n err,\n _.get(this, 'opts.backoff.options', null)\n );\n\n if (delay === -1) {\n // If delay is -1, we should no continue retrying\n moveToFailed = true;\n } else if (delay) {\n // If so, move to delayed (need to unlock job in this case!)\n const args = scripts.moveToDelayedArgs(\n this.queue,\n this.id,\n Date.now() + delay,\n ignoreLock\n );\n multi.moveToDelayed(args);\n command = 'delayed';\n } else {\n // If not, retry immediately\n multi.retryJob(scripts.retryJobArgs(this, ignoreLock));\n command = 'retry';\n }\n } else {\n // If not, move to failed\n moveToFailed = true;\n }\n\n if (moveToFailed) {\n this.finishedOn = Date.now();\n const args = scripts.moveToFailedArgs(\n this,\n err.message,\n this.opts.removeOnFail,\n ignoreLock\n );\n multi.moveToFinished(args);\n command = 'failed';\n }\n const results = await multi.exec();\n const code = _.last(results)[1];\n if (code < 0) {\n throw scripts.finishedErrors(code, this.id, command, 'active');\n }\n};\n\nJob.prototype.moveToDelayed = function(timestamp, ignoreLock) {\n return scripts.moveToDelayed(this.queue, this.id, timestamp, ignoreLock);\n};\n\nJob.prototype.promote = function() {\n const queue = this.queue;\n const jobId = this.id;\n return queue.isReady().then(() =>\n scripts.promote(queue, jobId).then(result => {\n if (result === -1) {\n throw new Error('Job ' + jobId + ' is not in a delayed state');\n }\n })\n );\n};\n\n/**\n * Attempts to retry the job. Only a job that has failed can be retried.\n *\n * @return {Promise} If resolved and return code is 1, then the queue emits a waiting event\n * otherwise the operation was not a success and throw the corresponding error. If the promise\n * rejects, it indicates that the script failed to execute\n */\nJob.prototype.retry = function() {\n return this.queue.isReady().then(() => {\n this.failedReason = null;\n this.finishedOn = null;\n this.processedOn = null;\n this.retriedOn = Date.now();\n\n return scripts.reprocessJob(this, { state: 'failed' }).then(result => {\n if (result === 1) {\n return;\n } else if (result === 0) {\n throw new Error(errors.Messages.RETRY_JOB_NOT_EXIST);\n } else if (result === -1) {\n throw new Error(errors.Messages.RETRY_JOB_IS_LOCKED);\n } else if (result === -2) {\n throw new Error(errors.Messages.RETRY_JOB_NOT_FAILED);\n }\n });\n });\n};\n\n/**\n * Logs one row of log data.\n *\n * @params logRow: string String with log data to be logged.\n *\n */\nJob.prototype.log = function(logRow) {\n return scripts.addLog(this.queue, this.id, logRow);\n};\n\nJob.prototype.isCompleted = function() {\n return this._isDone('completed');\n};\n\nJob.prototype.isFailed = function() {\n return this._isDone('failed');\n};\n\nJob.prototype.isDelayed = function() {\n return this._isDone('delayed');\n};\n\nJob.prototype.isActive = function() {\n return this._isInList('active');\n};\n\nJob.prototype.isWaiting = function() {\n return this._isInList('wait');\n};\n\nJob.prototype.isPaused = function() {\n return this._isInList('paused');\n};\n\nJob.prototype.isStuck = function() {\n return this.getState().then(state => {\n return state === 'stuck';\n });\n};\n\nJob.prototype.isDiscarded = function() {\n return this._discarded;\n};\n\nJob.prototype.getState = function() {\n const fns = [\n { fn: 'isCompleted', state: 'completed' },\n { fn: 'isFailed', state: 'failed' },\n { fn: 'isDelayed', state: 'delayed' },\n { fn: 'isActive', state: 'active' },\n { fn: 'isWaiting', state: 'waiting' },\n { fn: 'isPaused', state: 'paused' }\n ];\n\n return fns\n .reduce((result, fn) => {\n return result.then(state => {\n if (state) {\n return state;\n }\n return this[fn.fn]().then(result => {\n return result ? fn.state : null;\n });\n });\n }, Promise.resolve())\n .then(result => {\n return result ? result : 'stuck';\n });\n};\n\nJob.prototype.remove = function() {\n const queue = this.queue;\n const job = this;\n\n return queue.isReady().then(() => {\n return scripts.remove(queue, job.id).then(removed => {\n if (removed) {\n queue.emit('removed', job);\n } else {\n throw new Error('Could not remove job ' + job.id);\n }\n });\n });\n};\n\n/**\n * Returns a promise the resolves when the job has finished. (completed or failed).\n */\nJob.prototype.finished = async function() {\n await Promise.all([\n this.queue._registerEvent('global:completed'),\n this.queue._registerEvent('global:failed')\n ]);\n\n await this.queue.isReady();\n\n const status = await scripts.isFinished(this);\n const finished = status > 0;\n if (finished) {\n const job = await Job.fromId(this.queue, this.id);\n if (status == 2) {\n throw new Error(job.failedReason);\n } else {\n return job.returnvalue;\n }\n } else {\n return new Promise((resolve, reject) => {\n const onCompleted = (jobId, resultValue) => {\n if (String(jobId) === String(this.id)) {\n let result = void 0;\n try {\n if (typeof resultValue === 'string') {\n result = JSON.parse(resultValue);\n }\n } catch (err) {\n //swallow exception because the resultValue got corrupted somehow.\n debuglog('corrupted resultValue: ' + resultValue, err);\n }\n resolve(result);\n removeListeners();\n }\n };\n\n const onFailed = (jobId, failedReason) => {\n if (String(jobId) === String(this.id)) {\n reject(new Error(failedReason));\n removeListeners();\n }\n };\n\n this.queue.on('global:completed', onCompleted);\n this.queue.on('global:failed', onFailed);\n\n const removeListeners = () => {\n clearInterval(interval);\n this.queue.removeListener('global:completed', onCompleted);\n this.queue.removeListener('global:failed', onFailed);\n };\n\n //\n // Watchdog\n //\n const interval = setInterval(() => {\n if (this._isQueueClosing()) {\n removeListeners();\n // TODO(manast) maybe we would need a more graceful way to get out of this interval.\n reject(\n new Error('cannot check if job is finished in a closing queue.')\n );\n } else {\n scripts.isFinished(this).then(status => {\n const finished = status > 0;\n if (finished) {\n Job.fromId(this.queue, this.id).then(job => {\n removeListeners();\n if (status == 2) {\n reject(new Error(job.failedReason));\n } else {\n resolve(job.returnvalue);\n }\n });\n }\n });\n }\n }, FINISHED_WATCHDOG);\n });\n }\n};\n\n// -----------------------------------------------------------------------------\n// Private methods\n// -----------------------------------------------------------------------------\nJob.prototype._isQueueClosing = function() {\n return this.queue.closing;\n};\n\nJob.prototype._isDone = function(list) {\n return this.queue.client\n .zscore(this.queue.toKey(list), this.id)\n .then(score => {\n return score !== null;\n });\n};\n\nJob.prototype._isInList = function(list) {\n return scripts.isJobInList(\n this.queue.client,\n this.queue.toKey(list),\n this.id\n );\n};\n\nJob.prototype._saveAttempt = function(multi, err) {\n this.attemptsMade++;\n\n this.stacktrace = this.stacktrace || [];\n\n if (err && err.stack) {\n this.stacktrace.push(err.stack);\n if (this.opts.stackTraceLimit) {\n this.stacktrace = this.stacktrace.slice(-this.opts.stackTraceLimit);\n }\n }\n\n const args = scripts.saveStacktraceArgs(\n this,\n JSON.stringify(this.stacktrace),\n err && err.message,\n );\n\n multi.saveStacktrace(args);\n};\n\nJob.fromJSON = function(queue, json, jobId) {\n const opts = JSON.parse(json.opts || '{}');\n const data = opts.preventParsingData\n ? json.data\n : JSON.parse(json.data || '{}');\n\n const job = new Job(queue, json.name || Job.DEFAULT_JOB_NAME, data, opts);\n\n job.id = json.id || jobId;\n\n try {\n job._progress = JSON.parse(json.progress || 0);\n } catch (err) {\n console.error(\n `Error parsing progress ${json.progress} with ${err.message}`\n );\n }\n\n job.delay = parseInt(json.delay);\n job.timestamp = parseInt(json.timestamp);\n if (json.finishedOn) {\n job.finishedOn = parseInt(json.finishedOn);\n }\n\n if (json.processedOn) {\n job.processedOn = parseInt(json.processedOn);\n }\n\n if (json.retriedOn) {\n job.retriedOn = parseInt(json.retriedOn);\n }\n\n job.failedReason = json.failedReason;\n job.attemptsMade = parseInt(json.attemptsMade || 0);\n\n job.stacktrace = getTraces(json.stacktrace);\n\n if (typeof json.returnvalue === 'string') {\n job.returnvalue = getReturnValue(json.returnvalue);\n }\n\n if (json.deid) {\n job.debounceId = json.deid;\n }\n\n return job;\n};\n\nfunction getTraces(stacktrace) {\n const _traces = utils.tryCatch(JSON.parse, JSON, [stacktrace]);\n\n if (_traces === utils.errorObject || !(_traces instanceof Array)) {\n return [];\n } else {\n return _traces;\n }\n}\n\nfunction getReturnValue(_value) {\n const value = utils.tryCatch(JSON.parse, JSON, [_value]);\n if (value !== utils.errorObject) {\n return value;\n } else {\n debuglog('corrupted returnvalue: ' + _value, value);\n }\n}\n\nmodule.exports = Job;\n" }, { "path": "lib/p-timeout.js", "content": "// Extracted from p-timeout https://github.com/sindresorhus/p-timeout\n// as it is not commonjs compatible. This is version 5.0.2\n'use strict';\n\nclass TimeoutError extends Error {\n constructor(message) {\n super(message);\n this.name = 'TimeoutError';\n }\n}\n\nmodule.exports.TimeoutError = TimeoutError;\n\nmodule.exports.pTimeout = function pTimeout(\n promise,\n milliseconds,\n fallback,\n options\n) {\n let timer;\n const cancelablePromise = new Promise((resolve, reject) => {\n if (typeof milliseconds !== 'number' || Math.sign(milliseconds) !== 1) {\n throw new TypeError(\n `Expected \\`milliseconds\\` to be a positive number, got \\`${milliseconds}\\``\n );\n }\n\n if (milliseconds === Number.POSITIVE_INFINITY) {\n resolve(promise);\n return;\n }\n\n options = {\n customTimers: { setTimeout, clearTimeout },\n ...options\n };\n\n timer = options.customTimers.setTimeout.call(\n undefined,\n () => {\n if (typeof fallback === 'function') {\n try {\n resolve(fallback());\n } catch (error) {\n reject(error);\n }\n\n return;\n }\n\n const message =\n typeof fallback === 'string'\n ? fallback\n : `Promise timed out after ${milliseconds} milliseconds`;\n const timeoutError =\n fallback instanceof Error ? fallback : new TimeoutError(message);\n\n if (typeof promise.cancel === 'function') {\n promise.cancel();\n }\n\n reject(timeoutError);\n },\n milliseconds\n );\n\n (async () => {\n try {\n resolve(await promise);\n } catch (error) {\n reject(error);\n } finally {\n options.customTimers.clearTimeout.call(undefined, timer);\n }\n })();\n });\n\n cancelablePromise['clear'] = () => {\n clearTimeout(timer);\n timer = undefined;\n };\n\n return cancelablePromise;\n};\n" }, { "path": "lib/process/child-pool.js", "content": "'use strict';\n\nconst fork = require('child_process').fork;\nconst path = require('path');\nconst _ = require('lodash');\nconst getPort = require('get-port');\nconst { killAsync } = require('./utils');\n\nconst CHILD_KILL_TIMEOUT = 30000;\n\nconst ChildPool = function ChildPool() {\n if (!(this instanceof ChildPool)) {\n return new ChildPool();\n }\n\n this.retained = {};\n this.free = {};\n};\n\nconst convertExecArgv = function(execArgv) {\n const standard = [];\n const promises = [];\n\n _.forEach(execArgv, arg => {\n if (arg.indexOf('--inspect') === -1) {\n standard.push(arg);\n } else {\n const argName = arg.split('=')[0];\n promises.push(\n getPort().then(port => {\n return `${argName}=${port}`;\n })\n );\n }\n });\n\n return Promise.all(promises).then(convertedArgs => {\n return standard.concat(convertedArgs);\n });\n};\n\nChildPool.prototype.retain = function(processFile) {\n const _this = this;\n let child = _this.getFree(processFile).pop();\n\n if (child) {\n _this.retained[child.pid] = child;\n return Promise.resolve(child);\n }\n\n return convertExecArgv(process.execArgv).then(execArgv => {\n child = fork(path.join(__dirname, './master.js'), {\n execArgv\n });\n child.processFile = processFile;\n\n _this.retained[child.pid] = child;\n\n child.on('exit', _this.remove.bind(_this, child));\n\n return initChild(child, child.processFile)\n .then(() => {\n return child;\n })\n .catch(err => {\n this.remove(child);\n throw err;\n });\n });\n};\n\nChildPool.prototype.release = function(child) {\n delete this.retained[child.pid];\n this.getFree(child.processFile).push(child);\n};\n\nChildPool.prototype.remove = function(child) {\n delete this.retained[child.pid];\n\n const free = this.getFree(child.processFile);\n\n const childIndex = free.indexOf(child);\n if (childIndex > -1) {\n free.splice(childIndex, 1);\n }\n};\n\nChildPool.prototype.kill = function(child, signal) {\n this.remove(child);\n return killAsync(child, signal || 'SIGKILL', CHILD_KILL_TIMEOUT);\n};\n\nChildPool.prototype.clean = function() {\n const children = _.values(this.retained).concat(this.getAllFree());\n this.retained = {};\n this.free = {};\n\n const allKillPromises = [];\n children.forEach(child => {\n allKillPromises.push(this.kill(child, 'SIGTERM'));\n });\n return Promise.all(allKillPromises).then(() => {});\n};\n\nChildPool.prototype.getFree = function(id) {\n return (this.free[id] = this.free[id] || []);\n};\n\nChildPool.prototype.getAllFree = function() {\n return _.flatten(_.values(this.free));\n};\n\nasync function initChild(child, processFile) {\n const onComplete = new Promise((resolve, reject) => {\n const onMessageHandler = msg => {\n if (msg.cmd === 'init-complete') {\n resolve();\n } else if (msg.cmd === 'error') {\n reject(msg.error);\n }\n child.off('message', onMessageHandler);\n };\n child.on('message', onMessageHandler);\n });\n\n await new Promise(resolve =>\n child.send({ cmd: 'init', value: processFile }, resolve)\n );\n await onComplete;\n}\nfunction ChildPoolSingleton(isSharedChildPool = false) {\n if (isSharedChildPool === false) {\n return new ChildPool();\n } else if (\n !(this instanceof ChildPool) &&\n ChildPoolSingleton.instance === undefined\n ) {\n ChildPoolSingleton.instance = new ChildPool();\n }\n\n return ChildPoolSingleton.instance;\n}\n\nmodule.exports = ChildPoolSingleton;\n" }, { "path": "lib/process/master.js", "content": "/**\n * Master of child processes. Handles communication between the\n * processor and the main process.\n *\n */\n'use strict';\n\nlet status;\nlet processor;\nlet currentJobPromise;\n\nconst { promisify } = require('util');\nconst { asyncSend } = require('./utils');\n\n// https://stackoverflow.com/questions/18391212/is-it-not-possible-to-stringify-an-error-using-json-stringify\nif (!('toJSON' in Error.prototype)) {\n Object.defineProperty(Error.prototype, 'toJSON', {\n value: function() {\n const alt = {};\n\n Object.getOwnPropertyNames(this).forEach(function(key) {\n alt[key] = this[key];\n }, this);\n\n return alt;\n },\n configurable: true,\n writable: true\n });\n}\n\nasync function waitForCurrentJobAndExit() {\n status = 'TERMINATING';\n try {\n await currentJobPromise;\n } finally {\n // it's an exit handler\n // eslint-disable-next-line no-process-exit\n process.exit(process.exitCode || 0);\n }\n}\n\nprocess.on('SIGTERM', waitForCurrentJobAndExit);\nprocess.on('SIGINT', waitForCurrentJobAndExit);\n\nprocess.on('message', msg => {\n switch (msg.cmd) {\n case 'init':\n try {\n processor = require(msg.value);\n } catch (err) {\n status = 'Errored';\n err.message = `Error loading process file ${msg.value}. ${err.message}`;\n return process.send({\n cmd: 'error',\n error: err\n });\n }\n\n if (processor.default) {\n // support es2015 module.\n processor = processor.default;\n }\n if (processor.length > 1) {\n processor = promisify(processor);\n } else {\n const origProcessor = processor;\n processor = function() {\n try {\n return Promise.resolve(origProcessor.apply(null, arguments));\n } catch (err) {\n return Promise.reject(err);\n }\n };\n }\n status = 'IDLE';\n process.send({\n cmd: 'init-complete'\n });\n break;\n\n case 'start':\n if (status !== 'IDLE') {\n return process.send({\n cmd: 'error',\n err: new Error('cannot start a not idling child process')\n });\n }\n status = 'STARTED';\n currentJobPromise = (async () => {\n try {\n const result = (await processor(wrapJob(msg.job))) || {};\n await asyncSend(process, {\n cmd: 'completed',\n value: result\n });\n } catch (err) {\n if (!err.message) {\n // eslint-disable-next-line no-ex-assign\n err = new Error(err);\n }\n await asyncSend(process, {\n cmd: 'failed',\n value: err\n });\n } finally {\n status = 'IDLE';\n currentJobPromise = null;\n }\n })();\n break;\n case 'stop':\n break;\n }\n});\n\n/*eslint no-process-exit: \"off\"*/\nprocess.on('uncaughtException', err => {\n if (!err.message) {\n err = new Error(err);\n }\n process.send({\n cmd: 'failed',\n value: err\n });\n\n // An uncaughException leaves this process in a potentially undetermined state so\n // we must exit\n process.exit(-1);\n});\n\n/**\n * Enhance the given job argument with some functions\n * that can be called from the sandboxed job processor.\n *\n * Note, the `job` argument is a JSON deserialized message\n * from the main node process to this forked child process,\n * the functions on the original job object are not in tact.\n * The wrapped job adds back some of those original functions.\n */\nfunction wrapJob(job) {\n /*\n * Emulate the real job `progress` function.\n * If no argument is given, it behaves as a sync getter.\n * If an argument is given, it behaves as an async setter.\n */\n let progressValue = job.progress;\n job.progress = function(progress) {\n if (progress) {\n // Locally store reference to new progress value\n // so that we can return it from this process synchronously.\n progressValue = progress;\n // Send message to update job progress.\n return asyncSend(process, {\n cmd: 'progress',\n value: progress\n });\n } else {\n // Return the last known progress value.\n return progressValue;\n }\n };\n /**\n * Update job info\n */\n job.update = function(data) {\n process.send({\n cmd: 'update',\n value: data\n });\n };\n /*\n * Emulate the real job `log` function.\n */\n job.log = function(row) {\n return asyncSend(process, {\n cmd: 'log',\n value: row\n });\n };\n /*\n * Emulate the real job `update` function.\n */\n job.update = function(data) {\n process.send({\n cmd: 'update',\n value: data\n });\n job.data = data;\n };\n /*\n * Emulate the real job `discard` function.\n */\n job.discard = function() {\n process.send({\n cmd: 'discard'\n });\n };\n return job;\n}\n" }, { "path": "lib/process/sandbox.js", "content": "'use strict';\n\nconst { asyncSend } = require('./utils');\n\nmodule.exports = function(processFile, childPool) {\n return function process(job) {\n return childPool.retain(processFile).then(async child => {\n let msgHandler;\n let exitHandler;\n\n await asyncSend(child, {\n cmd: 'start',\n job: job\n });\n\n const done = new Promise((resolve, reject) => {\n msgHandler = function(msg) {\n switch (msg.cmd) {\n case 'completed':\n resolve(msg.value);\n break;\n case 'failed':\n case 'error': {\n const err = new Error();\n Object.assign(err, msg.value);\n reject(err);\n break;\n }\n case 'progress':\n job.progress(msg.value);\n break;\n case 'update':\n job.update(msg.value);\n break;\n case 'discard':\n job.discard();\n break;\n case 'log':\n job.log(msg.value);\n break;\n }\n };\n\n exitHandler = (exitCode, signal) => {\n reject(\n new Error(\n 'Unexpected exit code: ' + exitCode + ' signal: ' + signal\n )\n );\n };\n\n child.on('message', msgHandler);\n child.on('exit', exitHandler);\n });\n\n return done.finally(() => {\n child.removeListener('message', msgHandler);\n child.removeListener('exit', exitHandler);\n\n if (child.exitCode !== null || /SIG.*/.test(child.signalCode)) {\n childPool.remove(child);\n } else {\n childPool.release(child);\n }\n });\n });\n };\n};\n" }, { "path": "lib/process/utils.js", "content": "'use strict';\n\nfunction hasProcessExited(child) {\n return !!(child.exitCode !== null || child.signalCode);\n}\n\nfunction onExitOnce(child) {\n return new Promise(resolve => {\n child.once('exit', () => resolve());\n });\n}\n\n/**\n * Sends a kill signal to a child resolving when the child has exited,\n * resorting to SIGKILL if the given timeout is reached\n *\n * @param {ChildProcess} child\n * @param {'SIGTERM' | 'SIGKILL'} [signal] initial signal to use\n * @param {number} [timeoutMs] time to wait until sending SIGKILL\n *\n * @returns {Promise} the killed child\n */\nfunction killAsync(child, signal, timeoutMs) {\n if (hasProcessExited(child)) {\n return Promise.resolve(child);\n }\n\n // catch any new on exit\n let onExit = onExitOnce(child);\n\n child.kill(signal || 'SIGKILL');\n\n if (timeoutMs === 0 || isFinite(timeoutMs)) {\n const timeout = setTimeout(() => {\n if (!hasProcessExited(child)) {\n child.kill('SIGKILL');\n }\n }, timeoutMs);\n\n onExit = onExit.then(() => {\n clearTimeout(timeout);\n });\n }\n return onExit;\n}\n\n/*\n asyncSend\n Same as process.send but waits until the send is complete\n the async version is used below because otherwise\n the termination handler may exit before the parent\n process has recived the messages it requires\n */\n\nconst asyncSend = (proc, msg) => {\n return new Promise((resolve, reject) => {\n proc.send(msg, err => {\n if (err) {\n reject(err);\n } else {\n resolve();\n }\n });\n });\n};\n\nmodule.exports = {\n killAsync,\n asyncSend\n};\n" }, { "path": "lib/queue.js", "content": "'use strict';\n\nconst Redis = require('ioredis');\nconst EventEmitter = require('events');\n\nconst _ = require('lodash');\n\nconst fs = require('fs');\nconst path = require('path');\nconst util = require('util');\nconst url = require('url');\nconst Job = require('./job');\nconst scripts = require('./scripts');\nconst errors = require('./errors');\nconst utils = require('./utils');\n\nconst TimerManager = require('./timer-manager');\nconst { promisify } = require('util');\nconst { pTimeout } = require('./p-timeout');\nconst semver = require('semver');\nconst debuglog = require('util').debuglog('bull');\nconst uuid = require('uuid');\n\nconst commands = require('./scripts/');\n\n/**\n Gets or creates a new Queue with the given name.\n\n The Queue keeps 6 data structures:\n - wait (list)\n - active (list)\n - delayed (zset)\n - priority (zset)\n - completed (zset)\n - failed (zset)\n\n --> priorities -- > completed\n / | /\n job -> wait -> active\n \\ ^ \\\n v | -- > failed\n delayed\n*/\n\n/**\n Delayed jobs are jobs that cannot be executed until a certain time in\n ms has passed since they were added to the queue.\n The mechanism is simple, a delayedTimestamp variable holds the next\n known timestamp that is on the delayed set (or MAX_TIMEOUT_MS if none).\n\n When the current job has finalized the variable is checked, if\n no delayed job has to be executed yet a setTimeout is set so that a\n delayed job is processed after timing out.\n*/\nconst MINIMUM_REDIS_VERSION = '2.8.18';\n\n/*\n interface QueueOptions {\n prefix?: string = 'bull',\n limiter?: RateLimiter,\n redis : RedisOpts, // ioredis defaults,\n createClient?: (type: enum('client', 'subscriber'), redisOpts?: RedisOpts) => redisClient,\n defaultJobOptions?: JobOptions,\n\n // Advanced settings\n settings?: QueueSettings {\n lockDuration?: number = 30000,\n lockRenewTime?: number = lockDuration / 2,\n stalledInterval?: number = 30000,\n maxStalledCount?: number = 1, // The maximum number of times a job can be recovered from the 'stalled' state\n guardInterval?: number = 5000,\n retryProcessDelay?: number = 5000,\n drainDelay?: number = 5\n isSharedChildPool?: boolean = false\n }\n }\n\n interface RateLimiter {\n max: number, // Number of jobs\n duration: number, // per duration milliseconds\n }\n*/\n\n// Queue(name: string, url?, opts?)\nconst Queue = function Queue(name, url, opts) {\n if (!(this instanceof Queue)) {\n return new Queue(name, url, opts);\n }\n\n if (_.isString(url)) {\n const clonedOpts = _.cloneDeep(opts || {});\n opts = {\n ...clonedOpts,\n redis: {\n ...redisOptsFromUrl(url),\n ...clonedOpts.redis\n }\n };\n } else {\n opts = _.cloneDeep(url || {});\n }\n\n if (!_.isObject(opts)) {\n throw TypeError('Options must be a valid object');\n }\n\n if (opts.limiter) {\n if (opts.limiter.max && opts.limiter.duration) {\n this.limiter = opts.limiter;\n } else {\n throw new TypeError('Limiter requires `max` and `duration` options');\n }\n }\n\n if (opts.defaultJobOptions) {\n this.defaultJobOptions = opts.defaultJobOptions;\n }\n\n this.name = name;\n this.token = uuid.v4();\n\n opts.redis = {\n enableReadyCheck: false,\n ...(_.isString(opts.redis)\n ? { ...redisOptsFromUrl(opts.redis) }\n : opts.redis)\n };\n\n _.defaults(opts.redis, {\n port: 6379,\n host: '127.0.0.1',\n db: opts.redis.db || opts.redis.DB,\n retryStrategy: function(times) {\n return Math.min(Math.exp(times), 20000);\n }\n });\n\n this.keyPrefix = opts.redis.keyPrefix || opts.prefix || 'bull';\n\n //\n // We cannot use ioredis keyPrefix feature since we\n // create keys dynamically in lua scripts.\n //\n delete opts.redis.keyPrefix;\n\n this.clients = [];\n\n const loadCommands = (providedScripts, client) => {\n const finalScripts = providedScripts || scripts;\n for (const property in finalScripts) {\n // Only define the command if not already defined\n if (!client[finalScripts[property].name]) {\n client.defineCommand(finalScripts[property].name, {\n numberOfKeys: finalScripts[property].keys,\n lua: finalScripts[property].content\n });\n }\n }\n };\n\n const lazyClient = redisClientGetter(this, opts, (type, client) => {\n // bubble up Redis error events\n const handler = this.emit.bind(this, 'error');\n client.on('error', handler);\n this.once('close', () => client.removeListener('error', handler));\n\n if (type === 'client') {\n this._initializing = (async () => loadCommands(commands, client))().then(\n () => {\n debuglog(name + ' queue ready');\n },\n err => {\n this.emit('error', new Error('Error initializing Lua scripts'));\n throw err;\n }\n );\n\n this._initializing.catch((/*err*/) => {});\n }\n });\n\n Object.defineProperties(this, {\n //\n // Queue client (used to add jobs, pause queues, etc);\n //\n client: {\n get: lazyClient('client')\n },\n //\n // Event subscriber client (receive messages from other instance of the queue)\n //\n eclient: {\n get: lazyClient('subscriber')\n },\n bclient: {\n get: lazyClient('bclient')\n }\n });\n\n if (opts.skipVersionCheck !== true) {\n getRedisVersion(this.client)\n .then(version => {\n if (semver.lt(version, MINIMUM_REDIS_VERSION)) {\n this.emit(\n 'error',\n new Error(\n 'Redis version needs to be greater than ' +\n MINIMUM_REDIS_VERSION +\n '. Current: ' +\n version\n )\n );\n }\n })\n .catch((/*err*/) => {\n // Ignore this error.\n });\n }\n\n this.handlers = {};\n this.delayTimer;\n this.processing = [];\n this.retrieving = 0;\n this.drained = true;\n\n this.settings = _.defaults(opts.settings, {\n lockDuration: 30000,\n stalledInterval: 30000,\n maxStalledCount: 1,\n guardInterval: 5000,\n retryProcessDelay: 5000,\n drainDelay: 5,\n backoffStrategies: {},\n isSharedChildPool: false\n });\n\n this.metrics = opts.metrics;\n\n this.settings.lockRenewTime =\n this.settings.lockRenewTime || this.settings.lockDuration / 2;\n\n this.on('error', () => {\n // Dummy handler to avoid process to exit with an unhandled exception.\n });\n\n // keeps track of active timers. used by close() to\n // ensure that disconnect() is deferred until all\n // scheduled redis commands have been executed\n this.timers = new TimerManager();\n\n // Bind these methods to avoid constant rebinding and/or creating closures\n // in processJobs etc.\n this.moveUnlockedJobsToWait = this.moveUnlockedJobsToWait.bind(this);\n this.processJob = this.processJob.bind(this);\n this.getJobFromId = Job.fromId.bind(null, this);\n\n const keys = {};\n _.each(\n [\n '',\n 'active',\n 'wait',\n 'waiting',\n 'paused',\n 'resumed',\n 'meta-paused',\n 'active',\n 'id',\n 'delayed',\n 'priority',\n 'stalled-check',\n 'completed',\n 'failed',\n 'stalled',\n 'repeat',\n 'limiter',\n 'drained',\n 'duplicated',\n 'progress',\n 'de' // debounce key\n ],\n key => {\n keys[key] = this.toKey(key);\n }\n );\n this.keys = keys;\n};\n\nfunction redisClientGetter(queue, options, initCallback) {\n const createClient = _.isFunction(options.createClient)\n ? options.createClient\n : function(type, config) {\n if (['bclient', 'subscriber'].includes(type)) {\n return new Redis({ ...config, maxRetriesPerRequest: null });\n } else {\n return new Redis(config);\n }\n };\n\n const connections = {};\n\n return function(type) {\n return function() {\n // Memoized connection\n if (connections[type] != null) {\n return connections[type];\n }\n const clientOptions = _.assign({}, options.redis);\n\n const client = (connections[type] = createClient(type, clientOptions));\n\n const opts = client.options.redisOptions || client.options;\n\n if (\n ['bclient', 'subscriber'].includes(type) &&\n (opts.enableReadyCheck || opts.maxRetriesPerRequest)\n ) {\n throw new Error(errors.Messages.MISSING_REDIS_OPTS);\n }\n\n // Since connections are lazily initialized, we can't check queue.client\n // without initializing a connection. So expose a boolean we can safely\n // query.\n queue[type + 'Initialized'] = true;\n\n if (!options.createClient) {\n queue.clients.push(client);\n }\n return initCallback(type, client), client;\n };\n };\n}\n\nfunction redisOptsFromUrl(urlString) {\n let redisOpts = {};\n try {\n const redisUrl = url.parse(urlString, true, true);\n redisOpts.port = parseInt(redisUrl.port || '6379', 10);\n redisOpts.host = redisUrl.hostname;\n redisOpts.db = redisUrl.pathname ? redisUrl.pathname.split('/')[1] : 0;\n if (redisUrl.auth) {\n const columnIndex = redisUrl.auth.indexOf(':');\n redisOpts.password = redisUrl.auth.slice(columnIndex + 1);\n if (columnIndex > 0) {\n redisOpts.username = redisUrl.auth.slice(0, columnIndex);\n }\n }\n\n if (redisUrl.query) {\n redisOpts = { ...redisOpts, ...redisUrl.query };\n }\n } catch (e) {\n throw new Error(e.message);\n }\n return redisOpts;\n}\n\nutil.inherits(Queue, EventEmitter);\n\n//\n// Extend Queue with \"aspects\"\n//\nrequire('./getters')(Queue);\nrequire('./worker')(Queue);\nrequire('./repeatable')(Queue);\n\n// --\nQueue.prototype.off = Queue.prototype.removeListener;\n\nconst _on = Queue.prototype.on;\n\nQueue.prototype.on = function(eventName) {\n this._registerEvent(eventName);\n return _on.apply(this, arguments);\n};\n\nconst _once = Queue.prototype.once;\n\nQueue.prototype.once = function(eventName) {\n this._registerEvent(eventName);\n return _once.apply(this, arguments);\n};\n\nQueue.prototype._initProcess = function() {\n if (!this._initializingProcess) {\n //\n // Only setup listeners if .on/.addEventListener called, or process function defined.\n //\n this.delayedTimestamp = Number.MAX_VALUE;\n this._initializingProcess = this.isReady()\n .then(() => {\n return this._registerEvent('delayed');\n })\n .then(() => {\n return this.updateDelayTimer();\n });\n\n this.errorRetryTimer = {};\n }\n\n return this._initializingProcess;\n};\n\nQueue.prototype._setupQueueEventListeners = function() {\n /*\n if(eventName !== 'cleaned' && eventName !== 'error'){\n args[0] = Job.fromJSON(this, args[0]);\n }\n */\n\n const activeKey = this.keys.active;\n const stalledKey = this.keys.stalled;\n const progressKey = this.keys.progress;\n const delayedKey = this.keys.delayed;\n const pausedKey = this.keys.paused;\n const resumedKey = this.keys.resumed;\n const waitingKey = this.keys.waiting;\n const completedKey = this.keys.completed;\n const failedKey = this.keys.failed;\n const drainedKey = this.keys.drained;\n const duplicatedKey = this.keys.duplicated;\n const debouncedKey = this.keys.de + 'bounced';\n\n const pmessageHandler = (pattern, channel, message) => {\n const keyAndToken = channel.split('@');\n const key = keyAndToken[0];\n const token = keyAndToken[1];\n switch (key) {\n case activeKey:\n utils.emitSafe(this, 'global:active', message, 'waiting');\n break;\n case waitingKey:\n if (this.token === token) {\n utils.emitSafe(this, 'waiting', message, null);\n }\n token && utils.emitSafe(this, 'global:waiting', message, null);\n break;\n case stalledKey:\n if (this.token === token) {\n utils.emitSafe(this, 'stalled', message);\n }\n utils.emitSafe(this, 'global:stalled', message);\n break;\n case duplicatedKey:\n if (this.token === token) {\n utils.emitSafe(this, 'duplicated', message);\n }\n utils.emitSafe(this, 'global:duplicated', message);\n break;\n case debouncedKey:\n if (this.token === token) {\n utils.emitSafe(this, 'debounced', message);\n }\n utils.emitSafe(this, 'global:debounced', message);\n break;\n }\n };\n\n const messageHandler = (channel, message) => {\n const key = channel.split('@')[0];\n switch (key) {\n case progressKey: {\n // New way to send progress message data\n try {\n const { progress, jobId } = JSON.parse(message);\n utils.emitSafe(this, 'global:progress', jobId, progress);\n } catch (err) {\n // If we fail we should try to parse the data using the deprecated method\n const commaPos = message.indexOf(',');\n const jobId = message.substring(0, commaPos);\n const progress = message.substring(commaPos + 1);\n utils.emitSafe(this, 'global:progress', jobId, JSON.parse(progress));\n }\n break;\n }\n case delayedKey: {\n const newDelayedTimestamp = _.ceil(message);\n if (newDelayedTimestamp < this.delayedTimestamp) {\n // The new delayed timestamp is before the currently newest known delayed timestamp\n // Assume this is the new delayed timestamp and call `updateDelayTimer()` to process any delayed jobs\n // This will also update the `delayedTimestamp`\n this.delayedTimestamp = newDelayedTimestamp;\n\n this.updateDelayTimer();\n }\n break;\n }\n case pausedKey:\n case resumedKey:\n utils.emitSafe(this, 'global:' + message);\n break;\n case completedKey: {\n const data = JSON.parse(message);\n utils.emitSafe(\n this,\n 'global:completed',\n data.jobId,\n data.val,\n 'active'\n );\n break;\n }\n case failedKey: {\n const data = JSON.parse(message);\n utils.emitSafe(this, 'global:failed', data.jobId, data.val, 'active');\n break;\n }\n case drainedKey:\n utils.emitSafe(this, 'global:drained');\n break;\n }\n };\n\n this.eclient.on('pmessage', pmessageHandler);\n this.eclient.on('message', messageHandler);\n\n this.once('close', () => {\n this.eclient.removeListener('pmessage', pmessageHandler);\n this.eclient.removeListener('message', messageHandler);\n });\n};\n\nQueue.prototype._registerEvent = function(eventName) {\n const internalEvents = ['waiting', 'delayed', 'duplicated', 'debounced'];\n\n if (\n eventName.startsWith('global:') ||\n internalEvents.indexOf(eventName) !== -1\n ) {\n if (!this.registeredEvents) {\n this._setupQueueEventListeners();\n this.registeredEvents = this.registeredEvents || {};\n }\n\n const _eventName = eventName.replace('global:', '');\n\n if (!this.registeredEvents[_eventName]) {\n return utils\n .isRedisReady(this.eclient)\n .then(() => {\n const channel = this.toKey(_eventName);\n if (['active', 'waiting', 'stalled', 'duplicated', 'debounced'].indexOf(_eventName) !== -1) {\n return (this.registeredEvents[_eventName] = this.eclient.psubscribe(\n channel + '*'\n ));\n } else {\n return (this.registeredEvents[_eventName] = this.eclient.subscribe(\n channel\n ));\n }\n })\n .then(() => {\n utils.emitSafe(this, 'registered:' + eventName);\n });\n } else {\n return this.registeredEvents[_eventName];\n }\n }\n return Promise.resolve();\n};\n\nQueue.ErrorMessages = errors.Messages;\n\nQueue.prototype.isReady = async function() {\n await this._initializing;\n return this;\n};\n\nasync function redisClientDisconnect(client) {\n if (client.status !== 'end') {\n let _resolve, _reject;\n return new Promise((resolve, reject) => {\n _resolve = resolve;\n _reject = reject;\n client.once('end', _resolve);\n\n pTimeout(\n client.quit().catch(err => {\n if (err.message !== 'Connection is closed.') {\n throw err;\n }\n }),\n 500\n )\n .catch(() => {\n // Ignore timeout error\n })\n .finally(() => {\n client.once('error', _reject);\n\n client.disconnect();\n if (['connecting', 'reconnecting'].includes(client.status)) {\n resolve();\n }\n });\n }).finally(() => {\n client.removeListener('end', _resolve);\n client.removeListener('error', _reject);\n });\n }\n}\n\nQueue.prototype.disconnect = async function() {\n await Promise.all(\n this.clients.map(client =>\n client.blocked ? client.disconnect() : redisClientDisconnect(client)\n )\n );\n};\n\nQueue.prototype.removeJobs = function(pattern) {\n return Job.remove(this, pattern);\n};\n\nQueue.prototype.close = function(doNotWaitJobs) {\n let isReady = true;\n if (this.closing) {\n return this.closing;\n }\n\n return (this.closing = this.isReady()\n .then(this._initializingProcess)\n .catch(() => {\n isReady = false;\n })\n .then(() => isReady && this.pause(true, doNotWaitJobs))\n .catch(() => void 0) // Ignore possible error from pause\n .finally(() => this._clearTimers())\n .then(() => {\n if (!this.childPool) {\n return;\n }\n const cleanPromise = this.childPool.clean().catch(() => {\n // Ignore this error and try to close anyway.\n });\n if (doNotWaitJobs) {\n return;\n }\n return cleanPromise;\n })\n .then(\n async () => this.disconnect(),\n err => console.error(err)\n )\n .finally(() => {\n this.closed = true;\n utils.emitSafe(this, 'close');\n }));\n};\n\nQueue.prototype._clearTimers = function() {\n _.each(this.errorRetryTimer, timer => {\n clearTimeout(timer);\n });\n clearTimeout(this.delayTimer);\n clearInterval(this.guardianTimer);\n clearInterval(this.moveUnlockedJobsToWaitInterval);\n this.timers.clearAll();\n return this.timers.whenIdle();\n};\n\n/**\n Processes a job from the queue. The callback is called for every job that\n is dequeued.\n\n @method process\n*/\nQueue.prototype.process = function(name, concurrency, handler) {\n switch (arguments.length) {\n case 1:\n handler = name;\n concurrency = 1;\n name = Job.DEFAULT_JOB_NAME;\n break;\n case 2: // (string, function) or (string, string) or (number, function) or (number, string)\n handler = concurrency;\n if (typeof name === 'string') {\n concurrency = 1;\n } else {\n concurrency = name;\n name = Job.DEFAULT_JOB_NAME;\n }\n break;\n }\n\n this.setHandler(name, handler);\n\n return this._initProcess().then(() => {\n return this.start(concurrency, name);\n });\n};\n\nQueue.prototype.start = function(concurrency, name) {\n return this.run(concurrency, name).catch(err => {\n utils.emitSafe(this, 'error', err, 'error running queue');\n throw err;\n });\n};\n\nQueue.prototype.setHandler = function(name, handler) {\n if (!handler) {\n throw new Error('Cannot set an undefined handler');\n }\n if (this.handlers[name]) {\n throw new Error('Cannot define the same handler twice ' + name);\n }\n\n this.setWorkerName();\n\n if (typeof handler === 'string') {\n const supportedFileTypes = ['.js', '.ts', '.flow', '.cjs'];\n const processorFile =\n handler +\n (supportedFileTypes.includes(path.extname(handler)) ? '' : '.js');\n\n if (!fs.existsSync(processorFile)) {\n throw new Error('File ' + processorFile + ' does not exist');\n }\n const isSharedChildPool = this.settings.isSharedChildPool;\n this.childPool =\n this.childPool || require('./process/child-pool')(isSharedChildPool);\n\n const sandbox = require('./process/sandbox');\n this.handlers[name] = sandbox(handler, this.childPool).bind(this);\n } else {\n handler = handler.bind(this);\n\n if (handler.length > 1) {\n this.handlers[name] = promisify(handler);\n } else {\n this.handlers[name] = function() {\n try {\n return Promise.resolve(handler.apply(null, arguments));\n } catch (err) {\n return Promise.reject(err);\n }\n };\n }\n }\n};\n\n/**\ninterface JobOptions\n{\n attempts: number;\n\n repeat: {\n tz?: string,\n endDate?: Date | string | number\n },\n preventParsingData: boolean;\n}\n*/\n\n/**\n Adds a job to the queue.\n @method add\n @param data: {} Custom data to store for this job. Should be JSON serializable.\n @param opts: JobOptions Options for this job.\n*/\nQueue.prototype.add = function(name, data, opts) {\n if (typeof name !== 'string') {\n opts = data;\n data = name;\n name = Job.DEFAULT_JOB_NAME;\n }\n opts = _.cloneDeep({ ...this.defaultJobOptions, ...opts });\n\n opts.jobId = jobIdForGroup(this.limiter, opts, data);\n\n if (opts.repeat) {\n return this.isReady().then(() => {\n return this.nextRepeatableJob(name, data, opts, true);\n });\n } else {\n return Job.create(this, name, data, opts);\n }\n};\n\n/**\n * Retry all the failed jobs.\n *\n * @param opts.count - number to limit how many jobs will be moved to wait status per iteration\n * @returns\n */\nQueue.prototype.retryJobs = async function(opts = {}) {\n let cursor = 0;\n do {\n cursor = await scripts.retryJobs(this, opts.count);\n } while (cursor);\n};\n\n /**\n * Removes a debounce key.\n *\n * @param id - identifier\n */\n Queue.prototype.removeDebounceKey = (id) => {\n return this.client.del(`${this.keys.de}:${id}`);\n }\n\n/**\n Adds an array of jobs to the queue.\n @method add\n @param jobs: [] The array of jobs to add to the queue. Each job is defined by 3 properties, 'name', 'data' and 'opts'. They follow the same signature as 'Queue.add'.\n*/\nQueue.prototype.addBulk = function(jobs) {\n const decoratedJobs = jobs.map(job => {\n const jobId = jobIdForGroup(this.limiter, job.opts, job.data);\n return {\n ...job,\n name: typeof job.name !== 'string' ? Job.DEFAULT_JOB_NAME : job.name,\n opts: {\n ...this.defaultJobOptions,\n ...job.opts,\n jobId\n }\n };\n });\n return Job.createBulk(this, decoratedJobs);\n};\n/**\n Empties the queue.\n\n Returns a promise that is resolved after the operation has been completed.\n Note that if some other process is adding jobs at the same time as emptying,\n the queues may not be really empty after this method has executed completely.\n Also, if the method does error between emptying the lists and removing all the\n jobs, there will be zombie jobs left in redis.\n\n TODO: Use EVAL to make this operation fully atomic.\n*/\nQueue.prototype.empty = function() {\n const queueKeys = this.keys;\n\n let multi = this.multi();\n\n multi.lrange(queueKeys.wait, 0, -1);\n multi.lrange(queueKeys.paused, 0, -1);\n multi.keys(this.toKey('*:limited'));\n multi.del(\n queueKeys.wait,\n queueKeys.paused,\n queueKeys['meta-paused'],\n queueKeys.delayed,\n queueKeys.priority,\n queueKeys.limiter,\n `${queueKeys.limiter}:index`\n );\n\n return multi.exec().then(res => {\n let [waiting, paused, limited] = res;\n\n waiting = waiting[1];\n paused = paused[1];\n limited = limited[1];\n\n const jobKeys = paused.concat(waiting).map(this.toKey, this);\n\n if (jobKeys.length || limited.length) {\n multi = this.multi();\n\n for (let i = 0; i < jobKeys.length; i += 10000) {\n multi.del.apply(multi, jobKeys.slice(i, i + 10000));\n }\n\n for (let i = 0; i < limited.length; i += 10000) {\n multi.del.apply(multi, limited.slice(i, i + 10000));\n }\n\n return multi.exec();\n }\n });\n};\n\n/**\n Pauses the processing of this queue, locally if true passed, otherwise globally.\n\n For global pause, we use an atomic RENAME operation on the wait queue. Since\n we have blocking calls with BRPOPLPUSH on the wait queue, as long as the queue\n is renamed to 'paused', no new jobs will be processed (the current ones\n will run until finalized).\n\n Adding jobs requires a LUA script to check first if the paused list exist\n and in that case it will add it there instead of the wait list.\n*/\nQueue.prototype.pause = function(isLocal, doNotWaitActive) {\n return this.isReady()\n .then(() => {\n if (isLocal) {\n if (!this.paused) {\n this.paused = new Promise(resolve => {\n this.resumeLocal = function() {\n this.paused = null; // Allow pause to be checked externally for paused state.\n resolve();\n };\n });\n }\n\n if (!this.bclientInitialized) {\n // bclient not yet initialized, so no jobs to wait for\n return;\n }\n\n if (doNotWaitActive) {\n // Force reconnection of blocking connection to abort blocking redis call immediately.\n return redisClientDisconnect(this.bclient).then(() =>\n this.bclient.connect()\n );\n }\n return this.whenCurrentJobsFinished();\n } else {\n return scripts.pause(this, true);\n }\n })\n .then(() => {\n return utils.emitSafe(this, 'paused');\n });\n};\n\nQueue.prototype.resume = function(isLocal /* Optional */) {\n return this.isReady()\n .then(() => {\n if (isLocal) {\n if (this.resumeLocal) {\n this.resumeLocal();\n }\n } else {\n return scripts.pause(this, false);\n }\n })\n .then(() => {\n utils.emitSafe(this, 'resumed');\n });\n};\n\nQueue.prototype.isPaused = async function(isLocal) {\n if (isLocal) {\n return !!this.paused;\n } else {\n await this.isReady();\n const multi = this.multi();\n\n multi.exists(this.keys['meta-paused']);\n\n // For forward compatibility with BullMQ.\n multi.hexists(this.toKey('meta'), 'paused');\n\n const [[, isPaused], [, isPausedNew]] = await multi.exec();\n\n return !!(isPaused || isPausedNew);\n }\n};\n\nQueue.prototype.run = function(concurrency, handlerName) {\n if (!Number.isInteger(concurrency)) {\n throw new Error('Cannot set Float as concurrency');\n }\n const promises = [];\n\n return this.isReady()\n .then(() => {\n return this.moveUnlockedJobsToWait();\n })\n .then(() => {\n return utils.isRedisReady(this.bclient);\n })\n .then(() => {\n while (concurrency--) {\n promises.push(\n new Promise(resolve => {\n this.processJobs(`${handlerName}:${concurrency}`, resolve);\n })\n );\n }\n\n this.startMoveUnlockedJobsToWait();\n\n return Promise.all(promises);\n });\n};\n\n// ---------------------------------------------------------------------\n// Private methods\n// ---------------------------------------------------------------------\n\n/**\n This function updates the delay timer, which is a timer that timeouts\n at the next known delayed job.\n*/\nQueue.prototype.updateDelayTimer = function() {\n if (this.closing) {\n return Promise.resolve();\n }\n\n return scripts\n .updateDelaySet(this, Date.now())\n .then(nextTimestamp => {\n this.delayedTimestamp = nextTimestamp\n ? nextTimestamp / 4096\n : Number.MAX_VALUE;\n\n // Clear any existing update delay timer\n if (this.delayTimer) {\n clearTimeout(this.delayTimer);\n }\n\n // Delay for the next update of delay set\n const delay = _.min([\n this.delayedTimestamp - Date.now(),\n this.settings.guardInterval\n ]);\n\n // Schedule next processing of the delayed jobs\n if (delay <= 0) {\n // Next set of jobs are due right now, process them also\n this.updateDelayTimer();\n } else {\n // Update the delay set when the next job is due\n // or the next guard time\n this.delayTimer = setTimeout(() => this.updateDelayTimer(), delay);\n }\n\n // Silence warnings about promise created but not returned.\n // This isn't an issue since we emit errors.\n // See http://bluebirdjs.com/docs/warning-explanations.html#warning-a-promise-was-created-in-a-handler-but-was-not-returned-from-it\n return null;\n })\n .catch(err => {\n utils.emitSafe(this, 'error', err, 'Error updating the delay timer');\n if (this.delayTimer) {\n clearTimeout(this.delayTimer);\n }\n\n this.delayTimer = setTimeout(\n () => this.updateDelayTimer(),\n this.settings.guardInterval\n );\n });\n};\n\n/**\n * Process jobs that have been added to the active list but are not being\n * processed properly. This can happen due to a process crash in the middle\n * of processing a job, leaving it in 'active' but without a job lock.\n */\nQueue.prototype.moveUnlockedJobsToWait = function() {\n if (this.closing) {\n return Promise.resolve();\n }\n\n return scripts\n .moveUnlockedJobsToWait(this)\n .then(([failed, stalled]) => {\n const handleFailedJobs = failed.map(jobId => {\n return this.getJobFromId(jobId).then(job => {\n utils.emitSafe(\n this,\n 'failed',\n job,\n new Error('job stalled more than allowable limit'),\n 'active'\n );\n return null;\n });\n });\n const handleStalledJobs = stalled.map(jobId => {\n return this.getJobFromId(jobId).then(job => {\n // Do not emit the event if the job was completed by another worker\n if (job !== null) {\n utils.emitSafe(this, 'stalled', job);\n }\n return null;\n });\n });\n return Promise.all(handleFailedJobs.concat(handleStalledJobs));\n })\n .catch(err => {\n utils.emitSafe(\n this,\n 'error',\n err,\n 'Failed to handle unlocked job in active'\n );\n });\n};\n\nQueue.prototype.startMoveUnlockedJobsToWait = function() {\n clearInterval(this.moveUnlockedJobsToWaitInterval);\n if (this.settings.stalledInterval > 0 && !this.closing) {\n this.moveUnlockedJobsToWaitInterval = setInterval(\n this.moveUnlockedJobsToWait,\n this.settings.stalledInterval\n );\n }\n};\n\n/*\n Process jobs. Note last argument 'job' is optional.\n*/\nQueue.prototype.processJobs = function(index, resolve, job) {\n const processJobs = this.processJobs.bind(this, index, resolve);\n process.nextTick(() => {\n this._processJobOnNextTick(processJobs, index, resolve, job);\n });\n};\n\nQueue.prototype._processJobOnNextTick = function(\n processJobs,\n index,\n resolve,\n job\n) {\n if (!this.closing) {\n (this.paused || Promise.resolve())\n .then(() => {\n const gettingNextJob = job ? Promise.resolve(job) : this.getNextJob();\n\n return (this.processing[index] = gettingNextJob\n .then(this.processJob)\n .then(processJobs, err => {\n if (!(this.closing && err.message === 'Connection is closed.')) {\n utils.emitSafe(this, 'error', err, 'Error processing job');\n\n //\n // Wait before trying to process again.\n //\n clearTimeout(this.errorRetryTimer[index]);\n this.errorRetryTimer[index] = setTimeout(() => {\n processJobs();\n }, this.settings.retryProcessDelay);\n }\n return null;\n }));\n })\n .catch(err => {\n utils.emitSafe(this, 'error', err, 'Error processing job');\n });\n } else {\n resolve(this.closing);\n }\n};\n\nQueue.prototype.processJob = function(job, notFetch = false) {\n let lockRenewId;\n let timerStopped = false;\n\n if (!job) {\n return Promise.resolve();\n }\n\n //\n // There are two cases to take into consideration regarding locks.\n // 1) The lock renewer fails to renew a lock, this should make this job\n // unable to complete, since some other worker is also working on it.\n // 2) The lock renewer is called more seldom than the check for stalled\n // jobs, so we can assume the job has been stalled and is already being processed\n // by another worker. See #308\n //\n const lockExtender = () => {\n lockRenewId = this.timers.set(\n 'lockExtender',\n this.settings.lockRenewTime,\n () => {\n scripts\n .extendLock(this, job.id, this.settings.lockDuration)\n .then(lock => {\n if (lock && !timerStopped) {\n lockExtender();\n }\n })\n .catch(err => {\n utils.emitSafe(this, 'lock-extension-failed', job, err);\n });\n }\n );\n };\n\n const timeoutMs = job.opts.timeout;\n\n const stopTimer = () => {\n timerStopped = true;\n this.timers.clear(lockRenewId);\n };\n\n const handleCompleted = result => {\n return job.moveToCompleted(result, undefined, notFetch).then(jobData => {\n utils.emitSafe(this, 'completed', job, result, 'active');\n return jobData ? this.nextJobFromJobData(jobData[0], jobData[1]) : null;\n });\n };\n\n const handleFailed = err => {\n const error = err;\n\n return job.moveToFailed(err).then(jobData => {\n utils.emitSafe(this, 'failed', job, error, 'active');\n return jobData ? this.nextJobFromJobData(jobData[0], jobData[1]) : null;\n });\n };\n\n lockExtender();\n const handler = this.handlers[job.name] || this.handlers['*'];\n\n if (!handler) {\n return handleFailed(\n new Error('Missing process handler for job type ' + job.name)\n );\n } else {\n let jobPromise = handler(job);\n\n if (timeoutMs) {\n jobPromise = pTimeout(jobPromise, timeoutMs);\n }\n\n // Local event with jobPromise so that we can cancel job.\n utils.emitSafe(this, 'active', job, jobPromise, 'waiting');\n\n return jobPromise\n .then(handleCompleted)\n .catch(handleFailed)\n .finally(() => {\n stopTimer();\n });\n }\n};\n\nQueue.prototype.multi = function() {\n return this.client.multi();\n};\n\n/**\n Returns a promise that resolves to the next job in queue.\n*/\nQueue.prototype.getNextJob = async function() {\n if (this.closing) {\n return Promise.resolve();\n }\n\n if (this.drained) {\n //\n // Waiting for new jobs to arrive\n //\n try {\n this.bclient.blocked = true;\n const jobId = await this.bclient.brpoplpush(\n this.keys.wait,\n this.keys.active,\n this.settings.drainDelay\n );\n this.bclient.blocked = false;\n\n if (jobId) {\n return this.moveToActive(jobId);\n }\n } catch (err) {\n // Swallow error if locally paused since we did force a disconnection\n if (!(this.paused && err.message === 'Connection is closed.')) {\n throw err;\n }\n }\n } else {\n return this.moveToActive();\n }\n};\n\nQueue.prototype.moveToActive = async function(jobId) {\n // For manual retrieving jobs we need to wait for the queue to be ready.\n await this.isReady();\n\n return scripts.moveToActive(this, jobId).then(([jobData, jobId]) => {\n return this.nextJobFromJobData(jobData, jobId);\n });\n};\n\nQueue.prototype.nextJobFromJobData = function(jobData, jobId) {\n if (jobData) {\n this.drained = false;\n const job = Job.fromJSON(this, jobData, jobId);\n if (job.opts.repeat) {\n return this.nextRepeatableJob(job.name, job.data, job.opts).then(() => {\n return job;\n });\n }\n return job;\n } else {\n this.drained = true;\n utils.emitSafe(this, 'drained');\n return null;\n }\n};\n\nQueue.prototype.retryJob = function(job) {\n return job.retry();\n};\n\nQueue.prototype.toKey = function(queueType) {\n return [this.keyPrefix, this.name, queueType].join(':');\n};\n\n/*@function clean\n *\n * Cleans jobs from a queue. Similar to remove but keeps jobs within a certain\n * grace period.\n *\n * @param {int} grace - The grace period\n * @param {string} [type=completed] - The type of job to clean. Possible values are completed, wait, active, paused, delayed, failed. Defaults to completed.\n * @param {int} The max number of jobs to clean\n */\nQueue.prototype.clean = function(grace, type, limit) {\n return this.isReady().then(() => {\n if (grace === undefined || grace === null) {\n throw new Error('You must define a grace period.');\n }\n\n if (!type) {\n type = 'completed';\n }\n\n if (\n _.indexOf(\n ['completed', 'wait', 'active', 'paused', 'delayed', 'failed'],\n type\n ) === -1\n ) {\n throw new Error('Cannot clean unknown queue type ' + type);\n }\n\n return scripts\n .cleanJobsInSet(this, type, Date.now() - grace, limit)\n .then(jobs => {\n utils.emitSafe(this, 'cleaned', jobs, type);\n return jobs;\n })\n .catch(err => {\n utils.emitSafe(this, 'error', err);\n throw err;\n });\n });\n};\n\n/* @method obliterate\n *\n * Completely destroys the queue and all of its contents irreversibly.\n * This method will the *pause* the queue and requires that there are no\n * active jobs. It is possible to bypass this requirement, i.e. not\n * having active jobs using the \"force\" option.\n *\n * Note: This operation requires to iterate on all the jobs stored in the queue\n * and can be slow for very large queues.\n *\n * @param { { force: boolean, count: number }} opts. Use force = true to force obliteration even\n * with active jobs in the queue. Use count with the maximun number of deleted keys per iteration,\n * 1000 is the default.\n */\nQueue.prototype.obliterate = async function(opts) {\n await this.pause();\n\n let cursor = 0;\n do {\n cursor = await scripts.obliterate(this, {\n force: false,\n count: 1000,\n ...opts\n });\n } while (cursor);\n};\n\n/**\n * Returns a promise that resolves when active jobs are finished\n *\n * @returns {Promise}\n */\nQueue.prototype.whenCurrentJobsFinished = function() {\n if (!this.bclientInitialized) {\n // bclient not yet initialized, so no jobs to wait for\n return Promise.resolve();\n }\n\n //\n // Force reconnection of blocking connection to abort blocking redis call immediately.\n //\n const forcedReconnection = redisClientDisconnect(this.bclient).then(() => {\n return this.bclient.connect();\n });\n\n return Promise.all(Object.values(this.processing)).then(\n () => forcedReconnection\n );\n};\n\n//\n// Private local functions\n//\n\nfunction getRedisVersion(client) {\n return client.info().then(doc => {\n const prefix = 'redis_version:';\n const lines = doc.split('\\r\\n');\n for (let i = 0; i < lines.length; i++) {\n if (lines[i].indexOf(prefix) === 0) {\n return lines[i].substr(prefix.length);\n }\n }\n });\n}\n\nfunction jobIdForGroup(limiter, opts, data) {\n const jobId = opts && opts.jobId;\n const groupKey = _.get(limiter, 'groupKey');\n if (groupKey) {\n return `${jobId || uuid.v4()}:${_.get(data, groupKey)}`;\n }\n return jobId;\n}\n\nmodule.exports = Queue;\n" }, { "path": "lib/repeatable.js", "content": "'use strict';\n\nconst _ = require('lodash');\nconst parser = require('cron-parser');\nconst crypto = require('crypto');\n\nconst Job = require('./job');\n\nmodule.exports = function(Queue) {\n Queue.prototype.nextRepeatableJob = function(\n name,\n data,\n opts,\n skipCheckExists\n ) {\n const client = this.client;\n const repeat = opts.repeat;\n const prevMillis = opts.prevMillis || 0;\n\n if (!prevMillis && opts.jobId) {\n repeat.jobId = opts.jobId;\n }\n\n const currentCount = repeat.count ? repeat.count + 1 : 1;\n\n if (!_.isUndefined(repeat.limit) && currentCount > repeat.limit) {\n return Promise.resolve();\n }\n\n let now = Date.now();\n\n if (!_.isUndefined(repeat.endDate) && now > new Date(repeat.endDate)) {\n return Promise.resolve();\n }\n\n now = prevMillis < now ? now : prevMillis;\n\n const nextMillis = getNextMillis(now, repeat);\n if (nextMillis) {\n const jobId = repeat.jobId ? repeat.jobId + ':' : ':';\n const repeatKey = getRepeatKey(name, repeat, jobId);\n\n const createNextJob = () => {\n return client.zadd(this.keys.repeat, nextMillis, repeatKey).then(() => {\n //\n // Generate unique job id for this iteration.\n //\n const customId = getRepeatJobId(\n name,\n jobId,\n nextMillis,\n md5(repeatKey)\n );\n now = Date.now();\n const delay = nextMillis - now;\n\n return Job.create(\n this,\n name,\n data,\n _.defaultsDeep(\n {\n repeat: {\n count: currentCount,\n key: repeatKey\n },\n jobId: customId,\n delay: delay < 0 ? 0 : delay,\n timestamp: now,\n prevMillis: nextMillis\n },\n opts\n )\n );\n });\n };\n\n if (skipCheckExists) {\n return createNextJob();\n }\n\n // Check that the repeatable job hasn't been removed\n // TODO: a lua script would be better here\n return client\n .zscore(this.keys.repeat, repeatKey)\n .then(repeatableExists => {\n // The job could have been deleted since this check\n if (repeatableExists) {\n return createNextJob();\n }\n return Promise.resolve();\n });\n } else {\n return Promise.resolve();\n }\n };\n\n Queue.prototype.removeRepeatable = function(name, repeat) {\n if (typeof name !== 'string') {\n repeat = name;\n name = Job.DEFAULT_JOB_NAME;\n }\n\n return this.isReady().then(() => {\n const jobId = repeat.jobId ? repeat.jobId + ':' : ':';\n const repeatJobKey = getRepeatKey(name, repeat, jobId);\n const repeatJobId = getRepeatJobId(name, jobId, '', md5(repeatJobKey));\n const queueKey = this.keys[''];\n return this.client.removeRepeatable(\n this.keys.repeat,\n this.keys.delayed,\n repeatJobId,\n repeatJobKey,\n queueKey\n );\n });\n };\n\n Queue.prototype.removeRepeatableByKey = function(repeatJobKey) {\n const repeatMeta = this._keyToData(repeatJobKey);\n const queueKey = this.keys[''];\n\n const jobId = repeatMeta.id ? repeatMeta.id + ':' : ':';\n const repeatJobId = getRepeatJobId(\n repeatMeta.name || Job.DEFAULT_JOB_NAME,\n jobId,\n '',\n md5(repeatJobKey)\n );\n\n return this.isReady().then(() => {\n return this.client.removeRepeatable(\n this.keys.repeat,\n this.keys.delayed,\n repeatJobId,\n repeatJobKey,\n queueKey\n );\n });\n };\n\n Queue.prototype._keyToData = function(key) {\n const data = key.split(':');\n\n return {\n key: key,\n name: data[0],\n id: data[1] || null,\n endDate: parseInt(data[2]) || null,\n tz: data[3] || null,\n cron: data[4]\n };\n };\n\n Queue.prototype.getRepeatableJobs = function(start, end, asc) {\n const key = this.keys.repeat;\n start = start || 0;\n end = end || -1;\n return (asc\n ? this.client.zrange(key, start, end, 'WITHSCORES')\n : this.client.zrevrange(key, start, end, 'WITHSCORES')\n ).then(result => {\n const jobs = [];\n for (let i = 0; i < result.length; i += 2) {\n const data = this._keyToData(result[i]);\n jobs.push({\n key: data.key,\n name: data.name,\n id: data.id,\n endDate: data.endDate,\n tz: data.cron ? data.tz : null,\n cron: data.cron || null,\n every: !data.cron ? parseInt(data.tz) : null,\n next: parseInt(result[i + 1])\n });\n }\n return jobs;\n });\n };\n\n Queue.prototype.getRepeatableCount = function() {\n return this.client.zcard(this.toKey('repeat'));\n };\n\n function getRepeatJobId(name, jobId, nextMillis, namespace) {\n return 'repeat:' + md5(name + jobId + namespace) + ':' + nextMillis;\n }\n\n function getRepeatKey(name, repeat, jobId) {\n const endDate = repeat.endDate\n ? new Date(repeat.endDate).getTime() + ':'\n : ':';\n const tz = repeat.tz ? repeat.tz + ':' : ':';\n const suffix = repeat.cron ? tz + repeat.cron : String(repeat.every);\n\n return name + ':' + jobId + endDate + suffix;\n }\n\n function getNextMillis(millis, opts) {\n if (opts.cron && opts.every) {\n throw new Error(\n 'Both .cron and .every options are defined for this repeatable job'\n );\n }\n\n if (opts.every) {\n return Math.floor(millis / opts.every) * opts.every + opts.every;\n }\n\n const currentDate =\n opts.startDate && new Date(opts.startDate) > new Date(millis)\n ? new Date(opts.startDate)\n : new Date(millis);\n const interval = parser.parseExpression(\n opts.cron,\n _.defaults(\n {\n currentDate\n },\n opts\n )\n );\n\n try {\n return interval.next().getTime();\n } catch (e) {\n // Ignore error\n }\n }\n\n function md5(str) {\n return crypto\n .createHash('md5')\n .update(str)\n .digest('hex');\n }\n};\n" }, { "path": "lib/scripts.js", "content": "/**\n * Includes all the scripts needed by the queue and jobs.\n */\n\n'use strict';\n\nconst _ = require('lodash');\nconst msgpackr = require('msgpackr');\n\nconst packer = new msgpackr.Packr({\n useRecords: false,\n encodeUndefinedAsNil: true\n});\n\nconst pack = packer.pack;\n\nconst scripts = {\n isJobInList(client, listKey, jobId) {\n return client.isJobInList([listKey, jobId]).then(result => {\n return result === 1;\n });\n },\n\n addJob(client, queue, job, opts) {\n const queueKeys = queue.keys;\n let keys = [\n queueKeys.wait,\n queueKeys.paused,\n queueKeys['meta-paused'],\n queueKeys.id,\n queueKeys.delayed,\n queueKeys.priority\n ];\n\n const args = [\n queueKeys[''],\n _.isUndefined(opts.customJobId) ? '' : opts.customJobId,\n job.name,\n job.data,\n pack(job.opts),\n job.timestamp,\n job.delay,\n job.delay ? job.timestamp + job.delay : 0,\n opts.priority || 0,\n opts.lifo ? 'RPUSH' : 'LPUSH',\n queue.token,\n job.debounceId ? `${queueKeys.de}:${job.debounceId}` : null,\n opts.debounce ? opts.debounce.id : null,\n opts.debounce ? opts.debounce.ttl : null,\n ];\n keys = keys.concat(args);\n return client.addJob(keys);\n },\n\n pause(queue, pause) {\n let src = 'wait',\n dst = 'paused';\n if (!pause) {\n src = 'paused';\n dst = 'wait';\n }\n\n const keys = _.map(\n [src, dst, 'meta-paused', pause ? 'paused' : 'resumed', 'meta'],\n name => {\n return queue.toKey(name);\n }\n );\n\n return queue.client.pause(keys.concat([pause ? 'paused' : 'resumed']));\n },\n\n async addLog(queue, jobId, logRow, keepLogs) {\n const client = await queue.client;\n\n const keys = [queue.toKey(jobId), queue.toKey(jobId) + ':logs'];\n\n const result = await client.addLog(\n keys.concat([jobId, logRow, keepLogs ? keepLogs : ''])\n );\n\n if (result < 0) {\n throw scripts.finishedErrors(result, jobId, 'addLog');\n }\n\n return result;\n },\n\n getCountsPerPriorityArgs(queue, priorities) {\n const keys = [\n queue.keys.wait,\n queue.keys.paused,\n queue.keys['meta-paused'],\n queue.keys.priority\n ];\n\n const args = priorities;\n\n return keys.concat(args);\n },\n\n async getCountsPerPriority(queue, priorities) {\n const client = await queue.client;\n const args = this.getCountsPerPriorityArgs(queue, priorities);\n\n return client.getCountsPerPriority(args);\n },\n\n moveToActive(queue, jobId) {\n const queueKeys = queue.keys;\n const keys = [queueKeys.wait, queueKeys.active, queueKeys.priority];\n\n keys[3] = keys[1] + '@' + queue.token;\n keys[4] = queueKeys.stalled;\n keys[5] = queueKeys.limiter;\n keys[6] = queueKeys.delayed;\n keys[7] = queueKeys.drained;\n\n const args = [\n queueKeys[''],\n queue.token,\n queue.settings.lockDuration,\n Date.now(),\n jobId\n ];\n\n if (queue.limiter) {\n args.push(\n queue.limiter.max,\n queue.limiter.duration,\n !!queue.limiter.bounceBack\n );\n queue.limiter.groupKey && args.push(true);\n }\n\n return queue.client.moveToActive(keys.concat(args)).then(raw2jobData);\n },\n\n updateProgress(job, progress) {\n const queue = job.queue;\n const keys = [job.id, 'progress'].map(name => {\n return queue.toKey(name);\n });\n\n const progressJson = JSON.stringify(progress);\n return queue.client\n .updateProgress(keys, [\n progressJson,\n JSON.stringify({ jobId: job.id, progress })\n ])\n .then(code => {\n if (code < 0) {\n throw scripts.finishedErrors(code, job.id, 'updateProgress');\n }\n queue.emit('progress', job, progress);\n });\n },\n\n updateData(job, data) {\n const queue = job.queue;\n const keys = [job.id].map(name => {\n return queue.toKey(name);\n });\n const dataJson = JSON.stringify(data);\n\n return queue.client.updateData(keys, [dataJson]);\n },\n\n saveStacktraceArgs(\n job,\n stacktrace,\n failedReason\n ) {\n const queue = job.queue;\n\n const keys = [queue.toKey(job.id)];\n\n return keys.concat([stacktrace, failedReason, job.attemptsMade]);\n },\n\n retryJobsArgs(queue, count) {\n const keys = [\n queue.toKey(''),\n queue.toKey('failed'),\n queue.toKey('wait'),\n queue.toKey('meta-paused'),\n queue.toKey('paused')\n ];\n\n const args = [count];\n\n return keys.concat(args);\n },\n\n async retryJobs(queue, count = 1000) {\n const client = await queue.client;\n\n const args = this.retryJobsArgs(queue, count);\n\n return client.retryJobs(args);\n },\n\n moveToFinishedArgs(\n job,\n val,\n propVal,\n shouldRemove,\n target,\n ignoreLock,\n notFetch\n ) {\n const queue = job.queue;\n const queueKeys = queue.keys;\n\n const metricsKey = queue.toKey(`metrics:${target}`);\n\n const keys = [\n queueKeys.active,\n queueKeys[target],\n queue.toKey(job.id),\n queueKeys.wait,\n queueKeys.priority,\n queueKeys.active + '@' + queue.token,\n queueKeys.delayed,\n queueKeys.stalled,\n metricsKey\n ];\n\n const keepJobs = pack(\n typeof shouldRemove === 'object'\n ? shouldRemove\n : typeof shouldRemove === 'number'\n ? { count: shouldRemove }\n : { count: shouldRemove ? 0 : -1 }\n );\n\n const args = [\n job.id,\n job.finishedOn,\n propVal,\n _.isUndefined(val) ? 'null' : val,\n ignoreLock ? '0' : queue.token,\n keepJobs,\n JSON.stringify({ jobId: job.id, val: val }),\n notFetch || queue.paused || queue.closing || queue.limiter ? 0 : 1,\n queueKeys[''],\n queue.settings.lockDuration,\n queue.token,\n queue.metrics && queue.metrics.maxDataPoints\n ];\n\n return keys.concat(args);\n },\n\n moveToFinished(\n job,\n val,\n propVal,\n shouldRemove,\n target,\n ignoreLock,\n notFetch = false\n ) {\n const args = scripts.moveToFinishedArgs(\n job,\n val,\n propVal,\n shouldRemove,\n target,\n ignoreLock,\n notFetch,\n job.queue.toKey('')\n );\n return job.queue.client.moveToFinished(args).then(result => {\n if (result < 0) {\n throw scripts.finishedErrors(result, job.id, 'finished', 'active');\n } else if (result) {\n return raw2jobData(result);\n }\n return 0;\n });\n },\n\n finishedErrors(code, jobId, command, state) {\n switch (code) {\n case -1:\n return new Error('Missing key for job ' + jobId + ' ' + command);\n case -2:\n return new Error('Missing lock for job ' + jobId + ' ' + command);\n case -3:\n return new Error(\n `Job ${jobId} is not in the ${state} state. ${command}`\n );\n case -6:\n return new Error(\n `Lock mismatch for job ${jobId}. Cmd ${command} from ${state}`\n );\n }\n },\n\n // TODO: add a retention argument for completed and finished jobs (in time).\n moveToCompleted(\n job,\n returnvalue,\n removeOnComplete,\n ignoreLock,\n notFetch = false\n ) {\n return scripts.moveToFinished(\n job,\n returnvalue,\n 'returnvalue',\n removeOnComplete,\n 'completed',\n ignoreLock,\n notFetch\n );\n },\n\n moveToFailedArgs(job, failedReason, removeOnFailed, ignoreLock) {\n return scripts.moveToFinishedArgs(\n job,\n failedReason,\n 'failedReason',\n removeOnFailed,\n 'failed',\n ignoreLock,\n true\n );\n },\n\n moveToFailed(job, failedReason, removeOnFailed, ignoreLock) {\n const args = scripts.moveToFailedArgs(\n job,\n failedReason,\n removeOnFailed,\n ignoreLock\n );\n return scripts.moveToFinished(args);\n },\n\n isFinished(job) {\n const keys = _.map(['completed', 'failed'], key => {\n return job.queue.toKey(key);\n });\n\n return job.queue.client.isFinished(keys.concat([job.id]));\n },\n\n moveToDelayedArgs(queue, jobId, timestamp, ignoreLock) {\n //\n // Bake in the job id first 12 bits into the timestamp\n // to guarantee correct execution order of delayed jobs\n // (up to 4096 jobs per given timestamp or 4096 jobs apart per timestamp)\n //\n // WARNING: Jobs that are so far apart that they wrap around will cause FIFO to fail\n //\n timestamp = _.isUndefined(timestamp) ? 0 : timestamp;\n\n timestamp = +timestamp || 0;\n timestamp = timestamp < 0 ? 0 : timestamp;\n if (timestamp > 0) {\n timestamp = timestamp * 0x1000 + (jobId & 0xfff);\n }\n\n const keys = _.map(['active', 'delayed', jobId, 'stalled'], name => {\n return queue.toKey(name);\n });\n return keys.concat([\n JSON.stringify(timestamp),\n jobId,\n ignoreLock ? '0' : queue.token\n ]);\n },\n\n moveToDelayed(queue, jobId, timestamp, ignoreLock) {\n const args = scripts.moveToDelayedArgs(queue, jobId, timestamp, ignoreLock);\n return queue.client.moveToDelayed(args).then(result => {\n switch (result) {\n case -1:\n throw new Error(\n 'Missing Job ' +\n jobId +\n ' when trying to move from active to delayed'\n );\n case -2:\n throw new Error(\n 'Job ' +\n jobId +\n ' was locked when trying to move from active to delayed'\n );\n }\n });\n },\n\n remove(queue, jobId) {\n const keys = [\n queue.keys.active,\n queue.keys.wait,\n queue.keys.delayed,\n queue.keys.paused,\n queue.keys.completed,\n queue.keys.failed,\n queue.keys.priority,\n queue.toKey(jobId),\n queue.toKey(`${jobId}:logs`),\n queue.keys.limiter,\n queue.toKey(''),\n ];\n return queue.client.removeJob(keys.concat([jobId, queue.token]));\n },\n\n async removeWithPattern(queue, pattern) {\n const keys = [\n queue.keys.active,\n queue.keys.wait,\n queue.keys.delayed,\n queue.keys.paused,\n queue.keys.completed,\n queue.keys.failed,\n queue.keys.priority,\n queue.keys.limiter\n ];\n\n const allRemoved = [];\n let cursor = '0',\n removed;\n do {\n [cursor, removed] = await queue.client.removeJobs(\n keys.concat([queue.toKey(''), pattern, cursor])\n );\n allRemoved.push.apply(allRemoved, removed);\n } while (cursor !== '0');\n\n return allRemoved;\n },\n\n extendLock(queue, jobId, duration) {\n return queue.client.extendLock([\n queue.toKey(jobId) + ':lock',\n queue.keys.stalled,\n queue.token,\n duration,\n jobId\n ]);\n },\n\n releaseLock(queue, jobId) {\n return queue.client.releaseLock([\n queue.toKey(jobId) + ':lock',\n queue.token\n ]);\n },\n\n takeLock(queue, job) {\n return queue.client.takeLock([\n job.lockKey(),\n queue.token,\n queue.settings.lockDuration\n ]);\n },\n\n /**\n It checks if the job in the top of the delay set should be moved back to the\n top of the wait queue (so that it will be processed as soon as possible)\n */\n updateDelaySet(queue, delayedTimestamp) {\n const keys = [\n queue.keys.delayed,\n queue.keys.active,\n queue.keys.wait,\n queue.keys.priority,\n queue.keys.paused,\n queue.keys['meta-paused']\n ];\n\n const args = [queue.toKey(''), delayedTimestamp, queue.token];\n return queue.client.updateDelaySet(keys.concat(args));\n },\n\n promote(queue, jobId) {\n const keys = [\n queue.keys.delayed,\n queue.keys.wait,\n queue.keys.paused,\n queue.keys['meta-paused'],\n queue.keys.priority\n ];\n\n const args = [queue.toKey(''), jobId, queue.token];\n\n return queue.client.promote(keys.concat(args));\n },\n\n /**\n * Looks for unlocked jobs in the active queue.\n *\n * The job was being worked on, but the worker process died and it failed to renew the lock.\n * We call these jobs 'stalled'. This is the most common case. We resolve these by moving them\n * back to wait to be re-processed. To prevent jobs from cycling endlessly between active and wait,\n * (e.g. if the job handler keeps crashing), we limit the number stalled job recoveries to settings.maxStalledCount.\n */\n moveUnlockedJobsToWait(queue) {\n const keys = [\n queue.keys.stalled,\n queue.keys.wait,\n queue.keys.active,\n queue.keys.failed,\n queue.keys['stalled-check'],\n queue.keys['meta-paused'],\n queue.keys.paused\n ];\n const args = [\n queue.settings.maxStalledCount,\n queue.toKey(''),\n Date.now(),\n queue.settings.stalledInterval\n ];\n return queue.client.moveStalledJobsToWait(keys.concat(args));\n },\n\n cleanJobsInSet(queue, set, ts, limit) {\n return queue.client.cleanJobsInSet([\n queue.toKey(set),\n queue.toKey('priority'),\n queue.keys.limiter,\n queue.toKey(''),\n ts,\n limit || 0,\n set\n ]);\n },\n\n retryJobArgs(job, ignoreLock) {\n const queue = job.queue;\n const jobId = job.id;\n\n const keys = _.map(\n ['active', 'wait', jobId, 'meta-paused', 'paused', 'stalled', 'priority'],\n name => {\n return queue.toKey(name);\n }\n );\n\n const pushCmd = (job.opts.lifo ? 'R' : 'L') + 'PUSH';\n\n return keys.concat([pushCmd, jobId, ignoreLock ? '0' : job.queue.token]);\n },\n\n /**\n * Attempts to reprocess a job\n *\n * @param {Job} job\n * @param {Object} options\n * @param {String} options.state The expected job state. If the job is not found\n * on the provided state, then it's not reprocessed. Supported states: 'failed', 'completed'\n *\n * @return {Promise} Returns a promise that evaluates to a return code:\n * 1 means the operation was a success\n * 0 means the job does not exist\n * -1 means the job is currently locked and can't be retried.\n * -2 means the job was not found in the expected set\n */\n reprocessJob(job, options) {\n const queue = job.queue;\n\n const keys = [\n queue.toKey(job.id),\n queue.toKey(job.id) + ':lock',\n queue.toKey(options.state),\n queue.toKey('wait'),\n queue.toKey('meta-paused'),\n queue.toKey('paused')\n ];\n\n const args = [\n job.id,\n (job.opts.lifo ? 'R' : 'L') + 'PUSH',\n queue.token,\n Date.now()\n ];\n\n return queue.client.reprocessJob(keys.concat(args));\n },\n\n obliterate(queue, opts) {\n const client = queue.client;\n\n const keys = [queue.keys['meta-paused'], queue.toKey('')];\n const args = [opts.count, opts.force ? 'force' : null];\n\n return client.obliterate(keys.concat(args)).then(result => {\n if (result < 0) {\n switch (result) {\n case -1:\n throw new Error('Cannot obliterate non-paused queue');\n case -2:\n throw new Error('Cannot obliterate queue with active jobs');\n }\n }\n return result;\n });\n }\n};\n\nmodule.exports = scripts;\n\nfunction array2obj(arr) {\n const obj = {};\n for (let i = 0; i < arr.length; i += 2) {\n obj[arr[i]] = arr[i + 1];\n }\n return obj;\n}\n\nfunction raw2jobData(raw) {\n if (raw) {\n const jobData = raw[0];\n if (jobData.length) {\n const job = array2obj(jobData);\n return [job, raw[1]];\n }\n }\n return [];\n}\n" }, { "path": "lib/timer-manager.js", "content": "'use strict';\n\nconst _ = require('lodash');\nconst uuid = require('uuid');\n\n/**\n Timer Manager\n\n Keep track of timers to ensure that disconnect() is\n only called (via close()) at a time when it's safe\n to do so.\n\n Queues currently use two timers:\n\n - The first one is used for delayed jobs and is\n preemptible i.e. it is possible to close a queue\n while delayed jobs are still pending (they will\n be processed when the queue is resumed). This timer\n is cleared by close() and is not managed here.\n\n - The second one is used to lock Redis while\n processing jobs. These timers are short-lived,\n and there can be more than one active at a\n time.\n\n The lock timer executes Redis commands, which\n means we can't close queues while it's active i.e.\n this won't work:\n\n queue.process(function (job, jobDone) {\n handle(job);\n queue.disconnect().then(jobDone);\n })\n\n The disconnect() call closes the Redis connections; then, when\n a queue tries to perform the scheduled Redis commands,\n they block until a Redis connection becomes available...\n\n The solution is to close the Redis connections when there are no\n active timers i.e. when the queue is idle. This helper class keeps\n track of the active timers and executes any queued listeners\n whenever that count goes to zero.\n\n Since disconnect() simply can't work if there are active handles,\n its close() wrapper postpones closing the Redis connections\n until the next idle state. This means that close() can safely\n be called from anywhere at any time, even from within a job\n handler:\n\n queue.process(function (job, jobDone) {\n handle(job);\n queue.close();\n jobDone();\n })\n*/\n\nfunction TimerManager() {\n this.idle = true;\n this.listeners = [];\n this.timers = {};\n}\n\n/**\n Create a new timer (setTimeout).\n\n Expired timers are automatically cleared\n\n @param {String} name - Name of a timer key. Used only for debugging.\n @param {Number} delay - delay of timeout\n @param {Function} fn - Function to execute after delay\n @returns {Number} id - The timer id. Used to clear the timer\n*/\nTimerManager.prototype.set = function(name, delay, fn) {\n const id = uuid.v4();\n const timer = setTimeout(\n (timerInstance, timeoutId) => {\n timerInstance.clear(timeoutId);\n try {\n fn();\n } catch (err) {\n console.error(err);\n }\n },\n delay,\n this,\n id\n );\n\n // XXX only the timer is used, but the\n // other fields are useful for\n // troubleshooting/debugging\n this.timers[id] = {\n name,\n timer\n };\n\n this.idle = false;\n return id;\n};\n\n/**\n Clear a timer (clearTimeout).\n\n Queued listeners are executed if there are no\n remaining timers\n*/\nTimerManager.prototype.clear = function(id) {\n const timers = this.timers;\n const timer = timers[id];\n if (!timer) {\n return;\n }\n clearTimeout(timer.timer);\n delete timers[id];\n if (!this.idle && _.size(timers) === 0) {\n while (this.listeners.length) {\n this.listeners.pop()();\n }\n this.idle = true;\n }\n};\n\nTimerManager.prototype.clearAll = function() {\n _.each(this.timers, (timer, id) => {\n this.clear(id);\n });\n};\n\n/**\n * Returns a promise that resolves when there are no active timers.\n */\nTimerManager.prototype.whenIdle = function() {\n return new Promise(resolve => {\n if (this.idle) {\n resolve();\n } else {\n this.listeners.unshift(resolve);\n }\n });\n};\n\nmodule.exports = TimerManager;\n" }, { "path": "lib/utils.js", "content": "'use strict';\nconst errorObject = { value: null };\nfunction tryCatch(fn, ctx, args) {\n try {\n return fn.apply(ctx, args);\n } catch (e) {\n errorObject.value = e;\n return errorObject;\n }\n}\n\n/**\n * Waits for a redis client to be ready.\n * @param {Redis} redis client\n */\nfunction isRedisReady(client) {\n return new Promise((resolve, reject) => {\n if (client.status === 'ready') {\n resolve();\n } else {\n function handleReady() {\n client.removeListener('end', handleEnd);\n client.removeListener('error', handleError);\n resolve();\n }\n\n let lastError;\n function handleError(err) {\n lastError = err;\n }\n\n function handleEnd() {\n client.removeListener('ready', handleReady);\n client.removeListener('error', handleError);\n reject(lastError);\n }\n\n client.once('ready', handleReady);\n client.on('error', handleError);\n client.once('end', handleEnd);\n }\n });\n}\n\nmodule.exports.errorObject = errorObject;\nmodule.exports.tryCatch = tryCatch;\nmodule.exports.isRedisReady = isRedisReady;\nmodule.exports.emitSafe = function(emitter, event, ...args) {\n try {\n return emitter.emit(event, ...args);\n } catch (err) {\n try {\n return emitter.emit('error', err);\n } catch (err) {\n // We give up if the error event also throws an exception.\n console.error(err);\n }\n }\n};\n\nmodule.exports.MetricsTime = {\n ONE_MINUTE: 1,\n FIVE_MINUTES: 5,\n FIFTEEN_MINUTES: 15,\n THIRTY_MINUTES: 30,\n ONE_HOUR: 60,\n ONE_WEEK: 60 * 24 * 7,\n TWO_WEEKS: 60 * 24 * 7 * 2,\n ONE_MONTH: 60 * 24 * 7 * 2 * 4\n};\n" }, { "path": "lib/worker.js", "content": "'use strict';\n\nconst utils = require('./utils');\nconst clientCommandMessageReg = /ERR unknown command ['`]\\s*client\\s*['`]/;\n\nmodule.exports = function(Queue) {\n // IDEA, How to store metadata associated to a worker.\n // create a key from the worker ID associated to the given name.\n // We keep a hash table bull:myqueue:workers where every worker is a hash key workername:workerId with json holding\n // metadata of the worker. The worker key gets expired every 30 seconds or so, we renew the worker metadata.\n //\n Queue.prototype.setWorkerName = function() {\n return utils\n .isRedisReady(this.client)\n .then(() => {\n const connectionName = this.clientName();\n this.bclient.options.connectionName = connectionName;\n return this.bclient.client('setname', connectionName);\n })\n .catch(err => {\n if (!clientCommandMessageReg.test(err.message)) throw err;\n });\n };\n\n Queue.prototype.getWorkers = function() {\n return utils\n .isRedisReady(this.client)\n .then(() => {\n return this.client.client('list');\n })\n .then(clients => {\n return this.parseClientList(clients);\n })\n .catch(err => {\n if (!clientCommandMessageReg.test(err.message)) throw err;\n });\n };\n\n Queue.prototype.base64Name = function() {\n return Buffer.from(this.name).toString('base64');\n };\n\n Queue.prototype.clientName = function() {\n return this.keyPrefix + ':' + this.base64Name();\n };\n\n Queue.prototype.parseClientList = function(list) {\n const lines = list.split('\\n');\n const clients = [];\n\n lines.forEach(line => {\n const client = {};\n const keyValues = line.split(' ');\n keyValues.forEach(keyValue => {\n const index = keyValue.indexOf('=');\n const key = keyValue.substring(0, index);\n const value = keyValue.substring(index + 1);\n client[key] = value;\n });\n const name = client['name'];\n if (name && name.startsWith(this.clientName())) {\n client['name'] = this.name;\n clients.push(client);\n }\n });\n return clients;\n };\n};\n" }, { "path": "package.json", "content": "{\n \"name\": \"bull\",\n \"version\": \"4.16.5\",\n \"description\": \"Job manager\",\n \"engines\": {\n \"node\": \">=12\"\n },\n \"main\": \"./index.js\",\n \"types\": \"./index.d.ts\",\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git://github.com/OptimalBits/bull.git\"\n },\n \"keywords\": [\n \"job\",\n \"queue\",\n \"task\",\n \"parallel\"\n ],\n \"author\": \"OptimalBits\",\n \"license\": \"MIT\",\n \"readmeFilename\": \"README.md\",\n \"dependencies\": {\n \"cron-parser\": \"^4.9.0\",\n \"get-port\": \"^5.1.1\",\n \"ioredis\": \"^5.3.2\",\n \"lodash\": \"^4.17.21\",\n \"msgpackr\": \"^1.11.2\",\n \"semver\": \"^7.5.2\",\n \"uuid\": \"^8.3.0\"\n },\n \"devDependencies\": {\n \"@commitlint/cli\": \"^7.6.1\",\n \"@commitlint/config-conventional\": \"^7.6.0\",\n \"@semantic-release/changelog\": \"^5.0.1\",\n \"@semantic-release/commit-analyzer\": \"^8.0.1\",\n \"@semantic-release/git\": \"^9.0.0\",\n \"@semantic-release/github\": \"^7.2.1\",\n \"@semantic-release/npm\": \"^7.1.1\",\n \"@semantic-release/release-notes-generator\": \"^9.0.2\",\n \"chai\": \"^4.2.0\",\n \"coveralls\": \"^3.1.0\",\n \"delay\": \"^4.3.0\",\n \"eslint\": \"^7.4.0\",\n \"eslint-plugin-mocha\": \"^7.0.1\",\n \"eslint-plugin-node\": \"^8.0.1\",\n \"expect.js\": \"^0.3.1\",\n \"fast-glob\": \"^3.3.2\",\n \"husky\": \"^4.2.5\",\n \"istanbul\": \"^0.4.5\",\n \"lint-staged\": \"^8.2.1\",\n \"minimatch\": \"^7.4.4\",\n \"mocha\": \"^8.1.1\",\n \"mocha-lcov-reporter\": \"^1.3.0\",\n \"moment\": \"^2.24.0\",\n \"npm-run-all\": \"^4.1.5\",\n \"nyc\": \"^15.1.0\",\n \"p-reflect\": \"^1.0.0\",\n \"prettier\": \"^1.19.1\",\n \"rimraf\": \"^3.0.2\",\n \"semantic-release\": \"^17.4.2\",\n \"sinon\": \"^7.5.0\"\n },\n \"scripts\": {\n \"clean:scripts\": \"rimraf rawScripts lib/scripts\",\n \"dc:up\": \"docker-compose -f docker-compose.yml up -d\",\n \"dc:down\": \"docker-compose -f docker-compose.yml down\",\n \"dry-run\": \"npm publish --dry-run\",\n \"generate:raw:scripts\": \"node generateRawScripts.js\",\n \"pretest\": \"npm-run-all clean:scripts generate:raw:scripts transform:commands lint\",\n \"lint\": \"eslint lib test *.js\",\n \"test\": \"NODE_ENV=test nyc mocha -- 'test/test_*' --recursive --exit\",\n \"test:nolint\": \"NODE_ENV=test mocha 'test/test_*' --recursive --exit\",\n \"coverage\": \"nyc report --reporter=text-lcov | coveralls\",\n \"postpublish\": \"git push && git push --tags\",\n \"prettier\": \"prettier --config package.json --write '**/*.js'\",\n \"precommit\": \"lint-staged\",\n \"build\": \"tsc\",\n \"transform:commands\": \"node ./commandTransform.js ./rawScripts ./lib/scripts\"\n },\n \"lint-staged\": {\n \"*.{js,json}\": [\n \"prettier --write\",\n \"git add\"\n ]\n },\n \"prettier\": {\n \"singleQuote\": true\n },\n \"husky\": {\n \"hooks\": {\n \"commit-msg\": \"commitlint -E HUSKY_GIT_PARAMS\"\n }\n },\n \"release\": {\n \"branches\": [\n \"develop\"\n ],\n \"plugins\": [\n \"@semantic-release/commit-analyzer\",\n \"@semantic-release/release-notes-generator\",\n [\n \"@semantic-release/changelog\",\n {\n \"changelogFile\": \"CHANGELOG.md\"\n }\n ],\n [\n \"@semantic-release/npm\",\n {\n \"npmPublish\": true\n }\n ],\n \"@semantic-release/github\",\n [\n \"@semantic-release/git\",\n {\n \"assets\": [\n \"package.json\",\n \"yarn.lock\",\n \"CHANGELOG.md\"\n ],\n \"message\": \"chore(release): ${nextRelease.version} [skip ci]\\n\\n${nextRelease.notes}\"\n }\n ]\n ]\n }\n}\n" }, { "path": "test/.eslintrc.yml", "content": "extends:\n - ../.eslintrc.yml\n\nenv:\n mocha: true\n\nrules:\n no-console: 2\n no-process-exit: 0\n" }, { "path": "test/fixtures/fixture_processor.js", "content": "/**\n * A processor file to be used in tests.\n *\n */\n'use strict';\n\nconst delay = require('delay');\n\nmodule.exports = function(/*job*/) {\n return delay(500).then(() => {\n return 42;\n });\n};\n" }, { "path": "test/fixtures/fixture_processor_bar.js", "content": "/**\n * A processor file to be used in tests.\n *\n */\n'use strict';\n\nconst delay = require('delay');\n\nmodule.exports = function(/*job*/) {\n return delay(500).then(() => {\n return 'bar';\n });\n};\n" }, { "path": "test/fixtures/fixture_processor_broken.js", "content": "'use strict';\nthrow new Error('Broken file processor');\n" }, { "path": "test/fixtures/fixture_processor_callback.js", "content": "/**\n * A processor file to be used in tests.\n *\n */\n'use strict';\n\nconst delay = require('delay');\n\nmodule.exports = function(job, done) {\n delay(500).then(() => {\n done(null, 42);\n });\n};\n" }, { "path": "test/fixtures/fixture_processor_callback_fail.js", "content": "/**\n * A processor file to be used in tests.\n *\n */\n'use strict';\n\nconst delay = require('delay');\n\nmodule.exports = function(job, done) {\n delay(500).then(() => {\n done(new Error('Manually failed processor'));\n });\n};\n" }, { "path": "test/fixtures/fixture_processor_crash.js", "content": "/**\n * A processor file to be used in tests.\n *\n */\n'use strict';\n\nmodule.exports = function(job) {\n setTimeout(() => {\n if (typeof job.data.exitCode !== 'number') {\n throw new Error('boom!');\n }\n process.exit(job.data.exitCode);\n }, 100);\n\n return new Promise(() => {});\n};\n" }, { "path": "test/fixtures/fixture_processor_data.js", "content": "/**\n * A processor file to be used in tests.\n *\n */\n'use strict';\n\nconst delay = require('delay');\n\nmodule.exports = function(job) {\n return delay(50).then(() => {\n job.update({ baz: 'qux' });\n return job.data;\n });\n};\n" }, { "path": "test/fixtures/fixture_processor_discard.js", "content": "/**\n * A processor file to be used in tests.\n *\n */\n'use strict';\n\nconst delay = require('delay');\n\nmodule.exports = function(job) {\n return delay(500).then(() => {\n job.discard();\n throw new Error('Manually discarded processor');\n });\n};\n" }, { "path": "test/fixtures/fixture_processor_exit.js", "content": "/**\n * A processor file to be used in tests.\n *\n */\n'use strict';\n\nconst delay = require('delay');\n\nmodule.exports = function(/*job*/) {\n return delay(500).then(() => {\n delay(100).then(() => {\n process.exit(0);\n });\n });\n};\n" }, { "path": "test/fixtures/fixture_processor_fail.js", "content": "/**\n * A processor file to be used in tests.\n *\n */\n'use strict';\n\nconst delay = require('delay');\n\nmodule.exports = function(/*job*/) {\n return delay(500).then(() => {\n throw new Error('Manually failed processor');\n });\n};\n" }, { "path": "test/fixtures/fixture_processor_foo.js", "content": "/**\n * A processor file to be used in tests.\n *\n */\n'use strict';\n\nconst delay = require('delay');\n\nmodule.exports = function(/*job*/) {\n return delay(500).then(() => {\n return 'foo';\n });\n};\n" }, { "path": "test/fixtures/fixture_processor_progress.js", "content": "/**\n * A processor file to be used in tests.\n *\n */\n'use strict';\n\nconst delay = require('delay');\n\nmodule.exports = function(job) {\n return delay(50)\n .then(() => {\n job.progress(10);\n job.log(job.progress());\n return delay(100);\n })\n .then(() => {\n job.progress(27);\n job.log(job.progress());\n return delay(150);\n })\n .then(() => {\n job.progress(78);\n job.log(job.progress());\n return delay(100);\n })\n .then(() => {\n job.progress(100);\n job.log(job.progress());\n })\n .then(() => {\n return 37;\n });\n};\n" }, { "path": "test/fixtures/fixture_processor_slow.js", "content": "/**\n * A processor file to be used in tests.\n *\n */\n'use strict';\n\nconst delay = require('delay');\n\nmodule.exports = function(/*job*/) {\n return delay(1000).then(() => {\n return 42;\n });\n};\n" }, { "path": "test/test_child-pool.js", "content": "'use strict';\n\nconst expect = require('chai').expect;\nconst childPool = require('../lib/process/child-pool');\n\ndescribe('Child pool', () => {\n let pool;\n\n beforeEach(() => {\n pool = new childPool();\n });\n\n afterEach(() => {\n pool.clean();\n });\n\n it('should return same child if free', () => {\n const processor = __dirname + '/fixtures/fixture_processor_bar.js';\n let child;\n return pool\n .retain(processor)\n .then(_child => {\n expect(_child).to.be.ok;\n child = _child;\n pool.release(child);\n\n expect(pool.retained).to.be.empty;\n\n return pool.retain(processor);\n })\n .then(newChild => {\n expect(child).to.be.eql(newChild);\n });\n });\n\n it('should return a new child if reused the last free one', () => {\n const processor = __dirname + '/fixtures/fixture_processor_bar.js';\n let child;\n return pool\n .retain(processor)\n .then(_child => {\n expect(_child).to.be.ok;\n child = _child;\n pool.release(child);\n\n expect(pool.retained).to.be.empty;\n\n return pool.retain(processor);\n })\n .then(newChild => {\n expect(child).to.be.eql(newChild);\n child = newChild;\n return pool.retain(processor);\n })\n .then(newChild => {\n expect(child).not.to.be.eql(newChild);\n });\n });\n\n it('should return a new child if none free', () => {\n const processor = __dirname + '/fixtures/fixture_processor_bar.js';\n let child;\n return pool\n .retain(processor)\n .then(_child => {\n expect(_child).to.be.ok;\n child = _child;\n\n expect(pool.retained).not.to.be.empty;\n\n return pool.retain(processor);\n })\n .then(newChild => {\n expect(child).to.not.be.eql(newChild);\n });\n });\n\n it('should return a new child if killed', () => {\n const processor = __dirname + '/fixtures/fixture_processor_bar.js';\n let child;\n return pool\n .retain(processor)\n .then(_child => {\n expect(_child).to.be.ok;\n child = _child;\n\n pool.kill(child);\n\n expect(pool.retained).to.be.empty;\n\n return pool.retain(processor);\n })\n .then(newChild => {\n expect(child).to.not.be.eql(newChild);\n });\n });\n\n it('should return a new child if many retained and none free', () => {\n const processor = __dirname + '/fixtures/fixture_processor_bar.js';\n let children;\n\n return Promise.all([\n pool.retain(processor),\n pool.retain(processor),\n pool.retain(processor),\n pool.retain(processor),\n pool.retain(processor),\n pool.retain(processor)\n ])\n .then(_children => {\n children = _children;\n expect(children).to.have.length(6);\n return pool.retain(processor);\n })\n .then(child => {\n expect(children).not.to.include(child);\n });\n });\n\n it('should return an old child if many retained and one free', () => {\n const processor = __dirname + '/fixtures/fixture_processor_bar.js';\n let children;\n\n return Promise.all([\n pool.retain(processor),\n pool.retain(processor),\n pool.retain(processor),\n pool.retain(processor),\n pool.retain(processor),\n pool.retain(processor)\n ])\n .then(_children => {\n children = _children;\n expect(children).to.have.length(6);\n\n pool.release(_children[0]);\n\n return pool.retain(processor);\n })\n .then(child => {\n expect(children).to.include(child);\n });\n });\n\n it('should returned a shared child pool is isSharedChildPool is true', () => {\n expect(childPool(true)).to.be.equal(new childPool(true));\n });\n\n it('should return a different childPool if isSharedChildPool is false', () => {\n expect(childPool()).to.not.be.equal(childPool());\n });\n\n it('should not overwrite the the childPool singleton when isSharedChildPool is false', () => {\n const childPoolA = new childPool(true);\n const childPoolB = new childPool(false);\n const childPoolC = new childPool(true);\n\n expect(childPoolA).to.be.equal(childPoolC);\n expect(childPoolB).to.not.be.equal(childPoolA);\n expect(childPoolB).to.not.be.equal(childPoolC);\n });\n});\n" }, { "path": "test/test_connection.js", "content": "'use strict';\n\nconst expect = require('expect.js');\nconst utils = require('./utils');\nconst { isRedisReady } = require('../lib/utils');\nconst Redis = require('ioredis');\nconst Queue = require('../lib/queue');\n\ndescribe('connection', () => {\n let client;\n\n beforeEach(() => {\n client = new Redis();\n return client.flushdb();\n });\n\n afterEach(() => {\n return client.quit();\n });\n\n it('should fail if reusing connections with invalid options', () => {\n const errMsg = Queue.ErrorMessages.MISSING_REDIS_OPTS;\n\n const client = new Redis();\n\n const opts = {\n createClient(type) {\n switch (type) {\n case 'client':\n return client;\n default:\n return new Redis();\n }\n }\n };\n const queue = utils.buildQueue('external connections', opts);\n expect(queue).to.be.ok();\n\n try {\n // eslint-disable-next-line no-unused-vars\n const _ = queue.bclient;\n throw new Error('should fail with invalid redis options');\n } catch (err) {\n expect(err.message).to.be.equal(errMsg);\n }\n\n try {\n // eslint-disable-next-line no-unused-vars\n const _ = queue.eclient;\n throw new Error('should fail with invalid redis options');\n } catch (err) {\n expect(err.message).to.be.equal(errMsg);\n }\n });\n\n it('should recover from a connection loss', async () => {\n const queue = utils.buildQueue();\n queue.on('error', () => {\n // error event has to be observed or the exception will bubble up\n });\n\n const done = new Promise((resolve, reject) => {\n queue\n .process((job, jobDone) => {\n expect(job.data.foo).to.be.equal('bar');\n jobDone();\n queue.close();\n })\n .then(() => {\n resolve();\n })\n .catch(reject);\n });\n\n // Simulate disconnect\n await queue.isReady();\n await isRedisReady(queue.client);\n queue.client.stream.end();\n queue.client.emit('error', new Error('ECONNRESET'));\n\n // add something to the queue\n await queue.add({ foo: 'bar' });\n\n await done;\n });\n\n it('should handle jobs added before and after a redis disconnect', done => {\n let count = 0;\n const queue = utils.buildQueue();\n\n queue\n .process((job, jobDone) => {\n if (count == 0) {\n expect(job.data.foo).to.be.equal('bar');\n jobDone();\n } else {\n jobDone();\n queue.close().then(done, done);\n }\n count++;\n })\n .catch(done);\n\n queue.on('completed', () => {\n if (count === 1) {\n queue.client.stream.end();\n queue.client.emit('error', new Error('ECONNRESET'));\n }\n });\n\n queue.isReady().then(() => {\n queue.add({ foo: 'bar' });\n });\n\n queue.on('error', (/*err*/) => {\n if (count === 1) {\n queue.add({ foo: 'bar' });\n }\n });\n });\n\n it('should not close external connections', () => {\n const redisOpts = {\n maxRetriesPerRequest: null,\n enableReadyCheck: false\n };\n\n const client = new Redis(redisOpts);\n const subscriber = new Redis(redisOpts);\n\n const opts = {\n createClient(type) {\n switch (type) {\n case 'client':\n return client;\n case 'subscriber':\n return subscriber;\n default:\n return new Redis();\n }\n }\n };\n\n const testQueue = utils.buildQueue('external connections', opts);\n\n return new Promise(resolve => {\n if (subscriber.status === 'ready') {\n return resolve();\n }\n subscriber.once('ready', resolve);\n })\n .then(() => {\n return testQueue.isReady();\n })\n .then(() => {\n return testQueue.add({ foo: 'bar' });\n })\n .then(() => {\n expect(testQueue.client).to.be.eql(client);\n expect(testQueue.eclient).to.be.eql(subscriber);\n\n return testQueue.close();\n })\n .then(() => {\n expect(client.status).to.be.eql('ready');\n expect(subscriber.status).to.be.eql('ready');\n return Promise.all([client.quit(), subscriber.quit()]);\n });\n });\n\n it('should fail if redis connection fails and does not reconnect', async () => {\n const queue = utils.buildQueue('connection fail 123', {\n redis: {\n host: 'localhost',\n port: 1234,\n retryStrategy: () => false\n }\n });\n try {\n await isRedisReady(queue.client);\n new Error('Did not fail connecting to invalid redis instance');\n } catch (err) {\n expect(err.code).to.be.eql('ECONNREFUSED');\n await queue.close();\n }\n });\n\n it('should close cleanly if redis connection fails', async () => {\n const queue = new Queue('connection fail', {\n redis: {\n host: 'localhost',\n port: 1235,\n retryStrategy: () => false\n }\n });\n\n await queue.close();\n });\n\n it('should accept ioredis options on the query string', async () => {\n const queue = new Queue(\n 'connection query string',\n 'redis://localhost?tls=RedisCloudFixed'\n );\n\n expect(queue.clients[0].options).to.have.property('tls');\n expect(queue.clients[0].options.tls).to.have.property('ca');\n\n await queue.close();\n });\n});\n" }, { "path": "test/test_events.js", "content": "'use strict';\n\nconst utils = require('./utils');\nconst redis = require('ioredis');\nconst delay = require('delay');\nconst Queue = require('../');\nconst Job = require('../lib/job');\nconst expect = require('chai').expect;\n\nconst _ = require('lodash');\n\ndescribe('events', () => {\n let queue;\n\n beforeEach(() => {\n const client = new redis();\n return client.flushdb().then(() => {\n queue = utils.buildQueue('test events', {\n settings: {\n stalledInterval: 100,\n lockDuration: 50\n }\n });\n return queue;\n });\n });\n\n afterEach(() => {\n return queue.close();\n });\n\n it('should emit waiting when a job has been added', done => {\n queue.on('waiting', () => {\n done();\n });\n\n queue.on('registered:waiting', () => {\n queue.add({ foo: 'bar' });\n });\n });\n\n it('should emit global:waiting when a job has been added', done => {\n queue.on('global:waiting', () => {\n done();\n });\n\n queue.on('registered:global:waiting', () => {\n queue.add({ foo: 'bar' });\n });\n });\n\n it('should emit stalled when a job has been stalled', done => {\n queue.on('completed', (/*job*/) => {\n done(new Error('should not have completed'));\n });\n\n queue.process((/*job*/) => {\n return delay(250);\n });\n\n queue.add({ foo: 'bar' });\n\n const queue2 = utils.buildQueue('test events', {\n settings: {\n stalledInterval: 100\n }\n });\n\n queue2.on('stalled', (/*job*/) => {\n queue2.close().then(done);\n });\n\n queue.on('active', () => {\n queue2.startMoveUnlockedJobsToWait();\n queue.close(true);\n });\n });\n\n it('should emit global:stalled when a job has been stalled', done => {\n queue.on('completed', (/*job*/) => {\n done(new Error('should not have completed'));\n });\n\n queue.process((/*job*/) => {\n return delay(250);\n });\n\n queue.add({ foo: 'bar' });\n\n const queue2 = utils.buildQueue('test events', {\n settings: {\n stalledInterval: 100\n }\n });\n\n queue2.on('global:stalled', (/*job*/) => {\n queue2.close().then(done);\n });\n\n queue.on('active', () => {\n queue2.startMoveUnlockedJobsToWait();\n queue.close(true);\n });\n });\n\n it('should emit global:failed when a job has stalled more than allowable times', done => {\n queue.on('completed', (/*job*/) => {\n done(new Error('should not have completed'));\n });\n\n queue.process((/*job*/) => {\n return delay(250);\n });\n\n queue.add({ foo: 'bar' });\n\n const queue2 = utils.buildQueue('test events', {\n settings: {\n stalledInterval: 100,\n maxStalledCount: 0\n }\n });\n\n queue2.on('global:failed', (jobId, error) => {\n expect(error).equal('job stalled more than maxStalledCount');\n expect(jobId).not.to.be.undefined;\n queue2.close().then(done);\n });\n\n queue.on('active', () => {\n queue2.startMoveUnlockedJobsToWait();\n queue.close(true);\n });\n });\n\n it('emits waiting event when a job is added', done => {\n const queue = utils.buildQueue();\n\n queue.once('waiting', jobId => {\n Job.fromId(queue, jobId).then(job => {\n expect(job.data.foo).to.be.equal('bar');\n queue.close().then(done);\n });\n });\n queue.once('registered:waiting', () => {\n queue.add({ foo: 'bar' });\n });\n });\n\n it('emits drained and global:drained event when all jobs have been processed', done => {\n const queue = utils.buildQueue('event drained', {\n settings: { drainDelay: 1 }\n });\n\n queue.process((job, done) => {\n done();\n });\n\n const drainedCallback = _.after(2, () => {\n queue.getJobCountByTypes('completed').then(completed => {\n expect(completed).to.be.equal(2);\n return queue.close().then(done);\n });\n });\n\n queue.once('drained', drainedCallback);\n queue.once('global:drained', drainedCallback);\n\n queue.add({ foo: 'bar' });\n queue.add({ foo: 'baz' });\n });\n\n it('should emit an event when a new message is added to the queue', done => {\n const client = new redis(6379, '127.0.0.1', {});\n client.select(0);\n const queue = new Queue('test pub sub');\n queue.on('waiting', jobId => {\n expect(parseInt(jobId, 10)).to.be.eql(1);\n client.quit();\n done();\n });\n queue.once('registered:waiting', () => {\n queue.add({ test: 'stuff' });\n });\n });\n\n it('should emit an event when a new priority message is added to the queue', done => {\n const client = new redis(6379, '127.0.0.1', {});\n client.select(0);\n const queue = new Queue('test pub sub');\n queue.on('waiting', jobId => {\n expect(parseInt(jobId, 10)).to.be.eql(1);\n client.quit();\n done();\n });\n queue.once('registered:waiting', () => {\n queue.add({ test: 'stuff' }, { priority: 1 });\n });\n });\n\n it('should emit an event when a job becomes active', done => {\n const queue = utils.buildQueue();\n queue.process((job, jobDone) => {\n jobDone();\n });\n queue.add({});\n queue.once('active', () => {\n queue.once('completed', () => {\n queue.close().then(done);\n });\n });\n });\n\n it('should emit an event if a job fails to extend lock', done => {\n const LOCK_RENEW_TIME = 1;\n queue = utils.buildQueue('queue fails to extend lock', {\n settings: {\n lockRenewTime: LOCK_RENEW_TIME\n }\n });\n queue.once('lock-extension-failed', (lockingFailedJob, error) => {\n expect(lockingFailedJob.data.foo).to.be.equal('lockingFailedJobFoo');\n expect(error.message).to.be.equal('Connection is closed.');\n queue.close().then(done);\n });\n queue.isReady().then(() => {\n queue.process(() => {\n utils.simulateDisconnect(queue);\n return delay(LOCK_RENEW_TIME + 0.25);\n });\n queue.add({ foo: 'lockingFailedJobFoo' });\n });\n });\n\n it('should listen to global events', done => {\n const queue1 = utils.buildQueue();\n const queue2 = utils.buildQueue();\n queue1.process((job, jobDone) => {\n jobDone();\n });\n\n let state;\n queue2.on('global:waiting', () => {\n expect(state).to.be.undefined;\n state = 'waiting';\n });\n queue2.once('registered:global:waiting', () => {\n queue2.once('global:active', () => {\n expect(state).to.be.equal('waiting');\n state = 'active';\n });\n });\n queue2.once('registered:global:active', () => {\n queue2.once('global:completed', () => {\n expect(state).to.be.equal('active');\n queue1.close().then(() => {\n queue2.close().then(done);\n });\n });\n });\n\n queue2.once('registered:global:completed', () => {\n queue1.add({});\n });\n });\n});\n" }, { "path": "test/test_getters.js", "content": "'use strict';\n\nconst redis = require('ioredis');\n\nconst utils = require('./utils');\nconst expect = require('chai').expect;\n\nconst _ = require('lodash');\n\ndescribe('Jobs getters', function() {\n this.timeout(12000);\n let queue;\n let client;\n\n beforeEach(() => {\n client = new redis();\n return client.flushdb();\n });\n\n beforeEach(() => {\n queue = utils.buildQueue();\n });\n\n afterEach(function() {\n this.timeout(\n queue.settings.stalledInterval * (1 + queue.settings.maxStalledCount)\n );\n return queue\n .clean(1000)\n .then(() => {\n return queue.close();\n })\n .then(() => {\n return client.quit();\n });\n });\n\n it('should get waiting jobs', () => {\n return Promise.all([\n queue.add({ foo: 'bar' }),\n queue.add({ baz: 'qux' })\n ]).then(() => {\n return queue.getWaiting().then(jobs => {\n expect(jobs).to.be.a('array');\n expect(jobs.length).to.be.equal(2);\n expect(jobs[0].data.foo).to.be.equal('bar');\n expect(jobs[1].data.baz).to.be.equal('qux');\n });\n });\n });\n\n it('should get paused jobs', () => {\n return queue.pause().then(() => {\n return Promise.all([\n queue.add({ foo: 'bar' }),\n queue.add({ baz: 'qux' })\n ]).then(() => {\n return queue.getWaiting().then(jobs => {\n expect(jobs).to.be.a('array');\n expect(jobs.length).to.be.equal(2);\n expect(jobs[0].data.foo).to.be.equal('bar');\n expect(jobs[1].data.baz).to.be.equal('qux');\n });\n });\n });\n });\n\n it('should get active jobs', done => {\n queue.process((job, jobDone) => {\n queue.getActive().then(jobs => {\n expect(jobs).to.be.a('array');\n expect(jobs.length).to.be.equal(1);\n expect(jobs[0].data.foo).to.be.equal('bar');\n done();\n });\n jobDone();\n });\n\n queue.add({ foo: 'bar' });\n });\n\n it('should get a specific job', done => {\n const data = { foo: 'sup!' };\n queue.add(data).then(job => {\n queue.getJob(job.id).then(returnedJob => {\n expect(returnedJob.data).to.eql(data);\n expect(returnedJob.id).to.be.eql(job.id);\n done();\n });\n });\n });\n\n it('should get completed jobs', done => {\n let counter = 2;\n\n queue.process((job, jobDone) => {\n jobDone();\n });\n\n queue.on('completed', () => {\n counter--;\n\n if (counter === 0) {\n queue.getCompleted().then(jobs => {\n expect(jobs).to.be.a('array');\n // We need a \"empty completed\" kind of function.\n //expect(jobs.length).to.be.equal(2);\n done();\n });\n }\n });\n\n queue.add({ foo: 'bar' });\n queue.add({ baz: 'qux' });\n });\n\n it('should get completed jobs excluding their data', done => {\n let counter = 2;\n const timestamp = Date.now();\n\n queue.process((job, jobDone) => {\n jobDone();\n });\n\n queue.on('completed', () => {\n counter--;\n\n if (counter === 0) {\n queue.getCompleted(0, -1, { excludeData: true }).then(jobs => {\n expect(jobs).to.be.a('array');\n expect(jobs).to.have.length(2);\n\n for (let i = 0; i < jobs.length; i++) {\n expect(jobs[i]).to.have.property('data');\n expect(jobs[i].data).to.be.empty;\n\n expect(jobs[i]).to.have.property('timestamp');\n expect(jobs[i].timestamp).to.be.gte(timestamp);\n expect(jobs[i]).to.have.property('processedOn');\n expect(jobs[i].processedOn).to.be.gte(timestamp);\n }\n\n done();\n });\n }\n });\n\n queue.add({ foo: 'bar' });\n queue.add({ baz: 'qux' });\n });\n\n it('should get failed jobs', done => {\n let counter = 2;\n\n queue.process((job, jobDone) => {\n jobDone(new Error('Forced error'));\n });\n\n queue.on('failed', () => {\n counter--;\n\n if (counter === 0) {\n queue.getFailed().then(jobs => {\n expect(jobs).to.be.a('array');\n done();\n });\n }\n });\n\n queue.add({ foo: 'bar' });\n queue.add({ baz: 'qux' });\n });\n\n describe('.getCountsPerPriority', () => {\n it('returns job counts per priority', done => {\n const jobsArray = Array.from(Array(42).keys()).map(index => ({\n name: 'test',\n data: {},\n opts: {\n priority: index % 4\n }\n }));\n queue.addBulk(jobsArray).then(() => {\n queue.getCountsPerPriority([0, 1, 2, 3]).then(counts => {\n expect(counts).to.be.eql({\n '0': 11,\n '1': 11,\n '2': 10,\n '3': 10\n });\n done();\n });\n });\n });\n\n describe('when queue is paused', () => {\n it('returns job counts per priority', async () => {\n await queue.pause();\n const jobsArray = Array.from(Array(42).keys()).map(index => ({\n name: 'test',\n data: {},\n opts: {\n priority: index % 4\n }\n }));\n await queue.addBulk(jobsArray);\n const counts = await queue.getCountsPerPriority([0, 1, 2, 3]);\n \n expect(counts).to.be.eql({\n '0': 11,\n '1': 11,\n '2': 10,\n '3': 10\n });\n });\n }); \n });\n\n it('fails jobs that exceed their specified timeout', done => {\n queue.process((job, jobDone) => {\n setTimeout(jobDone, 200);\n });\n\n queue.on('failed', (job, error) => {\n expect(error.message).to.contain('timed out');\n done();\n });\n\n queue.on('completed', () => {\n const error = new Error('The job should have timed out');\n done(error);\n });\n\n queue.add(\n { some: 'data' },\n {\n timeout: 100\n }\n );\n });\n\n it('should return all completed jobs when not setting start/end', done => {\n queue.process((job, completed) => {\n completed();\n });\n\n queue.on(\n 'completed',\n _.after(3, () => {\n queue\n .getJobs('completed')\n .then(jobs => {\n expect(jobs)\n .to.be.an('array')\n .that.have.length(3);\n expect(jobs[0]).to.have.property('finishedOn');\n expect(jobs[1]).to.have.property('finishedOn');\n expect(jobs[2]).to.have.property('finishedOn');\n\n expect(jobs[0]).to.have.property('processedOn');\n expect(jobs[1]).to.have.property('processedOn');\n expect(jobs[2]).to.have.property('processedOn');\n done();\n })\n .catch(done);\n })\n );\n\n queue.add({ foo: 1 });\n queue.add({ foo: 2 });\n queue.add({ foo: 3 });\n });\n\n it('should return all failed jobs when not setting start/end', done => {\n queue.process((job, completed) => {\n completed(new Error('error'));\n });\n\n queue.on(\n 'failed',\n _.after(3, () => {\n queue\n .getJobs('failed')\n .then(jobs => {\n expect(jobs)\n .to.be.an('array')\n .that.has.length(3);\n expect(jobs[0]).to.have.property('finishedOn');\n expect(jobs[1]).to.have.property('finishedOn');\n expect(jobs[2]).to.have.property('finishedOn');\n\n expect(jobs[0]).to.have.property('processedOn');\n expect(jobs[1]).to.have.property('processedOn');\n expect(jobs[2]).to.have.property('processedOn');\n done();\n })\n .catch(done);\n })\n );\n\n queue.add({ foo: 1 });\n queue.add({ foo: 2 });\n queue.add({ foo: 3 });\n });\n\n it('should return subset of jobs when setting positive range', done => {\n queue.process((job, completed) => {\n completed();\n });\n\n queue.on(\n 'completed',\n _.after(3, () => {\n queue\n .getJobs('completed', 1, 2, true)\n .then(jobs => {\n expect(jobs)\n .to.be.an('array')\n .that.has.length(2);\n expect(jobs[0].data.foo).to.be.eql(2);\n expect(jobs[1].data.foo).to.be.eql(3);\n expect(jobs[0]).to.have.property('finishedOn');\n expect(jobs[1]).to.have.property('finishedOn');\n expect(jobs[0]).to.have.property('processedOn');\n expect(jobs[1]).to.have.property('processedOn');\n done();\n })\n .catch(done);\n })\n );\n\n queue\n .add({ foo: 1 })\n .then(() => {\n return queue.add({ foo: 2 });\n })\n .then(() => {\n return queue.add({ foo: 3 });\n });\n });\n\n it('should return subset of jobs when setting a negative range', done => {\n queue.process((job, completed) => {\n completed();\n });\n\n queue.on(\n 'completed',\n _.after(3, () => {\n queue\n .getJobs('completed', -3, -1, true)\n .then(jobs => {\n expect(jobs)\n .to.be.an('array')\n .that.has.length(3);\n expect(jobs[0].data.foo).to.be.equal(1);\n expect(jobs[1].data.foo).to.be.eql(2);\n expect(jobs[2].data.foo).to.be.eql(3);\n done();\n })\n .catch(done);\n })\n );\n\n queue.add({ foo: 1 });\n queue.add({ foo: 2 });\n queue.add({ foo: 3 });\n });\n\n it('should return subset of jobs when range overflows', done => {\n queue.process((job, completed) => {\n completed();\n });\n\n queue.on(\n 'completed',\n _.after(3, () => {\n queue\n .getJobs('completed', -300, 99999, true)\n .then(jobs => {\n expect(jobs)\n .to.be.an('array')\n .that.has.length(3);\n expect(jobs[0].data.foo).to.be.equal(1);\n expect(jobs[1].data.foo).to.be.eql(2);\n expect(jobs[2].data.foo).to.be.eql(3);\n done();\n })\n .catch(done);\n })\n );\n\n queue.add({ foo: 1 });\n queue.add({ foo: 2 });\n queue.add({ foo: 3 });\n });\n\n it('should return jobs for multiple types', done => {\n let counter = 0;\n queue.process((/*job*/) => {\n counter++;\n if (counter == 2) {\n return queue.pause();\n }\n });\n\n queue.on(\n 'completed',\n _.after(2, () => {\n queue\n .getJobs(['completed', 'paused'])\n .then(jobs => {\n expect(jobs).to.be.an('array');\n expect(jobs).to.have.length(3);\n done();\n })\n .catch(done);\n })\n );\n\n queue.add({ foo: 1 });\n queue.add({ foo: 2 });\n queue.add({ foo: 3 });\n });\n});\n" }, { "path": "test/test_job.js", "content": "'use strict';\n\nconst Job = require('../lib/job');\nconst Queue = require('../lib/queue');\nconst expect = require('expect.js');\nconst redis = require('ioredis');\nconst uuid = require('uuid');\nconst delay = require('delay');\n\ndescribe('Job', () => {\n let queue;\n let client;\n\n beforeEach(() => {\n client = new redis();\n return client.flushdb();\n });\n\n beforeEach(() => {\n queue = new Queue('test-' + uuid.v4(), {\n redis: { port: 6379, host: '127.0.0.1' }\n });\n });\n\n afterEach(function() {\n this.timeout(\n queue.settings.stalledInterval * (1 + queue.settings.maxStalledCount)\n );\n return queue.close().then(() => {\n return client.quit();\n });\n });\n\n describe('.create', () => {\n let job;\n let data;\n let opts;\n\n beforeEach(() => {\n data = { foo: 'bar' };\n opts = { testOpt: 'enabled' };\n\n return Job.create(queue, data, opts).then(createdJob => {\n job = createdJob;\n });\n });\n\n it('returns a promise for the job', () => {\n expect(job).to.have.property('id');\n expect(job).to.have.property('data');\n });\n\n it('should not modify input options', () => {\n expect(opts).not.to.have.property('jobId');\n });\n\n it('saves the job in redis', () => {\n return Job.fromId(queue, job.id).then(storedJob => {\n expect(storedJob).to.have.property('id');\n expect(storedJob).to.have.property('data');\n\n expect(storedJob.data.foo).to.be.equal('bar');\n expect(storedJob.opts).to.be.a(Object);\n expect(storedJob.opts.testOpt).to.be('enabled');\n });\n });\n\n it('should use the custom jobId if one is provided', () => {\n const customJobId = 'customjob';\n return Job.create(queue, data, { jobId: customJobId }).then(\n createdJob => {\n expect(createdJob.id).to.be.equal(customJobId);\n }\n );\n });\n\n it('should process jobs with custom jobIds', done => {\n const customJobId = 'customjob';\n queue.process(() => {\n return Promise.resolve();\n });\n\n queue.add({ foo: 'bar' }, { jobId: customJobId });\n\n queue.on('completed', job => {\n if (job.id == customJobId) {\n done();\n }\n });\n });\n });\n\n describe('.createBulk', () => {\n let jobs;\n let inputJobs;\n\n beforeEach(() => {\n inputJobs = [\n {\n name: 'jobA',\n data: {\n foo: 'bar'\n },\n opts: {\n testOpt: 'enabled'\n }\n },\n {\n name: 'jobB',\n data: {\n foo: 'baz'\n },\n opts: {\n testOpt: 'disabled'\n }\n }\n ];\n\n return Job.createBulk(queue, inputJobs).then(createdJobs => {\n jobs = createdJobs;\n });\n });\n\n it('returns a promise for the jobs', () => {\n expect(jobs).to.have.length(2);\n\n expect(jobs[0]).to.have.property('id');\n expect(jobs[0]).to.have.property('data');\n });\n\n it('should not modify input options', () => {\n expect(inputJobs[0].opts).not.to.have.property('jobId');\n });\n\n it('saves the first job in redis', () => {\n return Job.fromId(queue, jobs[0].id).then(storedJob => {\n expect(storedJob).to.have.property('id');\n expect(storedJob).to.have.property('data');\n\n expect(storedJob.data.foo).to.be.equal('bar');\n expect(storedJob.opts).to.be.a(Object);\n expect(storedJob.opts.testOpt).to.be('enabled');\n });\n });\n\n it('saves the second job in redis', () => {\n return Job.fromId(queue, jobs[1].id).then(storedJob => {\n expect(storedJob).to.have.property('id');\n expect(storedJob).to.have.property('data');\n\n expect(storedJob.data.foo).to.be.equal('baz');\n expect(storedJob.opts).to.be.a(Object);\n expect(storedJob.opts.testOpt).to.be('disabled');\n });\n });\n });\n\n describe('.add jobs on priority queues', () => {\n it('add 4 jobs with different priorities', () => {\n return queue\n .add({ foo: 'bar' }, { jobId: '1', priority: 3 })\n .then(() => {\n return queue.add({ foo: 'bar' }, { jobId: '2', priority: 3 });\n })\n .then(() => {\n return queue.add({ foo: 'bar' }, { jobId: '3', priority: 2 });\n })\n .then(() => {\n return queue.add({ foo: 'bar' }, { jobId: '4', priority: 1 });\n })\n .then(() => {\n return queue\n .getWaiting()\n .then(result => {\n const waitingIDs = [];\n result.forEach(element => {\n waitingIDs.push(element.id);\n });\n return waitingIDs;\n })\n .then(waitingIDs => {\n expect(waitingIDs.length).to.be.equal(4);\n expect(waitingIDs).to.be.eql(['4', '3', '1', '2']);\n });\n });\n });\n });\n\n describe('.update', () => {\n it('should allow updating job data', () => {\n return Job.create(queue, { foo: 'bar' })\n .then(job => {\n return job.update({ baz: 'qux' }).then(() => {\n expect(job.data).to.be.eql({ baz: 'qux' });\n return job;\n });\n })\n .then(job => {\n return Job.fromId(queue, job.id).then(job => {\n expect(job.data).to.be.eql({ baz: 'qux' });\n });\n });\n });\n\n describe('when job was removed', () => {\n it('throws an error', async () => {\n const job = await Job.create(queue, { foo: 'bar' });\n await job.remove();\n await job.update({ baz: 'qux' }).catch(err => {\n expect(err.message).to.be.equal('Missing key for job 1 updateData');\n });\n });\n });\n });\n\n describe('.remove', () => {\n it('removes the job from redis', () => {\n return Job.create(queue, { foo: 'bar' })\n .then(job => {\n return job.remove().then(() => {\n return job;\n });\n })\n .then(job => {\n return Job.fromId(queue, job.id);\n })\n .then(storedJob => {\n expect(storedJob).to.be(null);\n });\n });\n\n it('fails to remove a locked job', () => {\n return Job.create(queue, 1, { foo: 'bar' }).then(job => {\n return job\n .takeLock()\n .then(lock => {\n expect(lock).to.be.truthy;\n })\n .then(() => {\n return Job.fromId(queue, job.id).then(job => {\n return job.remove();\n });\n })\n .then(() => {\n throw new Error('Should not be able to remove a locked job');\n })\n .catch((/*err*/) => {\n // Good!\n });\n });\n });\n\n it('removes any job from active set', () => {\n return queue.add({ foo: 'bar' }).then(job => {\n // Simulate a job in active state but not locked\n return queue\n .getNextJob()\n .then(() => {\n return job\n .isActive()\n .then(isActive => {\n expect(isActive).to.be(true);\n return job.releaseLock();\n })\n .then(() => {\n return job.remove();\n });\n })\n .then(() => {\n return Job.fromId(queue, job.id);\n })\n .then(stored => {\n expect(stored).to.be(null);\n return job.getState();\n })\n .then(state => {\n // This check is a bit of a hack. A job that is not found in any list will return the state\n // stuck.\n expect(state).to.equal('stuck');\n });\n });\n });\n\n it('emits removed event', cb => {\n queue.once('removed', job => {\n expect(job.data.foo).to.be.equal('bar');\n cb();\n });\n Job.create(queue, { foo: 'bar' }).then(job => {\n job.remove();\n });\n });\n\n it('a succesful job should be removable', done => {\n queue.process(() => {\n return Promise.resolve();\n });\n\n queue.add({ foo: 'bar' });\n\n queue.on('completed', job => {\n job\n .remove()\n .then(done)\n .catch(done);\n });\n });\n\n it('a failed job should be removable', done => {\n queue.process(() => {\n throw new Error();\n });\n\n queue.add({ foo: 'bar' });\n\n queue.on('failed', job => {\n job\n .remove()\n .then(done)\n .catch(done);\n });\n });\n });\n\n describe('.removeFromPattern', () => {\n it('remove jobs matching pattern', async () => {\n const jobIds = ['foo', 'foo1', 'foo2', 'foo3', 'foo4', 'bar', 'baz'];\n await Promise.all(\n jobIds.map(jobId => Job.create(queue, { foo: 'bar' }, { jobId }))\n );\n\n await queue.removeJobs('foo*');\n\n for (let i = 0; i < jobIds.length; i++) {\n const storedJob = await Job.fromId(queue, jobIds[i]);\n if (jobIds[i].startsWith('foo')) {\n expect(storedJob).to.be(null);\n } else {\n expect(storedJob).to.not.be(null);\n }\n }\n });\n });\n\n describe('.remove on priority queues', () => {\n it('remove a job with jobID 1 and priority 3 and check the new order in the queue', () => {\n return queue\n .add({ foo: 'bar' }, { jobId: '1', priority: 3 })\n .then(() => {\n return queue.add({ foo: 'bar' }, { jobId: '2', priority: 3 });\n })\n .then(() => {\n return queue.add({ foo: 'bar' }, { jobId: '3', priority: 2 });\n })\n .then(() => {\n return queue.add({ foo: 'bar' }, { jobId: '4', priority: 1 });\n })\n .then(() => {\n return queue.getJob('1').then(job => {\n return job.remove().then(() => {\n return queue\n .getWaiting()\n .then(result => {\n const waitingIDs = [];\n result.forEach(element => {\n waitingIDs.push(element.id);\n });\n return waitingIDs;\n })\n .then(waitingIDs => {\n expect(waitingIDs.length).to.be.equal(3);\n expect(waitingIDs).to.be.eql(['4', '3', '2']);\n });\n });\n });\n });\n });\n\n it('add a new job with priority 10 and ID 5 and check the new order (along with the previous 4 jobs)', () => {\n return queue\n .add({ foo: 'bar' }, { jobId: '1', priority: 3 })\n .then(() => {\n return queue.add({ foo: 'bar' }, { jobId: '2', priority: 3 });\n })\n .then(() => {\n return queue.add({ foo: 'bar' }, { jobId: '3', priority: 2 });\n })\n .then(() => {\n return queue.add({ foo: 'bar' }, { jobId: '4', priority: 1 });\n })\n .then(() => {\n return queue.getJob('1').then(job => {\n return job.remove().then(() => {\n return queue\n .getWaiting()\n .then(result => {\n const waitingIDs = [];\n result.forEach(element => {\n waitingIDs.push(element.id);\n });\n return waitingIDs;\n })\n .then(waitingIDs => {\n expect(waitingIDs.length).to.be.equal(3);\n expect(waitingIDs).to.be.eql(['4', '3', '2']);\n return true;\n })\n .then(() => {\n return queue\n .add({ foo: 'bar' }, { jobId: '5', priority: 10 })\n .then(() => {\n return queue\n .getWaiting()\n .then(result => {\n const waitingIDs = [];\n result.forEach(element => {\n waitingIDs.push(element.id);\n });\n return waitingIDs;\n })\n .then(waitingIDs => {\n expect(waitingIDs.length).to.be.equal(4);\n expect(waitingIDs).to.be.eql(['4', '3', '2', '5']);\n });\n });\n });\n });\n });\n });\n });\n });\n\n describe('.retry', () => {\n it('emits waiting event', cb => {\n queue.add({ foo: 'bar' });\n queue.process((job, done) => {\n done(new Error('the job failed'));\n });\n\n queue.once('failed', job => {\n queue.once('global:waiting', jobId2 => {\n Job.fromId(queue, jobId2).then(job2 => {\n expect(job2.data.foo).to.be.equal('bar');\n cb();\n });\n });\n queue.once('registered:global:waiting', () => {\n job.retry();\n });\n });\n });\n\n it('sets retriedOn to a timestamp', cb => {\n queue.add({ foo: 'bar' });\n queue.process((job, done) => {\n done(new Error('the job failed'));\n });\n\n queue.once('failed', job => {\n queue.once('global:waiting', jobId2 => {\n const now = Date.now();\n expect(job.retriedOn)\n .to.be.a('number')\n .and.to.be.within(now - 1000, now);\n\n Job.fromId(queue, jobId2).then(job2 => {\n expect(job2.retriedOn)\n .to.be.a('number')\n .and.to.be.within(now - 1000, now);\n cb();\n });\n });\n queue.once('registered:global:waiting', () => {\n job.retry();\n });\n });\n });\n });\n\n describe('Locking', () => {\n let job;\n\n beforeEach(() => {\n return Job.create(queue, { foo: 'bar' }).then(createdJob => {\n job = createdJob;\n });\n });\n\n it('can take a lock', () => {\n return job\n .takeLock()\n .then(lockTaken => {\n expect(lockTaken).to.be.truthy;\n })\n .then(() => {\n return job.releaseLock().then(lockReleased => {\n expect(lockReleased).to.not.exist;\n });\n });\n });\n\n it('take an already taken lock', () => {\n return job\n .takeLock()\n .then(lockTaken => {\n expect(lockTaken).to.be.truthy;\n })\n .then(() => {\n return job.takeLock().then(lockTaken => {\n expect(lockTaken).to.be.truthy;\n });\n });\n });\n\n it('can release a lock', () => {\n return job\n .takeLock()\n .then(lockTaken => {\n expect(lockTaken).to.be.truthy;\n })\n .then(() => {\n return job.releaseLock().then(lockReleased => {\n expect(lockReleased).to.not.exist;\n });\n });\n });\n });\n\n describe('.progress', () => {\n it('can set and get progress as number', () => {\n return Job.create(queue, { foo: 'bar' }).then(job => {\n return job.progress(42).then(() => {\n return Job.fromId(queue, job.id).then(async storedJob => {\n expect(storedJob.progress()).to.be(42);\n });\n });\n });\n });\n it('can set and get progress as object', async () => {\n const job = await Job.create(queue, { foo: 'bar' });\n await job.progress({ total: 120, completed: 40 });\n const storedJob = await Job.fromId(queue, job.id);\n expect(storedJob.progress()).to.eql({ total: 120, completed: 40 });\n });\n\n describe('when job was removed', () => {\n it('throws an error', async () => {\n const job = await Job.create(queue, { foo: 'bar' });\n await job.remove();\n await job.progress({ total: 120, completed: 40 }).catch(err => {\n expect(err.message).to.be.equal(\n 'Missing key for job 1 updateProgress'\n );\n });\n });\n });\n });\n\n describe('.log', () => {\n it('can log two rows with text', () => {\n const firstLog = 'some log text 1';\n const secondLog = 'some log text 2';\n return Job.create(queue, { foo: 'bar' }).then(job =>\n job\n .log(firstLog)\n .then(() => job.log(secondLog))\n .then(() => queue.getJobLogs(job.id))\n .then(logs =>\n expect(logs).to.be.eql({ logs: [firstLog, secondLog], count: 2 })\n )\n .then(() => queue.getJobLogs(job.id, 0, 1))\n .then(logs =>\n expect(logs).to.be.eql({ logs: [firstLog, secondLog], count: 2 })\n )\n .then(() => queue.getJobLogs(job.id, 0, 4000))\n .then(logs =>\n expect(logs).to.be.eql({ logs: [firstLog, secondLog], count: 2 })\n )\n .then(() => queue.getJobLogs(job.id, 1, 1))\n .then(logs => expect(logs).to.be.eql({ logs: [secondLog], count: 2 }))\n .then(() => queue.getJobLogs(job.id, 0, 1, false))\n .then(logs =>\n expect(logs).to.be.eql({ logs: [secondLog, firstLog], count: 2 })\n )\n .then(() => queue.getJobLogs(job.id, 0, 4000, false))\n .then(logs =>\n expect(logs).to.be.eql({ logs: [secondLog, firstLog], count: 2 })\n )\n .then(() => queue.getJobLogs(job.id, 1, 1, false))\n .then(logs => expect(logs).to.be.eql({ logs: [firstLog], count: 2 }))\n .then(() => job.remove())\n .then(() => queue.getJobLogs(job.id))\n .then(logs => expect(logs).to.be.eql({ logs: [], count: 0 }))\n );\n });\n\n describe('when job was removed', () => {\n it('throws an error', async () => {\n const job = await Job.create(queue, { foo: 'bar' });\n await job.remove();\n await job.log('some log text 1').catch(err => {\n expect(err.message).to.be.equal('Missing key for job 1 addLog');\n });\n });\n });\n });\n\n describe('.moveToCompleted', () => {\n it('marks the job as completed and returns new job', () => {\n return Job.create(queue, { foo: 'bar' }).then(job1 => {\n return Job.create(queue, { foo: 'bar' }, { lifo: true }).then(job2 => {\n return job2\n .isCompleted()\n .then(isCompleted => {\n expect(isCompleted).to.be(false);\n })\n .then(() => {\n return scripts.moveToActive(queue);\n })\n .then(() => {\n return job2.moveToCompleted('succeeded', true);\n })\n .then(job1Id => {\n return job2.isCompleted().then(isCompleted => {\n expect(isCompleted).to.be(true);\n expect(job2.returnvalue).to.be('succeeded');\n expect(job1Id[1]).to.be(job1.id);\n });\n });\n });\n });\n });\n });\n\n describe('.moveToFailed', () => {\n it('marks the job as failed', () => {\n return Job.create(queue, { foo: 'bar' }).then(job => {\n return job\n .isFailed()\n .then(isFailed => {\n expect(isFailed).to.be(false);\n })\n .then(() => {\n return scripts.moveToActive(queue);\n })\n .then(() => {\n return job.moveToFailed(new Error('test error'), true);\n })\n .then(() => {\n return job.isFailed().then(isFailed => {\n expect(isFailed).to.be(true);\n expect(job.stacktrace).not.be(null);\n expect(job.stacktrace.length).to.be(1);\n });\n });\n });\n });\n\n it('moves the job to wait for retry if attempts are given', () => {\n return Job.create(queue, { foo: 'bar' }, { attempts: 3 }).then(job => {\n return job\n .isFailed()\n .then(isFailed => {\n expect(isFailed).to.be(false);\n })\n .then(() => {\n return scripts.moveToActive(queue);\n })\n .then(() => {\n return job.moveToFailed(new Error('test error'), true);\n })\n .then(() => {\n return job.isFailed().then(isFailed => {\n expect(isFailed).to.be(false);\n expect(job.stacktrace).not.be(null);\n expect(job.stacktrace.length).to.be(1);\n return job.isWaiting().then(isWaiting => {\n expect(isWaiting).to.be(true);\n });\n });\n });\n });\n });\n\n it('unlocks the job when moving it to delayed', () => {\n queue.process(() => {\n throw new Error('Oh dear');\n });\n return Job.create(\n queue,\n { foo: 'bar' },\n { attempts: 3, backoff: 100 }\n ).then(job => {\n return new Promise(resolve => {\n queue.once('failed', resolve);\n })\n .then(() => {\n const client = new redis();\n return client.get(job.lockKey());\n })\n .then(lockValue => {\n expect(lockValue).to.be(null);\n });\n });\n });\n\n it('marks the job as failed when attempts made equal to attempts given', () => {\n return Job.create(queue, { foo: 'bar' }, { attempts: 1 }).then(job => {\n return job\n .isFailed()\n .then(isFailed => {\n expect(isFailed).to.be(false);\n })\n .then(() => {\n return scripts.moveToActive(queue);\n })\n .then(() => {\n return job.moveToFailed(new Error('test error'), true);\n })\n .then(() => {\n return job.isFailed().then(isFailed => {\n expect(isFailed).to.be(true);\n expect(job.stacktrace).not.be(null);\n expect(job.stacktrace.length).to.be(1);\n });\n });\n });\n });\n\n it('moves the job to delayed for retry if attempts are given and backoff is non zero', () => {\n return Job.create(\n queue,\n { foo: 'bar' },\n { attempts: 3, backoff: 300 }\n ).then(job => {\n return job\n .isFailed()\n .then(isFailed => {\n expect(isFailed).to.be(false);\n })\n .then(() => {\n return scripts.moveToActive(queue);\n })\n .then(() => {\n return job.moveToFailed(new Error('test error'), true);\n })\n .then(() => {\n return job.isFailed().then(isFailed => {\n expect(isFailed).to.be(false);\n expect(job.stacktrace).not.be(null);\n expect(job.stacktrace.length).to.be(1);\n return job.isDelayed().then(isDelayed => {\n expect(isDelayed).to.be(true);\n });\n });\n });\n });\n });\n\n it('applies stacktrace limit on failure', () => {\n const stackTraceLimit = 1;\n return Job.create(\n queue,\n { foo: 'bar' },\n { stackTraceLimit, attempts: 2 }\n ).then(job => {\n return job\n .isFailed()\n .then(isFailed => {\n expect(isFailed).to.be(false);\n })\n .then(() => {\n return scripts.moveToActive(queue);\n })\n .then(() => {\n return job.moveToFailed(new Error('test error'), true);\n })\n .then(() => {\n return scripts.moveToActive(queue);\n })\n .then(() => {\n return job.moveToFailed(new Error('test error'), true).then(() => {\n return job.isFailed().then(isFailed => {\n expect(isFailed).to.be(true);\n expect(job.stacktrace).not.be(null);\n expect(job.stacktrace.length).to.be(stackTraceLimit);\n });\n });\n });\n });\n });\n });\n\n describe('.promote', () => {\n it('can promote a delayed job to be executed immediately', () => {\n return Job.create(queue, { foo: 'bar' }, { delay: 1500 }).then(job => {\n return job\n .isDelayed()\n .then(isDelayed => {\n expect(isDelayed).to.be(true);\n })\n .then(() => {\n return job.promote();\n })\n .then(() => {\n return job.isDelayed().then(isDelayed => {\n expect(isDelayed).to.be(false);\n return job.isWaiting().then(isWaiting => {\n expect(isWaiting).to.be(true);\n return;\n });\n });\n });\n });\n });\n\n it('should process a promoted job according to its priority', done => {\n queue.process(() => {\n return delay(100);\n });\n\n const completed = [];\n\n queue.on('completed', job => {\n completed.push(job.id);\n if (completed.length > 3) {\n expect(completed).to.be.eql(['1', '2', '3', '4']);\n done();\n }\n });\n const processStarted = new Promise(resolve =>\n queue.once('active', resolve)\n );\n\n const add = (id, ms) =>\n queue.add({}, { jobId: id, delay: ms, priority: 1 });\n\n add('1')\n .then(() => add('2', 1))\n .then(() => processStarted)\n .then(() => add('3', 5000))\n .then(job => {\n job.promote();\n })\n .then(() => add('4', 1));\n });\n\n it('should not promote a job that is not delayed', () => {\n return Job.create(queue, { foo: 'bar' }).then(job => {\n return job\n .isDelayed()\n .then(isDelayed => {\n expect(isDelayed).to.be(false);\n })\n .then(() => {\n return job.promote();\n })\n .then(() => {\n throw new Error('Job should not be promoted!');\n })\n .catch(err => {\n expect(err).to.be.ok();\n });\n });\n });\n\n it('should promote delayed job to the right queue if queue is paused', async () => {\n await queue.add('normal', { foo: 'bar' });\n const delayedJob = await queue.add(\n 'delayed',\n { foo: 'bar' },\n { delay: 1 }\n );\n\n await queue.pause();\n await delayedJob.promote();\n await queue.resume();\n\n const waitingJobsCount = await queue.getWaitingCount();\n expect(waitingJobsCount).to.be.equal(2);\n const delayedJobsNewState = await delayedJob.getState();\n expect(delayedJobsNewState).to.be.equal('waiting');\n });\n });\n\n // TODO:\n // Divide into several tests\n //\n const scripts = require('../lib/scripts');\n it('get job status', function() {\n this.timeout(12000);\n\n const client = new redis();\n return Job.create(queue, { foo: 'baz' })\n .then(job => {\n return job\n .isStuck()\n .then(isStuck => {\n expect(isStuck).to.be(false);\n return job.getState();\n })\n .then(state => {\n expect(state).to.be('waiting');\n return scripts.moveToActive(queue).then(() => {\n return job.moveToCompleted();\n });\n })\n .then(() => {\n return job.isCompleted();\n })\n .then(isCompleted => {\n expect(isCompleted).to.be(true);\n return job.getState();\n })\n .then(state => {\n expect(state).to.be('completed');\n return client.zrem(queue.toKey('completed'), job.id).then(() => {\n return client.lpush(queue.toKey('active'), job.id);\n });\n })\n .then(() => {\n return job.moveToDelayed(Date.now() + 10000, true);\n })\n .then(() => {\n return job.isDelayed();\n })\n .then(yes => {\n expect(yes).to.be(true);\n return job.getState();\n })\n .then(state => {\n expect(state).to.be('delayed');\n return client.zrem(queue.toKey('delayed'), job.id).then(() => {\n return client.lpush(queue.toKey('active'), job.id);\n });\n })\n .then(() => {\n return job.moveToFailed(new Error('test'), true);\n })\n .then(() => {\n return job.isFailed();\n })\n .then(isFailed => {\n expect(isFailed).to.be(true);\n return job.getState();\n })\n .then(state => {\n expect(state).to.be('failed');\n return client.zrem(queue.toKey('failed'), job.id);\n })\n .then(res => {\n expect(res).to.be(1);\n return job.getState();\n })\n .then(state => {\n expect(state).to.be('stuck');\n return client.rpop(queue.toKey('wait'));\n })\n .then(() => {\n return client.lpush(queue.toKey('paused'), job.id);\n })\n .then(() => {\n return job.isPaused();\n })\n .then(isPaused => {\n expect(isPaused).to.be(true);\n return job.getState();\n })\n .then(state => {\n expect(state).to.be('paused');\n return client.rpop(queue.toKey('paused'));\n })\n .then(() => {\n return client.lpush(queue.toKey('wait'), job.id);\n })\n .then(() => {\n return job.isWaiting();\n })\n .then(isWaiting => {\n expect(isWaiting).to.be(true);\n return job.getState();\n })\n .then(state => {\n expect(state).to.be('waiting');\n });\n })\n .then(() => {\n return client.quit();\n });\n });\n\n describe('.finished', () => {\n it('should resolve when the job has been completed', done => {\n queue.process(() => {\n return delay(500);\n });\n queue\n .add({ foo: 'bar' })\n .then(job => {\n return job.finished();\n })\n .then(done, done);\n });\n\n it('should resolve when the job has been completed and return object', done => {\n queue.process((/*job*/) => {\n return delay(500).then(() => {\n return { resultFoo: 'bar' };\n });\n });\n queue\n .add({ foo: 'bar' })\n .then(job => {\n return job.finished();\n })\n .then(jobResult => {\n expect(jobResult).to.be.an('object');\n expect(jobResult.resultFoo).equal('bar');\n done();\n });\n });\n\n it('should resolve when the job has been delayed and completed and return object', done => {\n queue.process((/*job*/) => {\n return delay(300).then(() => {\n return { resultFoo: 'bar' };\n });\n });\n queue\n .add({ foo: 'bar' })\n .then(job => {\n return delay(600).then(() => {\n return job.finished();\n });\n })\n .then(jobResult => {\n expect(jobResult).to.be.an('object');\n expect(jobResult.resultFoo).equal('bar');\n done();\n });\n });\n\n it('should resolve when the job has been completed and return string', done => {\n queue.process((/*job*/) => {\n return delay(500).then(() => {\n return 'a string';\n });\n });\n queue\n .add({ foo: 'bar' })\n .then(job => {\n return delay(600).then(() => {\n return job.finished();\n });\n })\n .then(jobResult => {\n expect(jobResult).to.be.an('string');\n expect(jobResult).equal('a string');\n done();\n });\n });\n\n it('should resolve when the job has been delayed and completed and return string', done => {\n queue.process((/*job*/) => {\n return delay(300).then(() => {\n return 'a string';\n });\n });\n queue\n .add({ foo: 'bar' })\n .then(job => {\n return job.finished();\n })\n .then(jobResult => {\n expect(jobResult).to.be.an('string');\n expect(jobResult).equal('a string');\n done();\n });\n });\n\n it('should reject when the job has been failed', done => {\n queue.process(() => {\n return delay(500).then(() => {\n return Promise.reject(new Error('test error'));\n });\n });\n\n queue\n .add({ foo: 'bar' })\n .then(job => {\n return job.finished();\n })\n .then(\n () => {\n done(Error('should have been rejected'));\n },\n err => {\n expect(err.message).equal('test error');\n done();\n }\n );\n });\n\n it('should resolve directly if already processed', done => {\n queue.process(() => {\n return Promise.resolve();\n });\n queue\n .add({ foo: 'bar' })\n .then(job => {\n return delay(500).then(() => {\n return job.finished();\n });\n })\n .then(() => {\n done();\n }, done);\n });\n\n it('should reject directly if already processed', done => {\n queue.process(() => {\n return Promise.reject(Error('test error'));\n });\n queue\n .add({ foo: 'bar' })\n .then(job => {\n return delay(500).then(() => {\n return job.finished();\n });\n })\n .then(\n () => {\n done(Error('should have been rejected'));\n },\n err => {\n expect(err.message).equal('test error');\n done();\n }\n );\n });\n });\n\n describe('.fromJSON', () => {\n let data;\n\n beforeEach(() => {\n data = { foo: 'bar' };\n });\n\n it('should parse JSON data by default', async () => {\n const job = await Job.create(queue, data, {});\n const jobParsed = Job.fromJSON(queue, job.toData());\n\n expect(jobParsed.data).to.eql(data);\n });\n\n it('should not parse JSON data if \"preventParsingData\" option is specified', async () => {\n const job = await Job.create(queue, data, { preventParsingData: true });\n const jobParsed = Job.fromJSON(queue, job.toData());\n const expectedData = JSON.stringify(data);\n\n expect(jobParsed.data).to.be(expectedData);\n });\n });\n});\n" }, { "path": "test/test_metrics.js", "content": "'use strict';\n\nconst expect = require('chai').expect;\nconst utils = require('./utils');\nconst sinon = require('sinon');\nconst redis = require('ioredis');\n\nconst ONE_SECOND = 1000;\nconst ONE_MINUTE = 60 * ONE_SECOND;\nconst ONE_HOUR = 60 * ONE_MINUTE;\n\nconst { MetricsTime } = require('../lib/utils');\n\ndescribe('metrics', () => {\n beforeEach(async function() {\n this.clock = sinon.useFakeTimers();\n const client = new redis();\n await client.flushdb();\n return client.quit();\n });\n\n it('should gather metrics for completed jobs', async function() {\n const date = new Date('2017-02-07 9:24:00');\n this.clock.setSystemTime(date);\n this.clock.tick(0);\n\n const timmings = [\n 0,\n 0, // For the fixtures to work we need to use 0 as first timing\n ONE_MINUTE / 2,\n ONE_MINUTE / 2,\n 0,\n 0,\n ONE_MINUTE,\n ONE_MINUTE,\n ONE_MINUTE * 3,\n ONE_SECOND * 70,\n ONE_SECOND * 50,\n ONE_HOUR,\n ONE_MINUTE\n ];\n\n const fixture = [\n '1',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '1',\n '1',\n '1',\n '0',\n '0',\n '1',\n '1',\n '3',\n '3'\n ];\n\n const numJobs = timmings.length;\n\n const queue = utils.buildQueue('metrics', {\n metrics: {\n maxDataPoints: MetricsTime.ONE_HOUR * 2\n }\n });\n\n queue.process(job => {\n this.clock.tick(timmings[job.data.index]);\n });\n\n let processed = 0;\n const completing = new Promise(resolve => {\n queue.on('completed', async () => {\n processed++;\n if (processed === numJobs) {\n resolve();\n }\n });\n });\n\n for (let i = 0; i < numJobs; i++) {\n await queue.add({ index: i });\n }\n\n await completing;\n\n const metrics = await queue.getMetrics('completed');\n\n const numPoints = Math.floor(\n timmings.reduce((sum, timing) => sum + timing, 0) / ONE_MINUTE\n );\n\n expect(metrics.meta.count).to.be.equal(numJobs);\n expect(metrics.data.length).to.be.equal(numPoints);\n expect(metrics.count).to.be.equal(metrics.data.length);\n expect(processed).to.be.equal(numJobs);\n expect(metrics.data).to.be.deep.equal(fixture);\n\n this.clock.restore();\n await queue.close();\n });\n\n it('should only keep metrics for \"maxDataPoints\"', async function() {\n const date = new Date('2017-02-07 9:24:00');\n this.clock.setSystemTime(date);\n this.clock.tick(0);\n\n const timmings = [\n 0, // For the fixtures to work we need to use 0 as first timing\n 0,\n ONE_MINUTE / 2,\n ONE_MINUTE / 2,\n 0,\n 0,\n ONE_MINUTE,\n ONE_MINUTE,\n ONE_MINUTE * 3,\n ONE_HOUR,\n 0,\n 0,\n ONE_MINUTE,\n ONE_MINUTE\n ];\n\n const fixture = [\n '1',\n '3',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0',\n '0'\n ];\n\n const numJobs = timmings.length;\n\n const queue = utils.buildQueue('metrics', {\n metrics: {\n maxDataPoints: MetricsTime.FIFTEEN_MINUTES\n }\n });\n\n queue.process(job => {\n this.clock.tick(timmings[job.data.index]);\n });\n\n let processed = 0;\n const completing = new Promise(resolve => {\n queue.on('completed', async () => {\n processed++;\n if (processed === numJobs) {\n resolve();\n }\n });\n });\n\n for (let i = 0; i < numJobs; i++) {\n await queue.add({ index: i });\n }\n\n await completing;\n\n const metrics = await queue.getMetrics('completed');\n\n expect(metrics.meta.count).to.be.equal(numJobs);\n expect(metrics.data.length).to.be.equal(MetricsTime.FIFTEEN_MINUTES);\n expect(metrics.count).to.be.equal(metrics.data.length);\n expect(processed).to.be.equal(numJobs);\n expect(metrics.data).to.be.deep.equal(fixture);\n\n this.clock.restore();\n await queue.close();\n });\n\n it('should gather metrics for failed jobs', async function() {\n const date = new Date('2017-02-07 9:24:00');\n this.clock.setSystemTime(date);\n this.clock.tick(0);\n\n const timmings = [\n 0, // For the fixtures to work we need to use 0 as first timing\n ONE_MINUTE,\n ONE_MINUTE / 5,\n ONE_MINUTE / 2,\n 0,\n ONE_MINUTE,\n ONE_MINUTE * 3,\n 0\n ];\n\n const fixture = ['0', '0', '1', '4', '1'];\n\n const numJobs = timmings.length;\n\n const queue = utils.buildQueue('metrics', {\n metrics: {\n maxDataPoints: MetricsTime.ONE_HOUR * 2\n }\n });\n\n queue.process(async job => {\n this.clock.tick(timmings[job.data.index]);\n throw new Error('test');\n });\n\n let processed = 0;\n const completing = new Promise(resolve => {\n queue.on('failed', async () => {\n processed++;\n if (processed === numJobs) {\n resolve();\n }\n });\n });\n\n for (let i = 0; i < numJobs; i++) {\n await queue.add({ index: i });\n }\n\n await completing;\n\n const metrics = await queue.getMetrics('failed');\n\n const numPoints = Math.floor(\n timmings.reduce((sum, timing) => sum + timing, 0) / ONE_MINUTE\n );\n\n expect(metrics.meta.count).to.be.equal(numJobs);\n expect(metrics.data.length).to.be.equal(numPoints);\n expect(metrics.count).to.be.equal(metrics.data.length);\n expect(processed).to.be.equal(numJobs);\n expect(metrics.data).to.be.deep.equal(fixture);\n\n this.clock.restore();\n await queue.close();\n });\n\n it('should get metrics with pagination', async function() {\n const date = new Date('2017-02-07 9:24:00');\n this.clock.setSystemTime(date);\n this.clock.tick(0);\n\n const timmings = [\n 0,\n 0, // For the fixtures to work we need to use 0 as first timing\n ONE_MINUTE / 2,\n ONE_MINUTE / 2,\n 0,\n 0,\n ONE_MINUTE,\n ONE_MINUTE,\n ONE_MINUTE * 3,\n ONE_HOUR,\n ONE_MINUTE\n ];\n\n const numJobs = timmings.length;\n\n const queue = utils.buildQueue('metrics', {\n metrics: {\n maxDataPoints: MetricsTime.ONE_HOUR * 2\n }\n });\n\n queue.process(async job => {\n this.clock.tick(timmings[job.data.index]);\n });\n\n let processed = 0;\n const completing = new Promise(resolve => {\n queue.on('completed', async () => {\n processed++;\n if (processed === numJobs) {\n resolve();\n }\n });\n });\n\n for (let i = 0; i < numJobs; i++) {\n await queue.add({ index: i });\n }\n\n await completing;\n\n expect(processed).to.be.equal(numJobs);\n\n const numPoints = Math.floor(\n timmings.reduce((sum, timing) => sum + timing, 0) / ONE_MINUTE\n );\n\n const pageSize = 10;\n const data = [];\n let skip = 0;\n\n while (skip < numPoints) {\n const metrics = await queue.getMetrics(\n 'completed',\n skip,\n skip + pageSize - 1\n );\n expect(metrics.meta.count).to.be.equal(numJobs);\n expect(metrics.data.length).to.be.equal(\n Math.min(numPoints - skip, pageSize)\n );\n\n data.push(...metrics.data);\n skip += pageSize;\n }\n\n const metrics = await queue.getMetrics('completed');\n expect(data).to.be.deep.equal(metrics.data);\n\n this.clock.restore();\n await queue.close();\n });\n});\n" }, { "path": "test/test_obliterate.js", "content": "'use strict';\n\nconst expect = require('chai').expect;\nconst uuid = require('uuid');\nconst utils = require('./utils');\nconst delay = require('delay');\n\ndescribe('Obliterate', () => {\n let queue;\n\n beforeEach(() => {\n queue = utils.buildQueue('cleaner' + uuid.v4());\n });\n\n afterEach(function() {\n this.timeout(\n queue.settings.stalledInterval * (1 + queue.settings.maxStalledCount)\n );\n return queue.close();\n });\n\n it('should obliterate an empty queue', async () => {\n await queue.obliterate();\n\n const client = await queue.client;\n const keys = await client.keys(`bull:${queue.name}*`);\n\n expect(keys.length).to.be.eql(0);\n });\n\n it('should obliterate a queue with jobs in different statuses', async () => {\n await queue.add({ foo: 'bar' });\n await queue.add({ foo: 'bar2' });\n await queue.add({ foo: 'bar3' }, { delay: 5000 });\n const job = await queue.add({ qux: 'baz' });\n\n let first = true;\n queue.process(async () => {\n if (first) {\n first = false;\n throw new Error('failed first');\n }\n return delay(250);\n });\n\n await job.finished();\n\n await queue.obliterate();\n const client = await queue.client;\n const keys = await client.keys(`bull:${queue.name}*`);\n expect(keys.length).to.be.eql(0);\n });\n\n it('should raise exception if queue has active jobs', async () => {\n await queue.add({ foo: 'bar' });\n const job = await queue.add({ qux: 'baz' });\n\n await queue.add({ foo: 'bar2' });\n await queue.add({ foo: 'bar3' }, { delay: 5000 });\n\n let first = true;\n queue.process(async () => {\n if (first) {\n first = false;\n throw new Error('failed first');\n }\n return delay(250);\n });\n\n await job.finished();\n\n try {\n await queue.obliterate();\n } catch (err) {\n const client = await queue.client;\n const keys = await client.keys(`bull:${queue.name}*`);\n expect(keys.length).to.be.not.eql(0);\n return;\n }\n\n throw new Error('Should raise an exception if there are active jobs');\n });\n\n it('should obliterate if queue has active jobs using \"force\"', async () => {\n await queue.add({ foo: 'bar' });\n const job = await queue.add({ qux: 'baz' });\n\n await queue.add({ foo: 'bar2' });\n await queue.add({ foo: 'bar3' }, { delay: 5000 });\n\n let first = true;\n queue.process(async () => {\n if (first) {\n first = false;\n throw new Error('failed first');\n }\n return delay(250);\n });\n await job.finished();\n\n await queue.obliterate({ force: true });\n const client = await queue.client;\n const keys = await client.keys(`bull:${queue.name}*`);\n expect(keys.length).to.be.eql(0);\n });\n\n it('should remove repeatable jobs', async () => {\n await queue.add(\n 'test',\n { foo: 'bar' },\n {\n repeat: {\n every: 1000\n }\n }\n );\n\n const repeatableJobs = await queue.getRepeatableJobs();\n expect(repeatableJobs).to.have.length(1);\n\n await queue.obliterate();\n const client = await queue.client;\n const keys = await client.keys(`bull:${queue.name}:*`);\n expect(keys.length).to.be.eql(0);\n });\n\n it('should remove job logs', async () => {\n const job = await queue.add({});\n\n queue.process(job => {\n return job.log('Lorem Ipsum Dolor Sit Amet');\n });\n\n await job.finished();\n\n await queue.obliterate({ force: true });\n\n const { logs } = await queue.getJobLogs(job.id);\n expect(logs).to.have.length(0);\n });\n\n it('should obliterate a queue with high number of jobs in different statuses', async () => {\n const arr1 = [];\n for (let i = 0; i < 300; i++) {\n arr1.push(queue.add({ foo: `barLoop${i}` }));\n }\n\n const [lastCompletedJob] = (await Promise.all(arr1)).splice(-1);\n\n let fail = false;\n queue.process(async () => {\n if (fail) {\n throw new Error('failed job');\n }\n });\n\n await lastCompletedJob.finished();\n\n fail = true;\n\n const arr2 = [];\n for (let i = 0; i < 300; i++) {\n arr2.push(queue.add({ foo: `barLoop${i}` }));\n }\n\n const [lastFailedJob] = (await Promise.all(arr2)).splice(-1);\n\n try {\n await lastFailedJob.finished();\n expect(true).to.be.equal(false);\n } catch (err) {\n expect(true).to.be.equal(true);\n }\n\n const arr3 = [];\n for (let i = 0; i < 1623; i++) {\n arr3.push(queue.add({ foo: `barLoop${i}` }, { delay: 10000 }));\n }\n await Promise.all(arr3);\n\n await queue.obliterate();\n const client = await queue.client;\n const keys = await client.keys(`bull:${queue.name}*`);\n expect(keys.length).to.be.eql(0);\n }).timeout(20000);\n});\n" }, { "path": "test/test_pause.js", "content": "'use strict';\n\nconst Queue = require('../');\n\nconst expect = require('chai').expect;\nconst redis = require('ioredis');\nconst utils = require('./utils');\nconst delay = require('delay');\nconst sinon = require('sinon');\n\ndescribe('.pause', () => {\n let client;\n beforeEach(() => {\n client = new redis({\n maxRetriesPerRequest: null,\n enableReadyCheck: false\n });\n return client.flushdb();\n });\n\n afterEach(async () => {\n sinon.restore();\n await utils.cleanupQueues();\n await client.flushdb();\n return client.quit();\n });\n\n describe('globally', () => {\n it('should pause a queue until resumed', () => {\n let ispaused = false,\n counter = 2;\n\n return utils.newQueue().then(queue => {\n const resultPromise = new Promise(resolve => {\n queue.process((job, jobDone) => {\n expect(ispaused).to.be.eql(false);\n expect(job.data.foo).to.be.equal('paused');\n jobDone();\n counter--;\n if (counter === 0) {\n resolve(queue.close());\n }\n });\n });\n\n return Promise.all([\n queue\n .pause()\n .then(() => {\n ispaused = true;\n return queue.isPaused().then(paused => {\n expect(paused).to.be.equal(true);\n return queue.add({ foo: 'paused' });\n });\n })\n .then(() => {\n return queue.add({ foo: 'paused' });\n })\n .then(() => {\n ispaused = false;\n return queue.resume().then(() => {\n return queue.isPaused().then(paused => {\n expect(paused).to.be.equal(false);\n });\n });\n }),\n resultPromise\n ]);\n });\n });\n\n it('should be able to pause a running queue and emit relevant events', done => {\n let ispaused = false,\n isresumed = true,\n first = true;\n\n utils.newQueue().then(queue => {\n queue.process(job => {\n expect(ispaused).to.be.eql(false);\n expect(job.data.foo).to.be.equal('paused');\n\n if (first) {\n first = false;\n ispaused = true;\n return queue.pause();\n } else {\n expect(isresumed).to.be.eql(true);\n queue.close().then(done, done);\n }\n });\n\n queue.add({ foo: 'paused' });\n queue.add({ foo: 'paused' });\n\n queue.on('paused', () => {\n ispaused = false;\n queue.resume().catch((/*err*/) => {\n // Swallow error.\n });\n });\n\n queue.on('resumed', () => {\n isresumed = true;\n });\n });\n });\n\n it('should not processed delayed jobs', function(done) {\n this.timeout(5000);\n const queue = new Queue('pause-test');\n\n queue.process(() => {\n done(new Error('should not process delayed jobs in paused queue.'));\n });\n\n queue.pause().then(() => {\n queue\n .add(\n {},\n {\n delay: 500\n }\n )\n .then(() => {\n return queue.getJobCounts();\n })\n .then(counts => {\n expect(counts).to.have.property('paused', 0);\n expect(counts).to.have.property('waiting', 0);\n expect(counts).to.have.property('delayed', 1);\n return delay(1000);\n })\n .then(() => {\n return queue.getJobCounts();\n })\n .then(counts => {\n expect(counts).to.have.property('paused', 1);\n expect(counts).to.have.property('waiting', 0);\n done();\n });\n });\n });\n });\n\n describe('locally', () => {\n it('should pause the queue locally', done => {\n let counter = 2;\n\n const queue = utils.buildQueue();\n\n queue\n .pause(true /* Local */)\n .then(() => {\n return queue.isPaused(true).then(paused => {\n expect(paused).to.be.equal(true);\n });\n })\n .then(() => {\n // Add the worker after the queue is in paused mode since the normal behavior is to pause\n // it after the current lock expires. This way, we can ensure there isn't a lock already\n // to test that pausing behavior works.\n queue\n .process((job, jobDone) => {\n expect(queue.paused).not.to.be.ok;\n jobDone();\n counter--;\n if (counter === 0) {\n queue.close().then(done);\n }\n })\n .catch(done);\n })\n .then(() => {\n return queue.add({ foo: 'paused' });\n })\n .then(() => {\n return queue.add({ foo: 'paused' });\n })\n .then(() => {\n expect(counter).to.be.eql(2);\n expect(queue.paused).to.be.ok; // Parameter should exist.\n return queue.resume(true /* Local */);\n })\n .then(() => {\n return queue.isPaused(true).then(paused => {\n expect(paused).to.be.equal(false);\n });\n })\n .catch(done);\n });\n\n it('should wait until active jobs are finished before resolving pause', done => {\n const queue = utils.buildQueue();\n const startProcessing = new Promise(resolve => {\n queue.process((/*job*/) => {\n resolve();\n return delay(200);\n });\n });\n\n queue.isReady().then(() => {\n const jobs = [];\n for (let i = 0; i < 10; i++) {\n jobs.push(queue.add(i));\n }\n //\n // Add start processing so that we can test that pause waits for this job to be completed.\n //\n jobs.push(startProcessing);\n Promise.all(jobs)\n .then(() => {\n return queue\n .pause(true)\n .then(() => {\n const active = queue\n .getJobCountByTypes(['active'])\n .then(count => {\n expect(count).to.be.eql(0);\n expect(queue.paused).to.be.ok;\n return null;\n });\n\n // One job from the 10 posted above will be processed, so we expect 9 jobs pending\n const paused = queue\n .getJobCountByTypes(['delayed', 'wait'])\n .then(count => {\n expect(count).to.be.eql(9);\n return null;\n });\n return Promise.all([active, paused]);\n })\n .then(() => {\n return queue.add({});\n })\n .then(() => {\n const active = queue\n .getJobCountByTypes(['active'])\n .then(count => {\n expect(count).to.be.eql(0);\n return null;\n });\n\n const paused = queue\n .getJobCountByTypes(['paused', 'wait', 'delayed'])\n .then(count => {\n expect(count).to.be.eql(10);\n return null;\n });\n\n return Promise.all([active, paused]);\n })\n .then(() => {\n return queue.close().then(done, done);\n });\n })\n .catch(done);\n });\n });\n\n it('should pause the queue locally when more than one worker is active', () => {\n const queue1 = utils.buildQueue('pause-queue');\n const queue1IsProcessing = new Promise(resolve => {\n queue1.process((job, jobDone) => {\n resolve();\n setTimeout(jobDone, 200);\n });\n });\n\n const queue2 = utils.buildQueue('pause-queue');\n const queue2IsProcessing = new Promise(resolve => {\n queue2.process((job, jobDone) => {\n resolve();\n setTimeout(jobDone, 200);\n });\n });\n\n queue1.add(1);\n queue1.add(2);\n queue1.add(3);\n queue1.add(4);\n\n return Promise.all([queue1IsProcessing, queue2IsProcessing]).then(() => {\n return Promise.all([\n queue1.pause(true /* local */),\n queue2.pause(true /* local */)\n ]).then(() => {\n const active = queue1.getJobCountByTypes(['active']).then(count => {\n expect(count).to.be.eql(0);\n });\n\n const pending = queue1.getJobCountByTypes(['wait']).then(count => {\n expect(count).to.be.eql(2);\n });\n\n const completed = queue1\n .getJobCountByTypes(['completed'])\n .then(count => {\n expect(count).to.be.eql(2);\n });\n\n return Promise.all([active, pending, completed]).then(() => {\n return Promise.all([queue1.close(), queue2.close()]);\n });\n });\n });\n });\n\n it('should wait for blocking job retrieval to complete before pausing', () => {\n const queue = utils.buildQueue();\n\n const startsProcessing = new Promise(resolve => {\n queue.process((/*job*/) => {\n resolve();\n return delay(200);\n });\n });\n\n return queue\n .add(1)\n .then(() => {\n return startsProcessing;\n })\n .then(() => {\n return queue.pause(true);\n })\n .then(() => {\n return queue.add(2);\n })\n .then(() => {\n const active = queue.getJobCountByTypes(['active']).then(count => {\n expect(count).to.be.eql(0);\n });\n\n const pending = queue.getJobCountByTypes(['wait']).then(count => {\n expect(count).to.be.eql(1);\n });\n\n const completed = queue\n .getJobCountByTypes(['completed'])\n .then(count => {\n expect(count).to.be.eql(1);\n });\n\n return Promise.all([active, pending, completed]).then(() => {\n return queue.close();\n });\n });\n });\n\n it('should not initialize blocking client if not already initialized', async () => {\n const createClient = sinon.spy(() => client);\n const queue = utils.buildQueue('pause-queue', { createClient });\n\n await queue.pause(true);\n const bClientCalls = createClient\n .getCalls()\n .filter(c => c.args[0] === 'bclient');\n expect(bClientCalls).to.have.lengthOf(0);\n });\n\n it('pauses fast when queue is drained', function(done) {\n this.timeout(10000);\n const queue = new Queue('test');\n\n queue.process((/*job*/) => {\n Promise.resolve();\n });\n\n queue.add({});\n\n queue.on('drained', () => {\n delay(500).then(() => {\n const start = new Date().getTime();\n return queue.pause(true).finally(() => {\n const finish = new Date().getTime();\n expect(finish - start).to.be.lt(1000);\n queue.close().then(done, done);\n });\n });\n });\n });\n\n describe('with doNotWaitActive=true', () => {\n it('should not wait for active jobs to finish', async () => {\n const queue = utils.buildQueue();\n await queue.add({});\n\n let finishJob;\n\n // wait for us to start processing job\n await new Promise(resolve => {\n queue.process(() => {\n // resolve promise, but continue processing job forever\n resolve();\n\n return new Promise(resolve => {\n finishJob = resolve;\n });\n });\n });\n\n return queue.pause(true, true).then(() => finishJob());\n });\n\n it('should not process new jobs', async () => {\n const queue = utils.buildQueue();\n\n // block on brpoplpush\n const nextJobPromise = queue.getNextJob();\n\n await queue.pause(true, true);\n\n // Add job. This should not be processed\n await queue.add({});\n\n const nextJob = await nextJobPromise;\n expect(nextJob).to.equal(\n undefined,\n 'getNextJob should return without getting job'\n );\n });\n\n it('should not initialize blocking client if not already initialized', async () => {\n const createClient = sinon.spy(() => client);\n const queue = utils.buildQueue('pause-queue', { createClient });\n\n await queue.pause(true, true);\n const bClientCalls = createClient\n .getCalls()\n .filter(c => c.args[0] === 'bclient');\n expect(bClientCalls).to.have.lengthOf(0);\n });\n });\n });\n});\n" }, { "path": "test/test_queue.js", "content": "'use strict';\n\nconst Queue = require('../');\nconst expect = require('chai').expect;\nconst redis = require('ioredis');\nconst sinon = require('sinon');\nconst _ = require('lodash');\nconst uuid = require('uuid');\nconst utils = require('./utils');\nconst delay = require('delay');\n\ndescribe('Queue', () => {\n const sandbox = sinon.createSandbox();\n let client;\n\n beforeEach(() => {\n client = new redis();\n return client.flushdb();\n });\n\n afterEach(() => {\n sandbox.restore();\n return client.quit();\n });\n\n describe('.close', () => {\n let testQueue;\n beforeEach(() => {\n return utils.newQueue('test').then(queue => {\n testQueue = queue;\n });\n });\n\n it('should call end on the client', done => {\n testQueue.client.once('end', () => {\n done();\n });\n testQueue.close();\n });\n\n it('should call end on the event subscriber client', done => {\n testQueue.eclient.once('end', () => {\n done();\n });\n testQueue.close();\n });\n\n it('should resolve the promise when each client has disconnected', () => {\n function checkStatus(status) {\n return (\n status === 'ready' || status === 'connecting' || status === 'connect'\n );\n }\n expect(testQueue.client.status).to.satisfy(checkStatus);\n expect(testQueue.eclient.status).to.satisfy(checkStatus);\n\n return testQueue.close().then(() => {\n expect(testQueue.client.status).to.be.eql('end');\n expect(testQueue.eclient.status).to.be.eql('end');\n });\n });\n\n it('should return a promise', () => {\n const closePromise = testQueue.close().then(() => {\n expect(closePromise.then).to.be.a('function');\n });\n return closePromise;\n });\n\n it('should close if the job expires after the lockRenewTime', function(done) {\n this.timeout(testQueue.settings.stalledInterval * 2, {\n settings: {\n lockDuration: 15,\n lockRenewTime: 5\n }\n });\n\n testQueue.process(() => {\n return delay(100);\n });\n\n testQueue.on('completed', () => {\n testQueue.close().then(done);\n });\n testQueue.add({ foo: 'bar' });\n });\n\n describe('should be callable from within', () => {\n it('a job handler that takes a callback', function(done) {\n this.timeout(12000); // Close can be a slow operation\n\n testQueue.process((job, jobDone) => {\n expect(job.data.foo).to.be.eql('bar');\n jobDone();\n testQueue.close().then(done);\n });\n\n testQueue.add({ foo: 'bar' }).then(job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n });\n });\n\n it('a job handler that returns a promise', done => {\n testQueue.process(job => {\n expect(job.data.foo).to.be.eql('bar');\n return Promise.resolve();\n });\n\n testQueue.on('completed', () => {\n testQueue.close().then(done);\n });\n\n testQueue.add({ foo: 'bar' }).then(job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n });\n });\n });\n });\n\n describe('instantiation', () => {\n it('should create a queue with standard redis opts', () => {\n const queue = new Queue('standard');\n\n expect(queue.client.options.host).to.be.eql('127.0.0.1');\n expect(queue.eclient.options.host).to.be.eql('127.0.0.1');\n\n expect(queue.client.options.port).to.be.eql(6379);\n expect(queue.eclient.options.port).to.be.eql(6379);\n\n expect(queue.client.options.db).to.be.eql(0);\n expect(queue.eclient.options.db).to.be.eql(0);\n\n return queue.close();\n });\n\n it('should create a queue with a redis connection string', () => {\n const queue = new Queue('connstring', 'redis://123.4.5.67:1234/2', {\n redis: { connectTimeout: 1000, retryStrategy: () => false }\n });\n\n expect(queue.client.options.host).to.be.eql('123.4.5.67');\n expect(queue.eclient.options.host).to.be.eql('123.4.5.67');\n\n expect(queue.client.options.port).to.be.eql(1234);\n expect(queue.eclient.options.port).to.be.eql(1234);\n\n expect(queue.client.options.db).to.be.eql(2);\n expect(queue.eclient.options.db).to.be.eql(2);\n\n return queue.close();\n }).timeout(10000);\n\n it('should create a queue with only a hostname', () => {\n const queue = new Queue('connstring', 'redis://127.2.3.4', {\n redis: { connectTimeout: 1000, retryStrategy: () => false }\n });\n\n expect(queue.client.options.host).to.be.eql('127.2.3.4');\n expect(queue.eclient.options.host).to.be.eql('127.2.3.4');\n\n expect(queue.client.options.port).to.be.eql(6379);\n expect(queue.eclient.options.port).to.be.eql(6379);\n\n expect(queue.client.condition.select).to.be.eql(0);\n expect(queue.eclient.condition.select).to.be.eql(0);\n\n return queue.close().catch((/*err*/) => {\n // Swallow error.\n });\n });\n\n it('should create a queue with connection string and password', () => {\n const queue = new Queue('connstring', 'redis://:123@127.2.3.4:6379');\n\n expect(queue.client.options.host).to.be.eql('127.2.3.4');\n expect(queue.eclient.options.host).to.be.eql('127.2.3.4');\n\n expect(queue.client.options.port).to.be.eql(6379);\n expect(queue.eclient.options.port).to.be.eql(6379);\n\n expect(queue.client.condition.select).to.be.eql(0);\n expect(queue.eclient.condition.select).to.be.eql(0);\n\n expect(queue.client.options.password).to.be.eql('123');\n expect(queue.eclient.options.password).to.be.eql('123');\n\n queue.close().catch((/*err*/) => {\n // Swallow error.\n });\n });\n\n it('creates a queue using the supplied redis DB', () => {\n const queue = new Queue('custom', { redis: { DB: 1 } });\n\n expect(queue.client.options.host).to.be.eql('127.0.0.1');\n expect(queue.eclient.options.host).to.be.eql('127.0.0.1');\n\n expect(queue.client.options.port).to.be.eql(6379);\n expect(queue.eclient.options.port).to.be.eql(6379);\n\n expect(queue.client.options.db).to.be.eql(1);\n expect(queue.eclient.options.db).to.be.eql(1);\n\n return queue.close();\n });\n\n it('creates a queue using the supplied redis url as opts', () => {\n const queue = new Queue('custom', {\n redis: 'redis://abc:123@127.2.3.4:1234/1'\n });\n\n expect(queue.client.options.host).to.be.eql('127.2.3.4');\n expect(queue.eclient.options.host).to.be.eql('127.2.3.4');\n\n expect(queue.client.options.port).to.be.eql(1234);\n expect(queue.eclient.options.port).to.be.eql(1234);\n\n expect(queue.client.options.db).to.be.eql(1);\n expect(queue.eclient.options.db).to.be.eql(1);\n\n return queue.close();\n });\n\n it('creates a queue using the supplied redis host', () => {\n const queue = new Queue('custom', { redis: { host: 'localhost' } });\n\n expect(queue.client.options.host).to.be.eql('localhost');\n expect(queue.eclient.options.host).to.be.eql('localhost');\n\n expect(queue.client.options.db).to.be.eql(0);\n expect(queue.eclient.options.db).to.be.eql(0);\n\n return queue.close();\n });\n\n it('creates a queue with dots in its name', () => {\n const queue = new Queue('using. dots. in.name.');\n\n return queue\n .add({ foo: 'bar' })\n .then(job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n })\n .then(() => {\n queue.process((job, jobDone) => {\n expect(job.data.foo).to.be.equal('bar');\n jobDone();\n });\n return null;\n })\n .then(() => {\n return queue.close();\n });\n });\n\n it('creates a queue accepting port as a string', () => {\n const queue = new Queue('foobar', '6379', 'localhost');\n\n return queue\n .add({ foo: 'bar' })\n .then(job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n })\n .then(() => {\n queue.process((job, jobDone) => {\n expect(job.data.foo).to.be.equal('bar');\n jobDone();\n });\n return null;\n })\n .then(() => {\n return queue.close();\n });\n });\n\n it('should create a queue with a prefix option', () => {\n const queue = new Queue('q', 'redis://127.0.0.1', { keyPrefix: 'myQ' });\n\n return queue\n .add({ foo: 'bar' })\n .then(job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n const client = new redis();\n return client.hgetall('myQ:q:' + job.id).then(result => {\n expect(result).to.not.be.null;\n return client.quit();\n });\n })\n .then(() => {\n return queue.close();\n });\n });\n\n it('should allow reuse redis connections', done => {\n const redisOpts = {\n maxRetriesPerRequest: null,\n enableReadyCheck: false\n };\n const client = new redis(redisOpts);\n const subscriber = new redis(redisOpts);\n\n const opts = {\n createClient(type, opts) {\n switch (type) {\n case 'client':\n return client;\n case 'subscriber':\n return subscriber;\n case 'bclient':\n return new redis({ ...opts, ...redisOpts });\n default:\n return new redis(opts);\n }\n }\n };\n const queueFoo = new Queue('foobar', opts);\n const queueQux = new Queue('quxbaz', opts);\n\n expect(queueFoo.client).to.be.equal(client);\n expect(queueFoo.eclient).to.be.equal(subscriber);\n\n expect(queueQux.client).to.be.equal(client);\n expect(queueQux.eclient).to.be.equal(subscriber);\n\n queueFoo\n .add({ foo: 'bar' })\n .then(job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n })\n .then(\n queueFoo.bclient.client('GETNAME').then(name => {\n expect(name).to.be.eql(queueFoo.clientName());\n })\n )\n .then(() => {\n return queueQux.add({ qux: 'baz' }).then(job => {\n expect(job.id).to.be.ok;\n expect(job.data.qux).to.be.eql('baz');\n let completed = 0;\n\n queueFoo.process((job, jobDone) => {\n jobDone();\n });\n\n queueQux.process((job, jobDone) => {\n jobDone();\n });\n\n queueFoo.on('completed', () => {\n completed++;\n if (completed == 2) {\n done();\n }\n });\n\n queueQux.on('completed', () => {\n completed++;\n if (completed == 2) {\n done();\n }\n });\n });\n }, done);\n });\n\n it('creates a queue with default job options', async () => {\n const defaultJobOptions = { removeOnComplete: true, removeOnFail: false };\n const queue = new Queue('custom', {\n defaultJobOptions\n });\n\n expect(queue.defaultJobOptions).to.be.eql(defaultJobOptions);\n\n const job = await queue.add('test', {}, { removeOnFail: true });\n\n expect(job.opts).have.property('removeOnComplete', true);\n expect(job.opts).have.property('removeOnFail', true);\n\n await queue.close();\n });\n\n it('should not change the options object', async () => {\n const originalOptions = { redis: { keyPrefix: 'myQ' } };\n const options = _.cloneDeep(originalOptions);\n\n let queue = new Queue('q', 'redis://127.0.0.1', options);\n expect(_.isEqual(options, originalOptions)).to.be.true;\n await queue.close();\n\n queue = new Queue('q', options);\n expect(_.isEqual(options, originalOptions)).to.be.true;\n await queue.close();\n });\n\n describe('bulk jobs', () => {\n it('should default name of job', () => {\n const queue = new Queue('custom');\n\n return queue.addBulk([{ name: 'specified' }, {}]).then(jobs => {\n expect(jobs).to.have.length(2);\n\n expect(jobs[0].name).to.equal('specified');\n expect(jobs[1].name).to.equal('__default__');\n });\n });\n\n it('should default options from queue', () => {\n const queue = new Queue('custom', {\n defaultJobOptions: {\n removeOnComplete: true\n }\n });\n\n return queue.addBulk([{}]).then(jobs => {\n expect(jobs[0].opts.removeOnComplete).to.equal(true);\n });\n });\n });\n });\n\n describe('a worker', () => {\n let queue;\n\n beforeEach(() => {\n const client = new redis();\n return client\n .flushdb()\n .then(() => {\n return utils.newQueue();\n })\n .then(_queue => {\n queue = _queue;\n });\n });\n\n afterEach(function() {\n this.timeout(\n queue.settings.stalledInterval * (1 + queue.settings.maxStalledCount)\n );\n return utils.cleanupQueues();\n });\n\n it('should process a job', done => {\n queue\n .process((job, jobDone) => {\n expect(job.data.foo).to.be.equal('bar');\n jobDone();\n done();\n })\n .catch(done);\n\n queue.add({ foo: 'bar' }).then(job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n }, done);\n });\n\n describe('bulk jobs', () => {\n it('should process jobs', done => {\n queue\n .process((job, jobDone) => {\n if (job.data.idx === 0) {\n expect(job.data.foo).to.be.equal('bar');\n jobDone();\n } else {\n expect(job.data.idx).to.be.equal(1);\n expect(job.data.foo).to.be.equal('baz');\n jobDone();\n done();\n }\n })\n .catch(done);\n\n queue\n .addBulk([\n { data: { idx: 0, foo: 'bar' } },\n { data: { idx: 1, foo: 'baz' } }\n ])\n .then(jobs => {\n expect(jobs).to.have.length(2);\n\n expect(jobs[0].id).to.be.ok;\n expect(jobs[0].data.foo).to.be.eql('bar');\n expect(jobs[1].id).to.be.ok;\n expect(jobs[1].data.foo).to.be.eql('baz');\n }, done);\n });\n });\n\n describe('auto job removal', () => {\n async function testRemoveOnFinish(opts, expectedCount, fail) {\n const clock = sinon.useFakeTimers();\n clock.reset();\n\n queue.process(async job => {\n await job.log('test log');\n if (fail) {\n throw new Error('job failed');\n }\n });\n\n const datas = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14];\n\n const processing = new Promise(resolve => {\n queue.on(fail ? 'failed' : 'completed', async job => {\n clock.tick(1000);\n\n if (job.data == 14) {\n const counts = await queue.getJobCounts(\n fail ? 'failed' : 'completed'\n );\n\n if (fail) {\n expect(counts.failed).to.be.equal(expectedCount);\n } else {\n expect(counts.completed).to.be.equal(expectedCount);\n }\n\n await Promise.all(\n jobIds.map(async (jobId, index) => {\n const job = await queue.getJob(jobId);\n const logs = await queue.getJobLogs(jobId);\n\n try {\n if (index >= datas.length - expectedCount) {\n expect(job).to.not.be.equal(null);\n expect(logs.logs).to.not.be.empty;\n } else {\n expect(job).to.be.equal(null);\n expect(logs.logs).to.be.empty;\n }\n } catch (err) {\n console.error(err);\n }\n })\n );\n\n resolve();\n }\n });\n });\n\n const jobOpts = {};\n if (fail) {\n jobOpts.removeOnFail = opts;\n } else {\n jobOpts.removeOnComplete = opts;\n }\n\n const jobIds = (\n await Promise.all(datas.map(async data => queue.add(data, jobOpts)))\n ).map(job => job.id);\n\n await processing;\n clock.restore();\n }\n\n it('should remove job after completed if removeOnComplete', done => {\n queue\n .process((job, jobDone) => {\n expect(job.data.foo).to.be.equal('bar');\n jobDone();\n })\n .catch(done);\n\n queue.add({ foo: 'bar' }, { removeOnComplete: true }).then(job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n }, done);\n\n queue.on('completed', job => {\n queue\n .getJob(job.id)\n .then(job => {\n expect(job).to.be.equal(null);\n })\n .then(() => {\n queue.getJobCounts().then(counts => {\n expect(counts.completed).to.be.equal(0);\n done();\n });\n });\n });\n });\n\n it('should remove a job after completed if the default job options specify removeOnComplete', done => {\n utils\n .newQueue('test-' + uuid.v4(), {\n defaultJobOptions: {\n removeOnComplete: true\n }\n })\n .then(myQueue => {\n myQueue.process(job => {\n expect(job.data.foo).to.be.equal('bar');\n });\n\n myQueue\n .add({ foo: 'bar' })\n .then(job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n }, done)\n .catch(done);\n\n myQueue.on('completed', job => {\n myQueue\n .getJob(job.id)\n .then(job => {\n expect(job).to.be.equal(null);\n })\n .then(() => {\n return myQueue.getJobCounts();\n })\n .then(counts => {\n expect(counts.completed).to.be.equal(0);\n\n return utils.cleanupQueues();\n })\n .then(done)\n .catch(done);\n });\n return null;\n })\n .catch(done);\n });\n\n describe('.retryJobs', () => {\n it('should retry all failed jobs', async () => {\n const jobCount = 8;\n\n let fail = true;\n queue.process(async () => {\n await delay(10);\n if (fail) {\n throw new Error('failed');\n }\n });\n\n let order = 0;\n const failing = new Promise(resolve => {\n queue.on('failed', job => {\n expect(order).to.be.eql(job.data.idx);\n if (order === jobCount - 1) {\n resolve();\n }\n order++;\n });\n });\n\n for (const index of Array.from(Array(jobCount).keys())) {\n await queue.add({ idx: index });\n }\n\n await failing;\n\n const failedCount = await queue.getJobCounts('failed');\n expect(failedCount.failed).to.be.equal(jobCount);\n\n order = 0;\n const completing = new Promise(resolve => {\n queue.on('completed', job => {\n expect(order).to.be.eql(job.data.idx);\n if (order === jobCount - 1) {\n resolve();\n }\n order++;\n });\n });\n\n fail = false;\n await queue.retryJobs({ count: 2 });\n\n await completing;\n\n const CompletedCount = await queue.getJobCounts('completed');\n expect(CompletedCount.completed).to.be.equal(jobCount);\n });\n\n it('should move to pause all failed jobs if the queue is paused', async () => {\n const jobCount = 8;\n\n let fail = true;\n queue.process(async () => {\n await delay(10);\n if (fail) {\n throw new Error('failed');\n }\n });\n\n let order = 0;\n const failing = new Promise(resolve => {\n queue.on('failed', job => {\n expect(order).to.be.eql(job.data.idx);\n if (order === jobCount - 1) {\n resolve();\n }\n order++;\n });\n });\n\n for (const index of Array.from(Array(jobCount).keys())) {\n await queue.add({ idx: index });\n }\n\n await failing;\n\n const failedCount = await queue.getJobCounts('failed');\n expect(failedCount.failed).to.be.equal(jobCount);\n\n order = 0;\n const completing = new Promise(resolve => {\n queue.on('completed', job => {\n expect(order).to.be.eql(job.data.idx);\n if (order === jobCount - 1) {\n resolve();\n }\n order++;\n });\n });\n\n fail = false;\n\n await queue.pause();\n\n await queue.retryJobs({ count: 2 });\n\n const pausedJobs = await queue.getJobs(['paused']);\n expect(pausedJobs).to.have.lengthOf(jobCount);\n\n await queue.resume();\n\n await completing;\n\n const CompletedCount = await queue.getJobCounts('completed');\n expect(CompletedCount.completed).to.be.equal(jobCount);\n });\n });\n\n it('should keep specified number of jobs after completed with removeOnComplete', async () => {\n const keepJobs = 3;\n await testRemoveOnFinish(keepJobs, keepJobs);\n });\n\n it('should keep of jobs newer than specified after completed with removeOnComplete', async () => {\n const age = 7;\n await testRemoveOnFinish({ age }, age);\n });\n\n it('should keep of jobs newer than specified and up to a count completed with removeOnComplete', async () => {\n const age = 7;\n const count = 5;\n await testRemoveOnFinish({ age, count }, count);\n });\n\n it('should keep of jobs newer than specified and up to a count fail with removeOnFail', async () => {\n const age = 7;\n const count = 5;\n await testRemoveOnFinish({ age, count }, count, true);\n });\n\n /*\n it('should keep specified number of jobs after completed with removeOnComplete', async () => {\n const keepJobs = 3;\n queue.process(async job => {\n await job.log('test log');\n });\n\n const datas = [0, 1, 2, 3, 4, 5, 6, 7, 8];\n\n const jobIds = await Promise.all(\n datas.map(\n async data =>\n (await queue.add(data, { removeOnComplete: keepJobs })).id\n )\n );\n\n return new Promise(resolve => {\n queue.on('completed', async job => {\n if (job.data == 8) {\n const counts = await queue.getJobCounts();\n expect(counts.completed).to.be.equal(keepJobs);\n\n await Promise.all(\n jobIds.map(async (jobId, index) => {\n const job = await queue.getJob(jobId);\n const logs = await queue.getJobLogs(jobId);\n if (index >= datas.length - keepJobs) {\n expect(job).to.not.be.equal(null);\n expect(logs.logs).to.not.be.empty;\n } else {\n expect(job).to.be.equal(null);\n expect(logs.logs).to.be.empty;\n }\n })\n );\n resolve();\n }\n });\n });\n });\n */\n\n it('should keep specified number of jobs after completed with global removeOnComplete', async () => {\n const keepJobs = 3;\n\n const localQueue = await utils.newQueue('test-' + uuid.v4(), {\n defaultJobOptions: {\n removeOnComplete: keepJobs\n }\n });\n localQueue.process(() => {});\n\n const datas = [0, 1, 2, 3, 4, 5, 6, 7, 8];\n\n const jobIds = await Promise.all(\n datas.map(async data => (await localQueue.add(data)).id)\n );\n\n return new Promise((resolve, reject) => {\n localQueue.on('completed', async job => {\n if (job.data == 8) {\n try {\n const counts = await localQueue.getJobCounts();\n expect(counts.completed).to.be.equal(keepJobs);\n\n await Promise.all(\n jobIds.map(async (jobId, index) => {\n const job = await localQueue.getJob(jobId);\n if (index >= datas.length - keepJobs) {\n expect(job).to.not.be.equal(null);\n } else {\n expect(job).to.be.equal(null);\n }\n })\n );\n } catch (err) {\n reject(err);\n }\n\n resolve();\n }\n });\n });\n });\n\n it('should remove job after failed if removeOnFail', done => {\n queue.process(job => {\n expect(job.data.foo).to.be.equal('bar');\n throw Error('error');\n });\n\n queue.add({ foo: 'bar' }, { removeOnFail: true }).then(job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n }, done);\n\n queue.on('failed', jobId => {\n queue\n .getJob(jobId)\n .then(job => {\n expect(job).to.be.equal(null);\n return null;\n })\n .then(() => {\n return queue.getJobCounts().then(counts => {\n expect(counts.failed).to.be.equal(0);\n done();\n });\n });\n });\n });\n\n it('should remove a job after fail if the default job options specify removeOnFail', done => {\n utils\n .newQueue('test-' + uuid.v4(), {\n defaultJobOptions: {\n removeOnFail: true\n }\n })\n .then(myQueue => {\n myQueue.process(job => {\n expect(job.data.foo).to.be.equal('bar');\n throw Error('error');\n });\n\n myQueue\n .add({ foo: 'bar' })\n .then(job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n }, done)\n .catch(done);\n\n myQueue.on('failed', jobId => {\n myQueue\n .getJob(jobId)\n .then(job => {\n expect(job).to.be.equal(null);\n })\n .then(() => {\n return myQueue.getJobCounts();\n })\n .then(counts => {\n expect(counts.completed).to.be.equal(0);\n\n return utils.cleanupQueues();\n })\n .then(done)\n .catch(done);\n });\n return null;\n })\n .catch(done);\n });\n\n it('should keep specified number of jobs after completed with removeOnFail', async () => {\n const keepJobs = 3;\n queue.process(() => {\n throw Error('error');\n });\n\n const datas = [0, 1, 2, 3, 4, 5, 6, 7, 8];\n\n const jobIds = await Promise.all(\n datas.map(\n async data => (await queue.add(data, { removeOnFail: keepJobs })).id\n )\n );\n\n return new Promise(resolve => {\n queue.on('failed', async job => {\n if (job.data == 8) {\n const counts = await queue.getJobCounts();\n expect(counts.failed).to.be.equal(keepJobs);\n\n await Promise.all(\n jobIds.map(async (jobId, index) => {\n const job = await queue.getJob(jobId);\n if (index >= datas.length - keepJobs) {\n expect(job).to.not.be.equal(null);\n } else {\n expect(job).to.be.equal(null);\n }\n })\n );\n\n resolve();\n }\n });\n });\n });\n\n it('should keep specified number of jobs after completed with global removeOnFail', async () => {\n const keepJobs = 3;\n\n const localQueue = await utils.newQueue('test-' + uuid.v4(), {\n defaultJobOptions: {\n removeOnFail: keepJobs\n }\n });\n localQueue.process(() => {\n throw Error('error');\n });\n\n const datas = [0, 1, 2, 3, 4, 5, 6, 7, 8];\n\n const jobIds = await Promise.all(\n datas.map(async data => (await localQueue.add(data)).id)\n );\n\n return new Promise((resolve, reject) => {\n localQueue.on('failed', async job => {\n if (job.data == 8) {\n try {\n const counts = await localQueue.getJobCounts();\n expect(counts.failed).to.be.equal(keepJobs);\n\n await Promise.all(\n jobIds.map(async (jobId, index) => {\n const job = await localQueue.getJob(jobId);\n if (index >= datas.length - keepJobs) {\n expect(job).to.not.be.equal(null);\n } else {\n expect(job).to.be.equal(null);\n }\n })\n );\n } catch (err) {\n reject(err);\n }\n\n resolve();\n }\n });\n });\n });\n });\n\n it('process a lifo queue', function(done) {\n this.timeout(3000);\n let currentValue = 0,\n first = true;\n utils.newQueue('test lifo').then(queue2 => {\n queue2.process((job, jobDone) => {\n // Catching the job before the pause\n expect(job.data.count).to.be.equal(currentValue--);\n jobDone();\n if (first) {\n first = false;\n } else if (currentValue === 0) {\n done();\n }\n });\n\n queue2.pause().then(() => {\n // Add a series of jobs in a predictable order\n const jobs = [\n { count: ++currentValue },\n { count: ++currentValue },\n { count: ++currentValue },\n { count: ++currentValue }\n ];\n return Promise.all(\n jobs.map(jobData => {\n return queue2.add(jobData, { lifo: true });\n })\n ).then(() => {\n queue2.resume();\n });\n });\n });\n });\n\n it('should processes jobs by priority', done => {\n const normalPriority = [];\n const mediumPriority = [];\n const highPriority = [];\n\n // for the current strategy this number should not exceed 8 (2^2*2)\n // this is done to maitain a deterministic output.\n const numJobsPerPriority = 6;\n\n for (let i = 0; i < numJobsPerPriority; i++) {\n normalPriority.push(queue.add({ p: 2 }, { priority: 2 }));\n mediumPriority.push(queue.add({ p: 3 }, { priority: 3 }));\n highPriority.push(queue.add({ p: 1 }, { priority: 1 }));\n }\n\n // wait for all jobs to enter the queue and then start processing\n Promise.all(normalPriority, mediumPriority, highPriority).then(() => {\n let currentPriority = 1;\n let counter = 0;\n let total = 0;\n\n queue.process((job, jobDone) => {\n expect(job.id).to.be.ok;\n expect(job.data.p).to.be.eql(currentPriority);\n jobDone();\n\n total++;\n if (++counter === numJobsPerPriority) {\n currentPriority++;\n counter = 0;\n\n if (currentPriority === 4 && total === numJobsPerPriority * 3) {\n done();\n }\n }\n });\n }, done);\n });\n\n it('process several jobs serially', function(done) {\n this.timeout(12000);\n let counter = 1;\n const maxJobs = 35;\n\n queue.process((job, jobDone) => {\n expect(job.data.num).to.be.equal(counter);\n expect(job.data.foo).to.be.equal('bar');\n jobDone();\n if (counter === maxJobs) {\n done();\n }\n counter++;\n });\n\n for (let i = 1; i <= maxJobs; i++) {\n queue.add({ foo: 'bar', num: i });\n }\n });\n\n describe('when job has been added again', () => {\n it('emits global duplicated event', async () => {\n queue.process(\n async () => {\n await delay(50);\n await queue.add({ foo: 'bar' }, { jobId: 'a1' });\n await delay(50);\n }\n );\n \n await queue.add({ foo: 'bar' }, { jobId: 'a1' });\n \n await new Promise(resolve => {\n queue.once('global:duplicated', (jobId) => {\n expect(jobId).to.be.equal('a1');\n resolve();\n });\n });\n });\n\n it('emits duplicated event', async () => {\n queue.process(\n async () => {\n await delay(50);\n await queue.add({ foo: 'bar' }, { jobId: 'a1' });\n await delay(50);\n }\n );\n \n await queue.add({ foo: 'bar' }, { jobId: 'a1' });\n \n await new Promise(resolve => {\n queue.once('duplicated', (jobId) => {\n expect(jobId).to.be.equal('a1');\n resolve();\n });\n });\n });\n });\n\n describe('when job is debounced when added again with same debounce id', () => {\n describe('when ttl is provided', () => {\n it('used a fixed time period and emits debounced event', async () => {\n const job = await queue.add(\n { foo: 'bar' },\n { debounce: { id: 'a1', ttl: 2000 } },\n );\n \n let debouncedCounter = 0;\n let secondJob = null;\n queue.on('debounced', (jobId) => {\n if (debouncedCounter > 1) {\n expect(jobId).to.be.equal(secondJob.id);\n } else {\n expect(jobId).to.be.equal(job.id);\n }\n debouncedCounter++;\n });\n \n await delay(1000);\n await queue.add(\n { foo: 'bar' },\n { debounce: { id: 'a1', ttl: 2000 } },\n );\n await queue.add(\n { foo: 'bar' },\n { debounce: { id: 'a1', ttl: 2000 } },\n );\n await delay(1100);\n secondJob = await queue.add(\n { foo: 'bar' },\n { debounce: { id: 'a1', ttl: 2000 } },\n );\n await queue.add(\n { foo: 'bar' },\n { debounce: { id: 'a1', ttl: 2000 } },\n );\n await queue.add(\n { foo: 'bar' },\n { debounce: { id: 'a1', ttl: 2000 } },\n );\n await delay(100);\n \n expect(debouncedCounter).to.be.equal(4);\n });\n });\n\n describe('when removing debounced job', () => {\n it('removes debounce key', async ()=> {\n const job = await queue.add(\n { foo: 'bar' },\n { debounce: { id: 'a1', ttl: 2000 } },\n );\n\n let debouncedCounter = 0;\n queue.on('debounced', () => {\n debouncedCounter++;\n });\n await job.remove();\n\n await queue.add(\n { foo: 'bar' },\n { debounce: { id: 'a1', ttl: 2000 } },\n );\n await delay(1000);\n await queue.add(\n { foo: 'bar' },\n { debounce: { id: 'a1', ttl: 2000 } },\n );\n await delay(1100);\n const secondJob = await queue.add(\n { foo: 'bar' },\n { debounce: { id: 'a1', ttl: 2000 } },\n );\n await secondJob.remove();\n\n await queue.add(\n { foo: 'bar' },\n { debounce: { id: 'a1', ttl: 2000 } },\n );\n await queue.add(\n { foo: 'bar' },\n { debounce: { id: 'a1', ttl: 2000 } },\n );\n await delay(100);\n\n expect(debouncedCounter).to.be.equal(2);\n });\n });\n\n describe('when ttl is not provided', ()=> {\n it('waits until job is finished before removing debounce key', async ()=> {\n queue.process(\n async () => {\n await delay(100);\n await queue.add(\n { foo: 'bar' },\n { debounce: { id: 'a1' } },\n );\n await delay(100);\n await queue.add(\n { foo: 'bar' },\n { debounce: { id: 'a1' } },\n );\n await delay(100);\n }\n );\n \n let debouncedCounter = 0;\n \n const completing = new Promise(resolve => {\n queue.once('completed', ({ id }) => {\n expect(id).to.be.equal('1');\n resolve();\n });\n \n queue.on('debounced', () => {\n debouncedCounter++;\n });\n });\n \n await queue.add({ foo: 'bar' }, { debounce: { id: 'a1' } });\n \n await completing;\n \n const secondJob = await queue.add(\n { foo: 'bar' },\n { debounce: { id: 'a1' } },\n );\n \n const count = await queue.getJobCountByTypes();\n \n expect(count).to.be.eql(2);\n \n expect(debouncedCounter).to.be.equal(2);\n expect(secondJob.id).to.be.equal('4');\n });\n }); \n });\n\n it('process a job that updates progress', done => {\n queue.process((job, jobDone) => {\n expect(job.data.foo).to.be.equal('bar');\n job.progress(42);\n jobDone();\n });\n\n queue\n .add({ foo: 'bar' })\n .then(job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n })\n .catch(done);\n\n queue.on('progress', (job, progress) => {\n expect(job).to.be.ok;\n expect(progress).to.be.eql(42);\n done();\n });\n });\n\n it('process a job that updates progress with an object', done => {\n queue.process((job, jobDone) => {\n expect(job.data.foo).to.be.equal('bar');\n job.progress({ myvalue: 42 });\n jobDone();\n });\n\n queue\n .add({ foo: 'bar' })\n .then(job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n })\n .catch(done);\n\n queue.on('progress', (job, progress) => {\n expect(job).to.be.ok;\n expect(progress).to.be.eql({ myvalue: 42 });\n done();\n });\n });\n\n it('process a job that updates progress with an object emits a global event', done => {\n let jobId;\n queue.process((job, jobDone) => {\n expect(job.data.foo).to.be.equal('bar');\n job.progress({ myvalue: 42 });\n jobDone();\n });\n\n queue\n .add({ foo: 'bar' })\n .then(job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n jobId = job.id;\n })\n .catch(done);\n\n queue.on('global:progress', (_jobId, progress) => {\n expect(jobId).to.be.eql(_jobId);\n expect(progress).to.be.eql({ myvalue: 42 });\n done();\n });\n });\n\n it('process a job that returns data in the process handler', done => {\n queue.process((job, jobDone) => {\n expect(job.data.foo).to.be.equal('bar');\n jobDone(null, 37);\n });\n\n queue\n .add({ foo: 'bar' })\n .then(job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n })\n .catch(done);\n\n queue.on('completed', (job, data) => {\n expect(job).to.be.ok;\n expect(data).to.be.eql(37);\n queue.getJob(job.id).then(job => {\n expect(job.returnvalue).to.be.eql(37);\n done();\n });\n });\n });\n\n it('process a job that returns a string in the process handler', done => {\n const testString = 'a very dignified string';\n queue.on('completed', (job /*, data*/) => {\n expect(job).to.be.ok;\n expect(job.returnvalue).to.be.equal(testString);\n setTimeout(() => {\n queue\n .getJob(job.id)\n .then(job => {\n expect(job).to.be.ok;\n expect(job.returnvalue).to.be.equal(testString);\n done();\n })\n .catch(done);\n }, 100);\n });\n\n queue.process((/*job*/) => {\n return Promise.resolve(testString);\n });\n\n queue.add({ testing: true });\n });\n\n it('process a job that returns data in the process handler and the returnvalue gets stored in the database', done => {\n queue.process((job, jobDone) => {\n expect(job.data.foo).to.be.equal('bar');\n jobDone(null, 37);\n });\n\n queue\n .add({ foo: 'bar' })\n .then(job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n })\n .catch(done);\n\n queue.on('completed', (job, data) => {\n expect(job).to.be.ok;\n expect(data).to.be.eql(37);\n queue.getJob(job.id).then(job => {\n expect(job.returnvalue).to.be.eql(37);\n queue.client.hget(queue.toKey(job.id), 'returnvalue').then(retval => {\n expect(JSON.parse(retval)).to.be.eql(37);\n done();\n });\n });\n });\n });\n\n it('process a job that returns a promise', done => {\n queue.process(job => {\n expect(job.data.foo).to.be.equal('bar');\n return delay(250).then(() => {\n return 'my data';\n });\n });\n\n queue\n .add({ foo: 'bar' })\n .then(job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n })\n .catch(done);\n\n queue.on('completed', (job, data) => {\n expect(job).to.be.ok;\n expect(data).to.be.eql('my data');\n done();\n });\n });\n\n it('process a job that returns data in a promise', done => {\n queue.process(job => {\n expect(job.data.foo).to.be.equal('bar');\n return delay(250, { value: 42 });\n });\n\n queue\n .add({ foo: 'bar' })\n .then(job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n })\n .catch(done);\n\n queue.on('completed', (job, data) => {\n expect(job).to.be.ok;\n expect(data).to.be.eql(42);\n done();\n });\n });\n\n it('process a synchronous job', done => {\n queue.process(job => {\n expect(job.data.foo).to.be.equal('bar');\n });\n\n queue\n .add({ foo: 'bar' })\n .then(job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n })\n .catch(done);\n\n queue.on('completed', job => {\n expect(job).to.be.ok;\n done();\n });\n });\n\n it('process stalled jobs when starting a queue', function(done) {\n this.timeout(6000);\n utils\n .newQueue('test queue stalled', {\n settings: {\n lockDuration: 15,\n lockRenewTime: 5,\n stalledInterval: 100\n }\n })\n .then(queueStalled => {\n const jobs = [\n queueStalled.add({ bar: 'baz' }),\n queueStalled.add({ bar1: 'baz1' }),\n queueStalled.add({ bar2: 'baz2' }),\n queueStalled.add({ bar3: 'baz3' })\n ];\n Promise.all(jobs).then(() => {\n const afterJobsRunning = function() {\n const stalledCallback = sandbox.spy();\n return queueStalled\n .close(true)\n .then(() => {\n return new Promise((resolve, reject) => {\n utils\n .newQueue('test queue stalled', {\n settings: {\n stalledInterval: 100\n }\n })\n .then(queue2 => {\n const doneAfterFour = _.after(4, () => {\n try {\n expect(stalledCallback.calledOnce).to.be.eql(true);\n queue.close().then(resolve);\n } catch (e) {\n queue.close().then(() => {\n reject(e);\n });\n }\n });\n queue2.on('completed', doneAfterFour);\n queue2.on('stalled', stalledCallback);\n\n queue2.process((job, jobDone2) => {\n jobDone2();\n });\n });\n });\n })\n .then(done, done);\n };\n\n const onceRunning = _.once(afterJobsRunning);\n queueStalled.process(() => {\n onceRunning();\n return delay(150);\n });\n });\n });\n });\n\n it('processes jobs that were added before the queue backend started', () => {\n return utils\n .newQueue('test queue added before', {\n settings: {\n lockRenewTime: 10\n }\n })\n .then(queueStalled => {\n const jobs = [\n queueStalled.add({ bar: 'baz' }),\n queueStalled.add({ bar1: 'baz1' }),\n queueStalled.add({ bar2: 'baz2' }),\n queueStalled.add({ bar3: 'baz3' })\n ];\n\n return Promise.all(jobs)\n .then(queueStalled.close.bind(queueStalled))\n .then(() => {\n return utils.newQueue('test queue added before').then(queue2 => {\n queue2.process((job, jobDone) => {\n jobDone();\n });\n\n return new Promise(resolve => {\n const resolveAfterAllJobs = _.after(jobs.length, resolve);\n queue2.on('completed', resolveAfterAllJobs);\n });\n });\n });\n });\n });\n\n it('process a named job that returns a promise', done => {\n queue.process('myname', job => {\n expect(job.data.foo).to.be.equal('bar');\n return delay(250).then(() => {\n return 'my data';\n });\n });\n\n queue\n .add('myname', { foo: 'bar' })\n .then(job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n })\n .catch(done);\n\n queue.on('completed', (job, data) => {\n expect(job).to.be.ok;\n expect(data).to.be.eql('my data');\n done();\n });\n });\n\n it('process a two named jobs that returns a promise', done => {\n queue.process('myname', job => {\n expect(job.data.foo).to.be.equal('bar');\n return delay(250).then(() => {\n return 'my data';\n });\n });\n\n queue.process('myname2', job => {\n expect(job.data.baz).to.be.equal('qux');\n return delay(250).then(() => {\n return 'my data 2';\n });\n });\n\n queue\n .add('myname', { foo: 'bar' })\n .then(job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n })\n .then(() => {\n return queue.add('myname2', { baz: 'qux' });\n })\n .catch(done);\n\n let one, two;\n queue.on('completed', (job, data) => {\n expect(job).to.be.ok;\n if (job.data.foo) {\n one = true;\n expect(data).to.be.eql('my data');\n }\n if (job.data.baz) {\n two = true;\n expect(data).to.be.eql('my data 2');\n }\n if (one && two) {\n done();\n }\n });\n });\n\n it('process all named jobs from one process function', done => {\n queue.process('*', job => {\n expect(job.data).to.be.ok;\n return delay(250).then(() => {\n return 'my data';\n });\n });\n\n queue.add('job_a', { foo: 'bar' }).then(job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n });\n\n queue.add('job_b', { baz: 'qux' }).then(job => {\n expect(job.id).to.be.ok;\n expect(job.data.baz).to.be.eql('qux');\n });\n\n let one, two;\n queue.on('completed', (job, data) => {\n expect(job).to.be.ok;\n if (job.data.foo) {\n one = true;\n expect(data).to.be.eql('my data');\n }\n if (job.data.baz) {\n two = true;\n expect(data).to.be.eql('my data');\n }\n if (one && two) {\n done();\n }\n });\n });\n\n it('fails job if missing named process', done => {\n queue.process((/*job*/) => {\n done(Error('should not process this job'));\n });\n\n queue.once('failed', (/*err*/) => {\n done();\n });\n\n queue.add('myname', { foo: 'bar' }).then(job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n });\n });\n\n it('should wait for all jobs when closing queue with named processors', async () => {\n let processedA = false;\n\n const startProcessing = new Promise(resolve => {\n queue.process('jobA', async () => {\n resolve();\n return new Promise(resolve => {\n setTimeout(() => {\n processedA = true;\n resolve();\n }, 500);\n });\n });\n });\n\n queue.process('jobB', async () => {});\n\n queue.add('jobA', {});\n\n await startProcessing;\n\n expect(processedA).to.be.eq(false);\n\n await queue.close();\n\n expect(processedA).to.be.eq(true);\n });\n\n it('processes several stalled jobs when starting several queues', function(done) {\n this.timeout(50000);\n\n const NUM_QUEUES = 10;\n const NUM_JOBS_PER_QUEUE = 10;\n const stalledQueues = [];\n const jobs = [];\n const redisOpts = { port: 6379, host: '127.0.0.1' };\n\n for (let i = 0; i < NUM_QUEUES; i++) {\n const queueStalled2 = new Queue('test queue stalled 2', {\n redis: redisOpts,\n settings: {\n lockDuration: 30,\n lockRenewTime: 10,\n stalledInterval: 100\n }\n });\n\n for (let j = 0; j < NUM_JOBS_PER_QUEUE; j++) {\n jobs.push(queueStalled2.add({ job: j }));\n }\n\n stalledQueues.push(queueStalled2);\n }\n\n const closeStalledQueues = function() {\n return Promise.all(\n stalledQueues.map(queue => {\n return queue.close(true);\n })\n );\n };\n\n Promise.all(jobs).then(() => {\n let processed = 0;\n const procFn = function() {\n // instead of completing we just close the queue to simulate a crash.\n utils.simulateDisconnect(this);\n processed++;\n if (processed === stalledQueues.length) {\n setTimeout(() => {\n const queue2 = new Queue('test queue stalled 2', {\n redis: redisOpts,\n settings: { stalledInterval: 100 }\n });\n queue2.on('error', err => {\n done(err);\n });\n queue2.process((job2, jobDone) => {\n jobDone();\n });\n\n let counter = 0;\n queue2.on('completed', () => {\n counter++;\n if (counter === NUM_QUEUES * NUM_JOBS_PER_QUEUE) {\n queue2.close().then(done);\n\n closeStalledQueues().then(() => {\n // This can take long time since queues are disconnected.\n });\n }\n });\n }, 100);\n }\n };\n\n const processes = [];\n stalledQueues.forEach(queue => {\n queue.on('error', (/*err*/) => {\n //\n // Swallow errors produced by the disconnect\n //\n });\n processes.push(queue.process(procFn));\n });\n return Promise.all(processes);\n });\n });\n\n it('does not process a job that is being processed when a new queue starts', function(done) {\n this.timeout(12000);\n let err = null;\n let anotherQueue;\n\n queue.on('completed', () => {\n utils.cleanupQueue(anotherQueue).then(done.bind(null, err));\n });\n\n queue.add({ foo: 'bar' }).then(addedJob => {\n queue\n .process((job, jobDone) => {\n expect(job.data.foo).to.be.equal('bar');\n\n if (addedJob.id !== job.id) {\n err = new Error(\n 'Processed job id does not match that of added job'\n );\n }\n setTimeout(jobDone, 500);\n })\n .catch(done);\n\n utils.newQueue().then(_anotherQueue => {\n anotherQueue = _anotherQueue;\n setTimeout(() => {\n anotherQueue.process((job, jobDone) => {\n err = new Error(\n 'The second queue should not have received a job to process'\n );\n jobDone();\n });\n }, 50);\n });\n });\n });\n\n it('process stalled jobs without requiring a queue restart', function(done) {\n this.timeout(12000);\n\n const queue2 = utils.buildQueue('running-stalled-job-' + uuid.v4(), {\n settings: {\n lockRenewTime: 5000,\n lockDuration: 500,\n stalledInterval: 1000\n }\n });\n\n const collect = _.after(2, () => {\n queue2.close().then(done);\n });\n\n queue2.on('completed', () => {\n const client = new redis();\n client\n .multi()\n .zrem(queue2.toKey('completed'), 1)\n .lpush(queue2.toKey('active'), 1)\n .exec();\n client.quit();\n collect();\n });\n\n queue2.process((job, jobDone) => {\n expect(job.data.foo).to.be.equal('bar');\n jobDone();\n });\n\n queue2\n .add({ foo: 'bar' })\n .then(job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n })\n .catch(done);\n });\n\n it('failed stalled jobs that stall more than allowable stalled limit', function(done) {\n const FAILED_MESSAGE = 'job stalled more than allowable limit';\n this.timeout(10000);\n\n const queue2 = utils.buildQueue('running-stalled-job-' + uuid.v4(), {\n settings: {\n lockRenewTime: 2500,\n lockDuration: 250,\n stalledInterval: 500,\n maxStalledCount: 1\n }\n });\n\n let processedCount = 0;\n queue2.process(job => {\n processedCount++;\n expect(job.data.foo).to.be.equal('bar');\n return delay(1500);\n });\n\n queue2.on('completed', () => {\n done(new Error('should not complete'));\n });\n\n queue2.on('failed', (job, err) => {\n expect(processedCount).to.be.eql(2);\n expect(job.failedReason).to.be.eql(FAILED_MESSAGE);\n expect(err.message).to.be.eql(FAILED_MESSAGE);\n done();\n });\n\n queue2\n .add({ foo: 'bar' })\n .then(job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n })\n .catch(done);\n });\n\n it('removes failed stalled jobs that stall more than allowable stalled limit when removeOnFail is present', function(done) {\n const FAILED_MESSAGE = 'job stalled more than allowable limit';\n this.timeout(10000);\n\n const queue2 = utils.buildQueue('running-stalled-job-' + uuid.v4(), {\n settings: {\n lockRenewTime: 2500,\n lockDuration: 250,\n stalledInterval: 500,\n maxStalledCount: 1\n }\n });\n\n let processedCount = 0;\n queue2.process(job => {\n processedCount++;\n expect(job.data.foo).to.be.equal('bar');\n return delay(1500);\n });\n\n queue2.on('completed', () => {\n done(new Error('should not complete'));\n });\n\n queue2.on('failed', (job, err) => {\n expect(processedCount).to.be.eql(2);\n expect(job).to.be.null;\n expect(err.message).to.be.eql(FAILED_MESSAGE);\n done();\n });\n\n queue2\n .add({ foo: 'bar' }, { removeOnFail: true })\n .then(job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n })\n .catch(done);\n });\n\n it('should clear job from stalled set when job completed', done => {\n const queue2 = utils.buildQueue('running-job-' + uuid.v4(), {\n settings: {\n stalledInterval: 10\n }\n });\n\n queue2.process(job => {\n expect(job.data.foo).to.be.equal('bar');\n return delay(100);\n });\n\n queue2.add({ foo: 'bar' }).then(\n job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n },\n err => {\n done(err);\n }\n );\n\n queue2.once('completed', async () => {\n const stalled = await queue2.getStalledCount();\n try {\n expect(stalled).to.be.equal(0);\n done();\n } catch (err) {\n done(err);\n }\n });\n });\n\n it('process a job that fails', done => {\n const jobError = new Error('Job Failed');\n\n queue.process((job, jobDone) => {\n expect(job.data.foo).to.be.equal('bar');\n jobDone(jobError);\n });\n\n queue.add({ foo: 'bar' }).then(\n job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n },\n err => {\n done(err);\n }\n );\n\n queue.once('failed', (job, err) => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n expect(err).to.be.eql(jobError);\n done();\n });\n });\n\n it('process a job that throws an exception', done => {\n const jobError = new Error('Job Failed');\n\n queue.process(job => {\n expect(job.data.foo).to.be.equal('bar');\n throw jobError;\n });\n\n queue.add({ foo: 'bar' }).then(\n job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n },\n err => {\n done(err);\n }\n );\n\n queue.once('failed', (job, err) => {\n expect(job).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n expect(err).to.be.eql(jobError);\n done();\n });\n });\n\n it('process a job that returns data with a circular dependency', done => {\n queue.on('failed', () => {\n done();\n });\n queue.on('completed', () => {\n done(Error('Should not complete'));\n });\n queue.process(() => {\n const circular = {};\n circular.x = circular;\n return Promise.resolve(circular);\n });\n\n queue.add({ foo: 'bar' });\n });\n\n it('process a job that returns a rejected promise', done => {\n const jobError = new Error('Job Failed');\n\n queue.process(job => {\n expect(job.data.foo).to.be.equal('bar');\n return Promise.reject(jobError);\n });\n\n queue.add({ foo: 'bar' }).then(\n job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n },\n err => {\n done(err);\n }\n );\n\n queue.once('failed', (job, err) => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n expect(err).to.be.eql(jobError);\n done();\n });\n });\n\n it('process a job that rejects with a nested error', done => {\n const cause = new Error('cause');\n const jobError = new Error('wrapper');\n jobError.cause = function() {\n return cause;\n };\n\n queue.process(job => {\n expect(job.data.foo).to.be.equal('bar');\n return Promise.reject(jobError);\n });\n\n queue.add({ foo: 'bar' }).then(\n job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n },\n err => {\n done(err);\n }\n );\n\n queue.once('failed', (job, err) => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n expect(err).to.be.eql(jobError);\n expect(err.cause()).to.be.eql(cause);\n done();\n });\n });\n\n // Skipped since the test is unstable and difficult to understand.\n it.skip('does not renew a job lock after the lock has been released [#397]', function() {\n this.timeout(queue.LOCK_RENEW_TIME * 4);\n\n const processing = queue.process(job => {\n expect(job.data.foo).to.be.equal('bar');\n return Promise.resolve();\n });\n\n const emit = queue.emit.bind(queue);\n queue.emit = function() {\n const args = arguments;\n return delay(queue.LOCK_RENEW_TIME * 2).then(() => {\n return emit.apply(null, args);\n });\n };\n\n setTimeout(queue.close.bind(queue), queue.LOCK_RENEW_TIME * 2.5);\n\n return queue\n .add({ foo: 'bar' })\n .then(job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n })\n .then(() => {\n return processing;\n });\n });\n\n it('retry a job that fails', done => {\n let called = 0;\n let failedOnce = false;\n const notEvenErr = new Error('Not even!');\n\n const retryQueue = utils.buildQueue('retry-test-queue');\n\n retryQueue.add({ foo: 'bar' }).then(job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n });\n\n retryQueue.process((job, jobDone) => {\n called++;\n if (called % 2 !== 0) {\n throw notEvenErr;\n }\n jobDone();\n });\n\n retryQueue.once('failed', (job, err) => {\n expect(job).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n expect(err).to.be.eql(notEvenErr);\n failedOnce = true;\n retryQueue.retryJob(job);\n });\n\n retryQueue.once('completed', () => {\n expect(failedOnce).to.be.eql(true);\n retryQueue.close().then(done);\n });\n });\n\n it('retry a job that fails on a paused queue moves the job to paused', async () => {\n let called = 0;\n let failedOnce = false;\n const notEvenErr = new Error('Not even!');\n\n const retryQueue = utils.buildQueue('retry-test-queue');\n\n const job = await retryQueue.add({ foo: 'bar' });\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n\n retryQueue.process((job, jobDone) => {\n called++;\n if (called % 2 !== 0) {\n throw notEvenErr;\n }\n jobDone();\n });\n\n const failed = new Promise(resolve => {\n retryQueue.once('failed', async (job, err) => {\n expect(job).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n expect(err).to.be.eql(notEvenErr);\n failedOnce = true;\n resolve();\n });\n });\n\n await failed;\n\n await retryQueue.pause();\n\n await retryQueue.retryJob(job);\n\n const pausedJobs = await retryQueue.getJobs(['paused']);\n expect(pausedJobs).to.have.length(1);\n\n await retryQueue.resume();\n\n const completed = new Promise(resolve => {\n retryQueue.once('completed', () => {\n expect(failedOnce).to.be.eql(true);\n retryQueue.close().then(resolve);\n });\n });\n\n await completed;\n });\n\n it('retry a job that fails using job retry method', done => {\n let called = 0;\n let failedOnce = false;\n const notEvenErr = new Error('Not even!');\n\n const retryQueue = utils.buildQueue('retry-test-queue');\n\n retryQueue.add({ foo: 'bar' }).then(job => {\n expect(job.id).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n });\n\n retryQueue.process((job, jobDone) => {\n called++;\n if (called % 2 !== 0) {\n throw notEvenErr;\n }\n jobDone();\n });\n\n retryQueue.once('failed', (job, err) => {\n expect(job).to.be.ok;\n expect(job.data.foo).to.be.eql('bar');\n expect(err).to.be.eql(notEvenErr);\n failedOnce = true;\n job.retry().then(() => {\n expect(job.failedReason).to.be.null;\n expect(job.processedOn).to.be.null;\n expect(job.finishedOn).to.be.null;\n\n retryQueue.getJob(job.id).then(updatedJob => {\n expect(updatedJob.failedReason).to.be.undefined;\n expect(updatedJob.processedOn).to.be.undefined;\n expect(updatedJob.finishedOn).to.be.undefined;\n });\n });\n });\n\n retryQueue.once('completed', () => {\n expect(failedOnce).to.be.eql(true);\n retryQueue.close().then(done);\n });\n });\n });\n\n describe('Delayed jobs', () => {\n let queue;\n\n beforeEach(() => {\n const client = new redis();\n return client.flushdb();\n });\n\n it('should process a delayed job only after delayed time', done => {\n const delay = 500;\n queue = new Queue('delayed queue simple');\n const client = new redis(6379, '127.0.0.1', {});\n const timestamp = Date.now();\n let publishHappened = false;\n client.on('ready', () => {\n client.on('message', (channel, message) => {\n expect(parseInt(message, 10)).to.be.a('number');\n publishHappened = true;\n });\n client.subscribe(queue.toKey('delayed'));\n });\n\n queue.process((job, jobDone) => {\n jobDone();\n });\n\n queue.on('completed', () => {\n expect(Date.now() > timestamp + delay);\n queue\n .getWaiting()\n .then(jobs => {\n expect(jobs.length).to.be.equal(0);\n })\n .then(() => {\n return queue.getDelayed().then(jobs => {\n expect(jobs.length).to.be.equal(0);\n });\n })\n .then(() => {\n expect(publishHappened).to.be.eql(true);\n queue.close(true).then(done, done);\n });\n });\n\n queue._initializingProcess.then(() => {\n queue.add({ delayed: 'foobar' }, { delay }).then(job => {\n expect(job.id).to.be.ok;\n expect(job.data.delayed).to.be.eql('foobar');\n expect(job.delay).to.be.eql(delay);\n });\n });\n });\n\n it('should process delayed jobs in correct order', done => {\n let order = 0;\n queue = new Queue('delayed queue multiple');\n\n queue.on('failed', (job, err) => {\n err.job = job;\n done(err);\n });\n\n queue.process((job, jobDone) => {\n order++;\n expect(order).to.be.equal(job.data.order);\n jobDone();\n if (order === 10) {\n queue.close().then(done, done);\n }\n });\n\n queue.add({ order: 1 }, { delay: 100 });\n queue.add({ order: 6 }, { delay: 600 });\n queue.add({ order: 10 }, { delay: 1000 });\n queue.add({ order: 2 }, { delay: 200 });\n queue.add({ order: 9 }, { delay: 900 });\n queue.add({ order: 5 }, { delay: 500 });\n queue.add({ order: 3 }, { delay: 300 });\n queue.add({ order: 7 }, { delay: 700 });\n queue.add({ order: 4 }, { delay: 400 });\n queue.add({ order: 8 }, { delay: 800 });\n });\n\n it('should process delayed jobs in correct order even in case of restart', function(done) {\n this.timeout(15000);\n\n const QUEUE_NAME = 'delayed queue multiple' + uuid.v4();\n let order = 1;\n\n queue = new Queue(QUEUE_NAME);\n\n const fn = function(job, jobDone) {\n expect(order).to.be.equal(job.data.order);\n jobDone();\n\n if (order === 4) {\n queue.close().then(done, done);\n }\n\n order++;\n };\n\n Promise.all([\n queue.add({ order: 2 }, { delay: 300 }),\n queue.add({ order: 4 }, { delay: 500 }),\n queue.add({ order: 1 }, { delay: 200 }),\n queue.add({ order: 3 }, { delay: 400 })\n ])\n .then(() => {\n //\n // Start processing so that jobs get into the delay set.\n //\n queue.process(fn);\n return delay(20);\n })\n .then(() => {\n /*\n //We simulate a restart\n console.log('RESTART');\n return queue.close().then(function () {\n console.log('CLOSED');\n return delay(100).then(function () {\n queue = new Queue(QUEUE_NAME);\n queue.process(fn);\n });\n });\n */\n });\n });\n\n it('should process delayed jobs with exact same timestamps in correct order (FIFO)', done => {\n const QUEUE_NAME = 'delayed queue multiple' + uuid.v4();\n queue = new Queue(QUEUE_NAME);\n let order = 1;\n\n const fn = function(job, jobDone) {\n expect(order).to.be.equal(job.data.order);\n jobDone();\n\n if (order === 12) {\n queue.close().then(done, done);\n }\n\n order++;\n };\n\n queue.isReady().then(() => {\n const now = Date.now();\n const _promises = [];\n let _i = 1;\n for (_i; _i <= 12; _i++) {\n _promises.push(\n queue.add(\n { order: _i },\n {\n delay: 1000,\n timestamp: now\n }\n )\n );\n }\n Promise.all(_promises).then(() => {\n queue.process(fn);\n });\n });\n });\n\n it('an unlocked job should not be moved to delayed', done => {\n const queue = new Queue('delayed queue');\n let job;\n\n queue.process((_job, callback) => {\n // Release the lock to simulate the event loop stalling (so failure to renew the lock).\n job = _job;\n job.releaseLock().then(() => {\n // Once it's failed, it should NOT be moved to delayed since this worker lost the lock.\n callback(new Error('retry this job'));\n });\n });\n\n queue.on('error', (/*err*/) => {\n job.isDelayed().then(isDelayed => {\n expect(isDelayed).to.be.equal(false);\n queue.close().then(done, done);\n });\n });\n\n queue.add({ foo: 'bar' }, { backoff: 1000, attempts: 2 });\n });\n\n it('an unlocked job should not be moved to waiting', done => {\n const queue = new Queue('delayed queue');\n let job;\n\n queue.process((_job, callback) => {\n job = _job;\n // Release the lock to simulate the event loop stalling (so failure to renew the lock).\n job.releaseLock().then(() => {\n // Once it's failed, it should NOT be moved to waiting since this worker lost the lock.\n callback(new Error('retry this job'));\n });\n });\n\n queue.on('error', (/*err*/) => {\n job.isWaiting().then(isWaiting => {\n expect(isWaiting).to.be.equal(false);\n queue.close().then(done, done);\n });\n });\n\n // Note that backoff:0 should immediately retry the job upon failure (ie put it in 'waiting')\n queue.add({ foo: 'bar' }, { backoff: 0, attempts: 2 });\n });\n });\n\n describe('Concurrency process', () => {\n let queue;\n\n beforeEach(() => {\n const client = new redis();\n queue = utils.buildQueue();\n return client.flushdb();\n });\n\n afterEach(function() {\n this.timeout(\n queue.settings.stalledInterval * (1 + queue.settings.maxStalledCount)\n );\n return queue.close();\n });\n\n it('should run job in sequence if I specify a concurrency of 1', done => {\n let processing = false;\n\n queue.process(1, (job, jobDone) => {\n expect(processing).to.be.equal(false);\n processing = true;\n delay(50).then(() => {\n processing = false;\n jobDone();\n });\n });\n\n queue.add({});\n queue.add({});\n\n queue.on(\n 'completed',\n _.after(2, () => done())\n );\n });\n\n //This job use delay to check that at any time we have 4 process in parallel.\n //Due to time to get new jobs and call process, false negative can appear.\n it('should process job respecting the concurrency set', done => {\n let nbProcessing = 0;\n let pendingMessageToProcess = 8;\n let wait = 100;\n\n queue\n .process(4, () => {\n nbProcessing++;\n expect(nbProcessing).to.be.lessThan(5);\n\n wait += 100;\n\n return delay(wait).then(() => {\n //We should not have 4 more in parallel.\n //At the end, due to empty list, no new job will process, so nbProcessing will decrease.\n expect(nbProcessing).to.be.eql(\n Math.min(pendingMessageToProcess, 4)\n );\n pendingMessageToProcess--;\n nbProcessing--;\n });\n })\n .catch(done);\n\n queue.add();\n queue.add();\n queue.add();\n queue.add();\n queue.add();\n queue.add();\n queue.add();\n queue.add();\n\n queue.on('completed', _.after(8, done.bind(null, null)));\n queue.on('failed', done);\n });\n\n it('should wait for all concurrent processing in case of pause', done => {\n let i = 0;\n let nbJobFinish = 0;\n\n queue\n .process(3, (job, jobDone) => {\n let error = null;\n\n if (++i === 4) {\n queue.pause().then(() => {\n delay(500).then(() => {\n // Wait for all the active jobs to finalize.\n expect(nbJobFinish).to.be.above(3);\n queue.resume();\n });\n });\n }\n\n // We simulate an error of one processing job.\n // They had a bug in pause() with this special case.\n if (i % 3 === 0) {\n error = new Error();\n }\n\n //100 - i*20 is to force to finish job n°4 before lower job that will wait longer\n delay(100 - i * 10).then(() => {\n nbJobFinish++;\n jobDone(error);\n });\n })\n .catch(done);\n\n queue.add({});\n queue.add({});\n queue.add({});\n queue.add({});\n queue.add({});\n queue.add({});\n queue.add({});\n queue.add({});\n\n const cb = _.after(8, done.bind(null, null));\n queue.on('completed', cb);\n queue.on('failed', cb);\n queue.on('error', done);\n });\n });\n\n describe('Retries and backoffs', () => {\n let queue;\n\n afterEach(function() {\n this.timeout(\n queue.settings.stalledInterval * (1 + queue.settings.maxStalledCount)\n );\n return queue.close();\n });\n\n it('should not retry a job if it has been marked as unrecoverable', done => {\n let tries = 0;\n queue = utils.buildQueue('test retries and backoffs');\n queue.isReady().then(() => {\n queue.process((job, jobDone) => {\n tries++;\n expect(tries).to.equal(1);\n job.discard();\n jobDone(new Error('unrecoverable error'));\n });\n\n queue.add(\n { foo: 'bar' },\n {\n attempts: 5\n }\n );\n });\n queue.on('failed', () => {\n done();\n });\n });\n\n it('should automatically retry a failed job if attempts is bigger than 1', done => {\n queue = utils.buildQueue('test retries and backoffs');\n queue.isReady().then(() => {\n let tries = 0;\n queue.process((job, jobDone) => {\n expect(job.attemptsMade).to.be.eql(tries);\n tries++;\n if (job.attemptsMade < 2) {\n throw new Error('Not yet!');\n }\n\n jobDone();\n });\n\n queue.add(\n { foo: 'bar' },\n {\n attempts: 3\n }\n );\n });\n queue.on('completed', () => {\n done();\n });\n });\n\n describe('when job has more priority than delayed jobs', () => {\n it('executes retried job first', done => {\n queue = utils.buildQueue('test retries and priority');\n let id = 0;\n queue.isReady().then(() => {\n queue.process(async job => {\n await delay(200);\n if (job.attemptsMade === 0) {\n id++;\n expect(job.id).to.be.eql(`${id}`);\n }\n if (job.id == '1' && job.attemptsMade < 1) {\n throw new Error('Not yet!');\n }\n });\n\n queue.add(\n { foo: 'bar' },\n {\n attempts: 2,\n priority: 1\n }\n );\n queue.add(\n {},\n {\n delay: 200,\n priority: 2\n }\n );\n queue.add(\n {},\n {\n delay: 200,\n priority: 2\n }\n );\n queue.add(\n {},\n {\n delay: 200,\n priority: 2\n }\n );\n });\n let count = 0;\n queue.on('completed', () => {\n if (count++ === 3) {\n done();\n }\n });\n });\n });\n\n it('should not retry a failed job more than the number of given attempts times', done => {\n queue = utils.buildQueue('test retries and backoffs');\n let tries = 0;\n queue.isReady().then(() => {\n queue.process((job, jobDone) => {\n tries++;\n if (job.attemptsMade < 3) {\n throw new Error('Not yet!');\n }\n expect(job.attemptsMade).to.be.eql(tries - 1);\n jobDone();\n });\n\n queue.add(\n { foo: 'bar' },\n {\n attempts: 3\n }\n );\n });\n queue.on('completed', () => {\n done(new Error('Failed job was retried more than it should be!'));\n });\n queue.on('failed', () => {\n if (tries === 3) {\n done();\n }\n });\n });\n\n it('should retry a job after a delay if a fixed backoff is given', function(done) {\n this.timeout(12000);\n queue = utils.buildQueue('test retries and backoffs');\n let start;\n queue.isReady().then(() => {\n queue.process((job, jobDone) => {\n if (job.attemptsMade < 2) {\n throw new Error('Not yet!');\n }\n jobDone();\n });\n\n start = Date.now();\n queue.add(\n { foo: 'bar' },\n {\n attempts: 3,\n backoff: 1000\n }\n );\n });\n queue.on('completed', () => {\n const elapse = Date.now() - start;\n expect(elapse).to.be.greaterThan(2000);\n done();\n });\n });\n\n it('should retry a job after a delay if an exponential backoff is given', function(done) {\n this.timeout(12000);\n queue = utils.buildQueue('test retries and backoffs');\n let start;\n queue.isReady().then(() => {\n queue.process((job, jobDone) => {\n if (job.attemptsMade < 2) {\n throw new Error('Not yet!');\n }\n jobDone();\n });\n\n start = Date.now();\n queue.add(\n { foo: 'bar' },\n {\n attempts: 3,\n backoff: {\n type: 'exponential',\n delay: 1000\n }\n }\n );\n });\n queue.on('completed', () => {\n const elapse = Date.now() - start;\n const expected = 1000 * (Math.pow(2, 2) - 1);\n expect(elapse).to.be.greaterThan(expected);\n done();\n });\n });\n\n it('should retry a job after a delay if a custom backoff is given', function(done) {\n this.timeout(12000);\n queue = utils.buildQueue('test retries and backoffs', {\n settings: {\n backoffStrategies: {\n custom(attemptsMade) {\n return attemptsMade * 1000;\n }\n }\n }\n });\n let start;\n queue.isReady().then(() => {\n queue.process((job, jobDone) => {\n if (job.attemptsMade < 2) {\n throw new Error('Not yet!');\n }\n jobDone();\n });\n\n start = Date.now();\n queue.add(\n { foo: 'bar' },\n {\n attempts: 3,\n backoff: {\n type: 'custom'\n }\n }\n );\n });\n queue.on('completed', () => {\n const elapse = Date.now() - start;\n expect(elapse).to.be.greaterThan(3000);\n done();\n });\n });\n\n it('should pass strategy options to custom backoff', function(done) {\n this.timeout(12000);\n queue = utils.buildQueue('test retries and backoffs', {\n settings: {\n backoffStrategies: {\n custom(attemptsMade, err, strategyOptions) {\n expect(strategyOptions.id).to.be.equal('FOO42');\n return attemptsMade * 1000;\n }\n }\n }\n });\n let start;\n queue.isReady().then(() => {\n queue.process((job, jobDone) => {\n if (job.attemptsMade < 2) {\n throw new Error('Not yet!');\n }\n jobDone();\n });\n\n start = Date.now();\n queue.add(\n { foo: 'bar' },\n {\n attempts: 3,\n backoff: {\n type: 'custom',\n options: {\n id: 'FOO42'\n }\n }\n }\n );\n });\n queue.on('completed', () => {\n const elapse = Date.now() - start;\n expect(elapse).to.be.greaterThan(3000);\n done();\n });\n });\n\n it('should not retry a job if the custom backoff returns -1', done => {\n queue = utils.buildQueue('test retries and backoffs', {\n settings: {\n backoffStrategies: {\n custom() {\n return -1;\n }\n }\n }\n });\n let tries = 0;\n queue.process((job, jobDone) => {\n tries++;\n if (job.attemptsMade < 3) {\n throw new Error('Not yet!');\n }\n jobDone();\n });\n\n queue.add(\n { foo: 'bar' },\n {\n attempts: 3,\n backoff: {\n type: 'custom'\n }\n }\n );\n queue.on('completed', () => {\n done(new Error('Failed job was retried more than it should be!'));\n });\n queue.on('failed', () => {\n if (tries === 1) {\n done();\n }\n });\n });\n\n it('should retry a job after a delay if a custom backoff is given based on the error thrown', function(done) {\n function CustomError() {}\n\n this.timeout(12000);\n queue = utils.buildQueue('test retries and backoffs', {\n settings: {\n backoffStrategies: {\n custom(attemptsMade, err) {\n if (err instanceof CustomError) {\n return 1500;\n }\n return 500;\n }\n }\n }\n });\n let start;\n queue.isReady().then(() => {\n queue.process((job, jobDone) => {\n if (job.attemptsMade < 2) {\n throw new CustomError('Hey, custom error!');\n }\n jobDone();\n });\n\n start = Date.now();\n queue.add(\n { foo: 'bar' },\n {\n attempts: 3,\n backoff: {\n type: 'custom'\n }\n }\n );\n });\n queue.on('completed', () => {\n const elapse = Date.now() - start;\n expect(elapse).to.be.greaterThan(3000);\n done();\n });\n });\n\n it('should be able to handle a custom backoff if it returns a promise', function(done) {\n this.timeout(12000);\n\n queue = utils.buildQueue('test retries and backoffs', {\n settings: {\n backoffStrategies: {\n async custom() {\n return new Promise(resolve => {\n setTimeout(() => {\n resolve(500);\n }, 500);\n });\n }\n }\n }\n });\n let start;\n queue.isReady().then(() => {\n queue.process((job, jobDone) => {\n if (job.attemptsMade < 2) {\n throw new Error('some error');\n }\n jobDone();\n });\n\n start = Date.now();\n queue.add(\n { foo: 'bar' },\n {\n attempts: 3,\n backoff: {\n type: 'custom'\n }\n }\n );\n });\n queue.on('completed', () => {\n const elapse = Date.now() - start;\n expect(elapse).to.be.greaterThan(2000);\n done();\n });\n });\n\n it('should not retry a job that has been removed', done => {\n queue = utils.buildQueue('retry a removed job');\n let attempts = 0;\n const failedError = new Error('failed');\n queue.process((job, jobDone) => {\n if (attempts === 0) {\n attempts++;\n throw failedError;\n } else {\n jobDone();\n }\n });\n queue.add({ foo: 'bar' });\n\n const failedHandler = _.once((job, err) => {\n expect(job.data.foo).to.equal('bar');\n expect(err).to.equal(failedError);\n expect(job.failedReason).to.equal(failedError.message);\n\n try {\n job\n .retry()\n .then(() => {\n return delay(100).then(() => {\n return queue.getCompletedCount().then(count => {\n return expect(count).to.equal(1);\n });\n });\n })\n .then(() => {\n return queue.clean(0).then(() => {\n return job.retry().catch(err => {\n expect(err.message).to.equal(\n Queue.ErrorMessages.RETRY_JOB_NOT_EXIST\n );\n });\n });\n })\n .then(() => {\n return Promise.all([\n queue.getCompletedCount().then(count => {\n return expect(count).to.equal(0);\n }),\n queue.getFailedCount().then(count => {\n return expect(count).to.equal(0);\n })\n ]);\n })\n .then(() => {\n done();\n }, done);\n } catch (err) {\n console.error(err);\n }\n });\n\n queue.on('failed', failedHandler);\n });\n\n it('should not retry a job that has been retried already', done => {\n queue = utils.buildQueue('retry already retried job');\n const failedError = new Error('failed');\n queue.isReady().then(() => {\n let attempts = 0;\n queue.process((job, jobDone) => {\n if (attempts === 0) {\n attempts++;\n throw failedError;\n } else {\n jobDone();\n }\n });\n\n queue.add({ foo: 'bar' });\n });\n\n const failedHandler = _.once((job, err) => {\n expect(job.data.foo).to.equal('bar');\n expect(err).to.equal(failedError);\n\n job\n .retry()\n .then(() => {\n return delay(100).then(() => {\n return queue.getCompletedCount().then(count => {\n return expect(count).to.equal(1);\n });\n });\n })\n .then(() => {\n return job.retry().catch(err => {\n expect(err.message).to.equal(\n Queue.ErrorMessages.RETRY_JOB_NOT_FAILED\n );\n });\n })\n .then(() => {\n return Promise.all([\n queue.getCompletedCount().then(count => {\n return expect(count).to.equal(1);\n }),\n queue.getFailedCount().then(count => {\n return expect(count).to.equal(0);\n })\n ]);\n })\n .then(() => {\n done();\n }, done);\n });\n\n queue.on('failed', failedHandler);\n });\n\n it('should not retry a job that is locked', done => {\n queue = utils.buildQueue('retry a locked job');\n const addedHandler = _.once(job => {\n expect(job.data.foo).to.equal('bar');\n\n delay(100).then(() => {\n job\n .retry()\n .catch(err => {\n expect(err.message).to.equal(\n Queue.ErrorMessages.RETRY_JOB_IS_LOCKED\n );\n return null;\n })\n .then(done, done);\n });\n });\n\n queue.process((/*job*/) => {\n return delay(300);\n });\n queue.add({ foo: 'bar' }).then(addedHandler);\n });\n\n it('an unlocked job should not be moved to failed', done => {\n queue = utils.buildQueue('test unlocked failed');\n\n queue.process((job, callback) => {\n // Release the lock to simulate the event loop stalling (so failure to renew the lock).\n job.releaseLock().then(() => {\n // Once it's failed, it should NOT be moved to failed since this worker lost the lock.\n callback(new Error('retry this job'));\n });\n });\n\n queue.on('failed', job => {\n job.isFailed().then(isFailed => {\n expect(isFailed).to.be.equal(false);\n });\n });\n\n queue.on('error', (/*err*/) => {\n queue.close().then(done, done);\n });\n\n // Note that backoff:0 should immediately retry the job upon failure (ie put it in 'waiting')\n queue.add({ foo: 'bar' }, { backoff: 0, attempts: 2 });\n });\n });\n\n describe('Drain queue', () => {\n it('should count zero after draining the queue', () => {\n const maxJobs = 100;\n const added = [];\n\n const queue = utils.buildQueue();\n\n for (let i = 1; i <= maxJobs; i++) {\n added.push(queue.add({ foo: 'bar', num: i }));\n }\n\n return Promise.all(added)\n .then(queue.count.bind(queue))\n .then(count => {\n expect(count).to.be.eql(maxJobs);\n })\n .then(queue.empty.bind(queue))\n .then(queue.count.bind(queue))\n .then(count => {\n expect(count).to.be.eql(0);\n return queue.close();\n });\n });\n });\n\n describe('Cleaner', () => {\n let queue;\n\n beforeEach(() => {\n queue = utils.buildQueue('cleaner' + uuid.v4());\n });\n\n afterEach(function() {\n this.timeout(\n queue.settings.stalledInterval * (1 + queue.settings.maxStalledCount)\n );\n return queue.close();\n });\n\n it('should reject the cleaner with no grace', done => {\n queue.clean().then(\n () => {\n done(new Error('Promise should not resolve'));\n },\n err => {\n expect(err).to.be.instanceof(Error);\n done();\n }\n );\n });\n\n it('should reject the cleaner an unknown type', done => {\n queue.clean(0, 'bad').then(\n () => {\n done(new Error('Promise should not resolve'));\n },\n e => {\n expect(e).to.be.instanceof(Error);\n done();\n }\n );\n });\n\n it('should clean an empty queue', done => {\n const testQueue = utils.buildQueue('cleaner' + uuid.v4());\n testQueue.isReady().then(() => {\n return testQueue.clean(0);\n });\n testQueue.on('error', err => {\n utils.cleanupQueue(testQueue);\n done(err);\n });\n testQueue.on('cleaned', (jobs, type) => {\n expect(type).to.be.eql('completed');\n expect(jobs.length).to.be.eql(0);\n utils.cleanupQueue(testQueue);\n done();\n });\n });\n\n it('should clean two jobs from the queue', done => {\n queue.add({ some: 'data' });\n queue.add({ some: 'data' });\n queue.process((job, jobDone) => {\n jobDone();\n });\n\n queue.on(\n 'completed',\n _.after(2, () => {\n delay(200).then(() => {\n queue.clean(0).then(jobs => {\n expect(jobs.length).to.be.eql(2);\n done();\n }, done);\n });\n })\n );\n });\n\n it('should only remove a job outside of the grace period', done => {\n queue.process((job, jobDone) => {\n jobDone();\n });\n queue.add({ some: 'data' });\n queue.add({ some: 'data' });\n delay(200)\n .then(() => {\n queue.add({ some: 'data' });\n queue.clean(100);\n return null;\n })\n .then(() => {\n return delay(100);\n })\n .then(() => {\n return queue.getCompleted();\n })\n .then(jobs => {\n expect(jobs.length).to.be.eql(1);\n return queue.empty();\n })\n .then(() => {\n done();\n });\n });\n\n it('should leave a job that was queued before but processed within the grace period', done => {\n queue.process((job, jobDone) => {\n jobDone();\n });\n queue.add({ some: 'data' });\n queue.add({ some: 'data' }, { delay: 100 });\n delay(200)\n .then(() => {\n // At this point both jobs have executed, but the delayed one was only processed about 100 milliseconds ago:\n return queue.clean(150);\n })\n .then(() => {\n return queue.getCompleted();\n })\n .then(jobs => {\n expect(jobs.length).to.be.eql(1);\n return queue.empty();\n })\n .then(() => {\n done();\n });\n });\n\n it('should clean all failed jobs', done => {\n queue.add({ some: 'data' });\n queue.add({ some: 'data' });\n queue.process((job, jobDone) => {\n jobDone(new Error('It failed'));\n });\n delay(100)\n .then(() => {\n return queue.clean(0, 'failed');\n })\n .then(jobs => {\n expect(jobs.length).to.be.eql(2);\n return queue.count();\n })\n .then(len => {\n expect(len).to.be.eql(0);\n done();\n });\n });\n\n it('should clean all waiting jobs', done => {\n queue.add({ some: 'data' });\n queue.add({ some: 'data' });\n delay(100)\n .then(() => {\n return queue.clean(0, 'wait');\n })\n .then(jobs => {\n expect(jobs.length).to.be.eql(2);\n return queue.count();\n })\n .then(len => {\n expect(len).to.be.eql(0);\n done();\n });\n });\n\n it('should clean all delayed jobs', done => {\n queue.add({ some: 'data' }, { delay: 5000 });\n queue.add({ some: 'data' }, { delay: 5000 });\n delay(100)\n .then(() => {\n return queue.clean(0, 'delayed');\n })\n .then(jobs => {\n expect(jobs.length).to.be.eql(2);\n return queue.count();\n })\n .then(len => {\n expect(len).to.be.eql(0);\n done();\n });\n });\n\n it('should clean the number of jobs requested', done => {\n queue.add({ some: 'data' });\n queue.add({ some: 'data' });\n queue.add({ some: 'data' });\n delay(100)\n .then(() => {\n return queue.clean(0, 'wait', 1);\n })\n .then(jobs => {\n expect(jobs.length).to.be.eql(1);\n return queue.count();\n })\n .then(len => {\n expect(len).to.be.eql(2);\n done();\n });\n });\n\n it('should succeed when the limit is higher than the actual number of jobs', async () => {\n await queue.add({ some: 'data' });\n await queue.add({ some: 'data' });\n await delay(100);\n const deletedJobs = await queue.clean(0, 'wait', 100);\n expect(deletedJobs).to.have.length(2);\n const remainingJobsCount = await queue.count();\n expect(remainingJobsCount).to.be.eql(0);\n });\n\n it(\"should clean the number of jobs requested even if first jobs timestamp doesn't match\", async () => {\n // This job shouldn't get deleted due to the 5000 grace\n await queue.add({ some: 'data' });\n // This job should get cleaned since 10000 > 5000 grace\n const jobToClean = await queue.add(\n { some: 'data' },\n { timestamp: Date.now() - 10000 }\n );\n // This job shouldn't get deleted due to the 5000 grace\n await queue.add({ some: 'data' });\n\n const cleaned = await queue.clean(5000, 'wait', 1);\n expect(cleaned.length).to.be.eql(1);\n expect(cleaned[0]).to.eql(jobToClean.id);\n\n const len = await queue.count();\n expect(len).to.be.eql(2);\n });\n\n it(\"shouldn't clean anything if all jobs are in grace period\", async () => {\n await queue.add({ some: 'data' });\n await queue.add({ some: 'data' });\n\n const cleaned = await queue.clean(5000, 'wait', 1);\n expect(cleaned.length).to.be.eql(0);\n\n const cleaned2 = await queue.clean(5000, 'wait');\n expect(cleaned2.length).to.be.eql(0);\n expect(await queue.count()).to.be.eql(2);\n });\n\n it('should properly clean jobs from the priority set', done => {\n const client = new redis(6379, '127.0.0.1', {});\n queue.add({ some: 'data' }, { priority: 5 });\n queue.add({ some: 'data' }, { priority: 5 });\n delay(100)\n .then(() => {\n return queue.clean(0, 'wait', 1);\n })\n .then(() => {\n return new Promise((resolve, reject) => {\n client.zcount(queue.toKey('priority'), '5', '5', (err, res) => {\n if (err) reject(err);\n else resolve(res);\n });\n });\n })\n .then(priority => {\n expect(priority).to.be.eql(1);\n done();\n });\n });\n\n it('should clean a job without a timestamp', done => {\n const client = new redis(6379, '127.0.0.1', {});\n\n queue.add({ some: 'data' });\n queue.add({ some: 'data' });\n queue.process((job, jobDone) => {\n jobDone(new Error('It failed'));\n });\n\n delay(100)\n .then(() => {\n return new Promise(resolve => {\n client.hdel('bull:' + queue.name + ':1', 'timestamp', resolve);\n });\n })\n .then(() => {\n return queue.clean(0, 'failed');\n })\n .then(jobs => {\n expect(jobs.length).to.be.eql(2);\n return queue.getFailed();\n })\n .then(failed => {\n expect(failed.length).to.be.eql(0);\n done();\n });\n });\n\n it('should clean completed jobs outside grace period', async () => {\n queue.process((job, jobDone) => {\n jobDone();\n });\n const [jobToClean] = await Promise.all([\n queue.add({ some: 'oldJob' }),\n queue.add({ some: 'gracePeriodJob' }, { delay: 50 })\n ]);\n await delay(100);\n\n const cleaned = await queue.clean(75, 'completed');\n\n expect(cleaned.length).to.be.eql(1);\n expect(cleaned[0]).to.eql(jobToClean.id);\n });\n\n it('should clean completed jobs outside grace period with limit', async () => {\n queue.process((job, jobDone) => {\n jobDone();\n });\n const [jobToClean] = await Promise.all([\n queue.add({ some: 'oldJob' }),\n queue.add({ some: 'gracePeriodJob' }, { delay: 50 })\n ]);\n await delay(100);\n\n const cleaned = await queue.clean(75, 'completed', 10);\n\n expect(cleaned.length).to.be.eql(1);\n expect(cleaned[0]).to.eql(jobToClean.id);\n });\n\n it('should clean completed jobs respecting limit', async () => {\n queue.process((job, jobDone) => {\n jobDone();\n });\n const jobsToCleanPromises = [];\n for (let i = 0; i < 3; i++) {\n jobsToCleanPromises.push(queue.add({ some: 'jobToClean' }));\n }\n\n const [jobsToClean] = await Promise.all([\n Promise.all(jobsToCleanPromises),\n queue.add({ some: 'gracePeriodJob' }, { delay: 50 })\n ]);\n await delay(100);\n\n const cleaned = await queue.clean(75, 'completed', 1);\n\n expect(cleaned.length).to.be.eql(1);\n const jobsToCleanIds = jobsToClean.map(job => job.id);\n expect(jobsToCleanIds).to.include(cleaned[0]);\n });\n\n it('should clean failed jobs without leaving any job keys', async () => {\n const numJobs = 10;\n queue.process((job, jobDone) => {\n jobDone(new Error('It failed'));\n });\n const jobsToCleanPromises = [];\n for (let i = 0; i < numJobs; i++) {\n jobsToCleanPromises.push(queue.add({ some: 'jobToClean' }));\n }\n\n await Promise.all([\n Promise.all(jobsToCleanPromises),\n queue.add({ some: 'gracePeriodJob' }, { delay: 50 })\n ]);\n await delay(100);\n\n await queue.clean(0, 'failed');\n\n const client = new redis();\n const keys = await client.keys(`bull:${queue.name}:*`);\n expect(keys.length).to.be.eql(2);\n });\n });\n});\n" }, { "path": "test/test_rate_limiter.js", "content": "'use strict';\n\nconst expect = require('chai').expect;\nconst utils = require('./utils');\nconst redis = require('ioredis');\nconst _ = require('lodash');\nconst assert = require('assert');\n\ndescribe('Rate limiter', () => {\n let queue;\n let client;\n\n beforeEach(() => {\n client = new redis();\n return client.flushdb().then(() => {\n queue = utils.buildQueue('test rate limiter', {\n limiter: {\n max: 1,\n duration: 1000\n }\n });\n return queue;\n });\n });\n\n afterEach(() => {\n return queue.close().then(() => {\n return client.quit();\n });\n });\n\n it('should throw exception if missing duration option', done => {\n try {\n utils.buildQueue('rate limiter fail', {\n limiter: {\n max: 5\n }\n });\n expect.fail('Should not allow missing `duration` option');\n } catch (err) {\n done();\n }\n });\n\n it('should throw exception if missing max option', done => {\n try {\n utils.buildQueue('rate limiter fail', {\n limiter: {\n duration: 5000\n }\n });\n expect.fail('Should not allow missing `max`option');\n } catch (err) {\n done();\n }\n });\n\n it('should obey the rate limit', done => {\n const startTime = new Date().getTime();\n const numJobs = 4;\n\n queue.process(() => {\n return Promise.resolve();\n });\n\n for (let i = 0; i < numJobs; i++) {\n queue.add({});\n }\n\n queue.on(\n 'completed',\n // after every job has been completed\n _.after(numJobs, () => {\n try {\n const timeDiff = new Date().getTime() - startTime;\n expect(timeDiff).to.be.above((numJobs - 1) * 1000);\n done();\n } catch (err) {\n done(err);\n }\n })\n );\n\n queue.on('failed', err => {\n done(err);\n });\n }).timeout(5000);\n\n // Skip because currently job priority is maintained in a best effort way, but cannot\n // be guaranteed for rate limited jobs.\n it.skip('should obey job priority', async () => {\n const newQueue = utils.buildQueue('test rate limiter', {\n limiter: {\n max: 1,\n duration: 150\n }\n });\n const numJobs = 20;\n const priorityBuckets = {\n 1: 0,\n 2: 0,\n 3: 0,\n 4: 0\n };\n\n const numPriorities = Object.keys(priorityBuckets).length;\n\n newQueue.process(job => {\n const priority = job.opts.priority;\n\n priorityBuckets[priority] = priorityBuckets[priority] - 1;\n\n for (let p = 1; p < priority; p++) {\n if (priorityBuckets[p] > 0) {\n const before = JSON.stringify(priorityBucketsBefore);\n const after = JSON.stringify(priorityBuckets);\n throw new Error(\n `Priority was not enforced, job with priority ${priority} was processed before all jobs with priority ${p} were processed. Bucket counts before: ${before} / after: ${after}`\n );\n }\n }\n });\n\n const result = new Promise((resolve, reject) => {\n newQueue.on('failed', (job, err) => {\n reject(err);\n });\n\n const afterNumJobs = _.after(numJobs, () => {\n try {\n expect(_.every(priorityBuckets, value => value === 0)).to.eq(true);\n resolve();\n } catch (err) {\n reject(err);\n }\n });\n\n newQueue.on('completed', () => {\n afterNumJobs();\n });\n });\n\n await newQueue.pause();\n const promises = [];\n\n for (let i = 0; i < numJobs; i++) {\n const opts = { priority: (i % numPriorities) + 1 };\n priorityBuckets[opts.priority] = priorityBuckets[opts.priority] + 1;\n promises.push(newQueue.add({ id: i }, opts));\n }\n\n const priorityBucketsBefore = _.reduce(\n priorityBuckets,\n (acc, value, key) => {\n acc[key] = value;\n return acc;\n },\n {}\n );\n\n await Promise.all(promises);\n\n await newQueue.resume();\n\n return result;\n }).timeout(60000);\n\n it('should put a job into the delayed queue when limit is hit', () => {\n const newQueue = utils.buildQueue('test rate limiter', {\n limiter: {\n max: 1,\n duration: 1000\n }\n });\n\n queue.on('failed', e => {\n assert.fail(e);\n });\n\n return Promise.all([\n newQueue.add({}),\n newQueue.add({}),\n newQueue.add({}),\n newQueue.add({})\n ]).then(() => {\n Promise.all([\n newQueue.getNextJob({}),\n newQueue.getNextJob({}),\n newQueue.getNextJob({}),\n newQueue.getNextJob({})\n ]).then(() => {\n return queue.getDelayedCount().then(\n delayedCount => {\n expect(delayedCount).to.eq(3);\n },\n () => {\n /*ignore error*/\n }\n );\n });\n });\n });\n\n it('should not put a job into the delayed queue when discard is true', () => {\n const newQueue = utils.buildQueue('test rate limiter', {\n limiter: {\n max: 1,\n duration: 1000,\n bounceBack: true\n }\n });\n\n newQueue.on('failed', e => {\n assert.fail(e);\n });\n return Promise.all([\n newQueue.add({}),\n newQueue.add({}),\n newQueue.add({}),\n newQueue.add({})\n ]).then(() => {\n Promise.all([\n newQueue.getNextJob({}),\n newQueue.getNextJob({}),\n newQueue.getNextJob({}),\n newQueue.getNextJob({})\n ]).then(() => {\n return newQueue.getDelayedCount().then(delayedCount => {\n expect(delayedCount).to.eq(0);\n return newQueue.getActiveCount().then(waitingCount => {\n expect(waitingCount).to.eq(1);\n });\n });\n });\n });\n });\n\n it('should rate limit by grouping', async function() {\n this.timeout(20000);\n const numGroups = 4;\n const numJobs = 20;\n const startTime = Date.now();\n\n const rateLimitedQueue = utils.buildQueue('test rate limiter with group', {\n limiter: {\n max: 1,\n duration: 1000,\n groupKey: 'accountId'\n }\n });\n\n rateLimitedQueue.process(() => {\n return Promise.resolve();\n });\n\n const completed = {};\n\n const running = new Promise((resolve, reject) => {\n const afterJobs = _.after(numJobs, () => {\n try {\n const timeDiff = Date.now() - startTime;\n expect(timeDiff).to.be.gte(numGroups * 1000);\n expect(timeDiff).to.be.below((numGroups + 1) * 1500);\n\n for (const group in completed) {\n let prevTime = completed[group][0];\n for (let i = 1; i < completed[group].length; i++) {\n const diff = completed[group][i] - prevTime;\n expect(diff).to.be.below(2100);\n expect(diff).to.be.gte(900);\n prevTime = completed[group][i];\n }\n }\n resolve();\n } catch (err) {\n reject(err);\n }\n });\n\n rateLimitedQueue.on('completed', ({ id }) => {\n const group = _.last(id.split(':'));\n completed[group] = completed[group] || [];\n completed[group].push(Date.now());\n\n afterJobs();\n });\n\n rateLimitedQueue.on('failed', async err => {\n await queue.close();\n reject(err);\n });\n });\n\n for (let i = 0; i < numJobs; i++) {\n rateLimitedQueue.add({ accountId: i % numGroups });\n }\n\n await running;\n await rateLimitedQueue.close();\n });\n});\n" }, { "path": "test/test_repeat.js", "content": "'use strict';\n\nconst expect = require('chai').expect;\nconst utils = require('./utils');\nconst sinon = require('sinon');\nconst redis = require('ioredis');\nconst moment = require('moment');\nconst _ = require('lodash');\n\nconst ONE_SECOND = 1000;\nconst ONE_MINUTE = 60 * ONE_SECOND;\nconst ONE_HOUR = 60 * ONE_MINUTE;\nconst ONE_DAY = 24 * ONE_HOUR;\nconst MAX_INT = 2147483647;\n\ndescribe('repeat', () => {\n let queue;\n let client;\n\n beforeEach(function() {\n this.clock = sinon.useFakeTimers();\n client = new redis();\n return client.flushdb().then(() => {\n queue = utils.buildQueue('repeat', {\n settings: {\n guardInterval: MAX_INT,\n stalledInterval: MAX_INT,\n drainDelay: 1 // Small delay so that .close is faster.\n }\n });\n return queue;\n });\n });\n\n afterEach(function() {\n this.clock.restore();\n return queue.close().then(() => {\n return client.quit();\n });\n });\n\n it('should create multiple jobs if they have the same cron pattern', done => {\n const cron = '*/10 * * * * *';\n const customJobIds = ['customjobone', 'customjobtwo'];\n\n Promise.all([\n queue.add({}, { jobId: customJobIds[0], repeat: { cron } }),\n queue.add({}, { jobId: customJobIds[1], repeat: { cron } })\n ])\n .then(() => {\n return queue.count();\n })\n .then(count => {\n expect(count).to.be.eql(2);\n done();\n })\n .catch(done);\n });\n\n it('should add a repetable job when using stardDate and endDate', async () => {\n const job1 = await queue.add(\n {\n name: 'job1'\n },\n {\n repeat: {\n cron: '0 * * * * *',\n startDate: '2020-09-02T22:29:00Z'\n }\n }\n );\n\n expect(job1).to.exist;\n expect(job1.opts).to.have.property('repeat');\n expect(job1.opts.repeat).to.be.deep.equal({\n count: 1,\n cron: '0 * * * * *',\n startDate: '2020-09-02T22:29:00Z',\n key: '__default__::::0 * * * * *'\n });\n\n const job2 = await queue.add(\n {\n name: 'job2'\n },\n {\n repeat: {\n cron: '0 * * * * *',\n startDate: '2020-09-02T22:29:00Z',\n endDate: '2020-09-05T01:44:37Z'\n }\n }\n );\n expect(job2).to.exist;\n expect(job2.opts).to.have.property('repeat');\n expect(job2.opts.repeat).to.be.deep.equal({\n count: 1,\n cron: '0 * * * * *',\n startDate: '2020-09-02T22:29:00Z',\n endDate: '2020-09-05T01:44:37Z',\n key: '__default__::1599270277000::0 * * * * *'\n });\n });\n\n it('should get repeatable jobs with different cron pattern', done => {\n const crons = [\n '10 * * * * *',\n '2 10 * * * *',\n '1 * * 5 * *',\n '2 * * 4 * *'\n ];\n\n Promise.all([\n queue.add('first', {}, { repeat: { cron: crons[0], endDate: 12345 } }),\n queue.add('second', {}, { repeat: { cron: crons[1], endDate: 610000 } }),\n queue.add(\n 'third',\n {},\n { repeat: { cron: crons[2], tz: 'Africa/Abidjan' } }\n ),\n queue.add(\n 'fourth',\n {},\n { repeat: { cron: crons[3], tz: 'Africa/Accra' } }\n ),\n queue.add('fifth', {}, { repeat: { every: 7563 } })\n ])\n .then(() => {\n return queue.getRepeatableCount();\n })\n .then(count => {\n expect(count).to.be.eql(5);\n return queue.getRepeatableJobs(0, -1, true);\n })\n .then(jobs => {\n return jobs.sort((a, b) => {\n return crons.indexOf(a.cron) > crons.indexOf(b.cron);\n });\n })\n .then(jobs => {\n expect(jobs)\n .to.be.and.an('array')\n .and.have.length(5)\n .and.to.deep.include({\n key: 'first::12345::10 * * * * *',\n name: 'first',\n id: null,\n endDate: 12345,\n tz: null,\n cron: '10 * * * * *',\n every: null,\n next: 10000\n })\n .and.to.deep.include({\n key: 'second::610000::2 10 * * * *',\n name: 'second',\n id: null,\n endDate: 610000,\n tz: null,\n cron: '2 10 * * * *',\n every: null,\n next: 602000\n })\n .and.to.deep.include({\n key: 'fourth:::Africa/Accra:2 * * 4 * *',\n name: 'fourth',\n id: null,\n endDate: null,\n tz: 'Africa/Accra',\n cron: '2 * * 4 * *',\n every: null,\n next: 259202000\n })\n .and.to.deep.include({\n key: 'third:::Africa/Abidjan:1 * * 5 * *',\n name: 'third',\n id: null,\n endDate: null,\n tz: 'Africa/Abidjan',\n cron: '1 * * 5 * *',\n every: null,\n next: 345601000\n })\n .and.to.deep.include({\n key: 'fifth:::7563',\n name: 'fifth',\n id: null,\n tz: null,\n endDate: null,\n cron: null,\n every: 7563,\n next: 7563\n });\n done();\n })\n .catch(done);\n });\n\n it('should repeat every 2 seconds', function(done) {\n this.timeout(20000);\n const _this = this;\n const date = new Date('2017-02-07 9:24:00');\n this.clock.setSystemTime(date);\n const nextTick = 2 * ONE_SECOND + 500;\n\n queue\n .add('repeat', { foo: 'bar' }, { repeat: { cron: '*/2 * * * * *' } })\n .then(() => {\n _this.clock.tick(nextTick);\n });\n\n queue.process('repeat', () => {\n // dummy\n });\n\n let prev;\n let counter = 0;\n queue.on('completed', job => {\n _this.clock.tick(nextTick);\n if (prev) {\n expect(prev.timestamp).to.be.lt(job.timestamp);\n expect(job.timestamp - prev.timestamp).to.be.gte(2000);\n }\n prev = job;\n counter++;\n if (counter == 20) {\n done();\n }\n });\n });\n\n it('should repeat every 2 seconds with startDate in future', function(done) {\n const _this = this;\n const date = new Date('2017-02-07 9:24:00');\n this.clock.setSystemTime(date);\n const nextTick = 2 * ONE_SECOND + 500;\n const delay = 5 * ONE_SECOND + 500;\n\n queue\n .add(\n 'repeat',\n { foo: 'bar' },\n {\n repeat: {\n cron: '*/2 * * * * *',\n startDate: new Date('2017-02-07 9:24:05')\n }\n }\n )\n .then(() => {\n _this.clock.tick(nextTick + delay);\n });\n\n queue.process('repeat', () => {\n // dummy\n });\n\n let prev;\n let counter = 0;\n queue.on('completed', job => {\n _this.clock.tick(nextTick);\n if (prev) {\n expect(prev.timestamp).to.be.lt(job.timestamp);\n expect(job.timestamp - prev.timestamp).to.be.gte(2000);\n }\n prev = job;\n counter++;\n if (counter == 20) {\n done();\n }\n });\n });\n\n it('should repeat every 2 seconds with startDate in past', function(done) {\n const _this = this;\n const date = new Date('2017-02-07 9:24:00');\n this.clock.setSystemTime(date);\n const nextTick = 2 * ONE_SECOND + 500;\n\n queue\n .add(\n 'repeat',\n { foo: 'bar' },\n {\n repeat: {\n cron: '*/2 * * * * *',\n startDate: new Date('2017-02-07 9:22:00')\n }\n }\n )\n .then(() => {\n _this.clock.tick(nextTick);\n });\n\n queue.process('repeat', () => {\n // dummy\n });\n\n let prev;\n let counter = 0;\n queue.on('completed', job => {\n _this.clock.tick(nextTick);\n if (prev) {\n expect(prev.timestamp).to.be.lt(job.timestamp);\n expect(job.timestamp - prev.timestamp).to.be.gte(2000);\n }\n prev = job;\n counter++;\n if (counter == 20) {\n done();\n }\n });\n });\n\n it('should repeat once a day for 5 days', function(done) {\n const _this = this;\n const date = new Date('2017-05-05 13:12:00');\n this.clock.setSystemTime(date);\n const nextTick = ONE_DAY;\n\n queue\n .add(\n 'repeat',\n { foo: 'bar' },\n {\n repeat: {\n cron: '0 1 * * *',\n endDate: new Date('2017-05-10 13:12:00')\n }\n }\n )\n .then(() => {\n _this.clock.tick(nextTick);\n });\n\n queue.process('repeat', () => {\n // Dummy\n });\n\n let prev;\n let counter = 0;\n queue.on('completed', job => {\n _this.clock.tick(nextTick);\n if (prev) {\n expect(prev.timestamp).to.be.lt(job.timestamp);\n expect(job.timestamp - prev.timestamp).to.be.gte(ONE_DAY);\n }\n prev = job;\n\n counter++;\n if (counter == 5) {\n queue.getWaiting().then(jobs => {\n expect(jobs.length).to.be.eql(0);\n queue.getDelayed().then(jobs => {\n expect(jobs.length).to.be.eql(0);\n done();\n });\n });\n }\n });\n });\n\n it('should repeat 7:th day every month at 9:25', function(done) {\n this.timeout(20000);\n const _this = this;\n const date = new Date('2017-02-02 7:21:42');\n this.clock.setSystemTime(date);\n\n function nextTick() {\n const now = moment();\n const nextMonth = moment().add(1, 'months');\n _this.clock.tick(nextMonth - now);\n }\n\n queue\n .add('repeat', { foo: 'bar' }, { repeat: { cron: '* 25 9 7 * *' } })\n .then(() => {\n nextTick();\n });\n\n queue.process('repeat', (/*job*/) => {\n // Dummy\n });\n\n let counter = 20;\n let prev;\n queue.on('completed', job => {\n if (prev) {\n expect(prev.timestamp).to.be.lt(job.timestamp);\n const diff = moment(job.timestamp).diff(\n moment(prev.timestamp),\n 'months',\n true\n );\n expect(diff).to.be.gte(1);\n }\n prev = job;\n\n counter--;\n if (counter == 0) {\n done();\n }\n nextTick();\n });\n });\n\n it('should create two jobs with the same ids', () => {\n const options = {\n repeat: {\n cron: '0 1 * * *'\n }\n };\n\n const p1 = queue.add({ foo: 'bar' }, options);\n const p2 = queue.add({ foo: 'bar' }, options);\n\n return Promise.all([p1, p2]).then(jobs => {\n expect(jobs.length).to.be.eql(2);\n expect(jobs[0].id).to.be.eql(jobs[1].id);\n });\n });\n\n it('should allow removing a named repeatable job', function(done) {\n const _this = this;\n const date = new Date('2017-02-07 9:24:00');\n this.clock.setSystemTime(date);\n\n const nextTick = 2 * ONE_SECOND + 250;\n const repeat = { cron: '*/2 * * * * *' };\n\n queue.add('remove', { foo: 'bar' }, { repeat }).then(() => {\n _this.clock.tick(nextTick);\n });\n\n queue.process('remove', () => {\n counter++;\n if (counter == 20) {\n return queue.removeRepeatable('remove', repeat).then(() => {\n _this.clock.tick(nextTick);\n return queue.getDelayed().then(delayed => {\n expect(delayed).to.be.empty;\n done();\n return null;\n });\n });\n } else if (counter > 20) {\n done(Error('should not repeat more than 20 times'));\n }\n });\n\n let prev;\n let counter = 0;\n queue.on('completed', job => {\n _this.clock.tick(nextTick);\n if (prev) {\n expect(prev.timestamp).to.be.lt(job.timestamp);\n expect(job.timestamp - prev.timestamp).to.be.gte(2000);\n }\n prev = job;\n });\n });\n\n it('should be able to remove repeatable jobs by key', async () => {\n const repeat = { cron: '*/2 * * * * *' };\n\n await queue.add('remove', { foo: 'bar' }, { repeat });\n\n const repeatableJobs = await queue.getRepeatableJobs();\n expect(repeatableJobs).to.have.length(1);\n await queue.removeRepeatableByKey(repeatableJobs[0].key);\n\n const repeatableJobs2 = await queue.getRepeatableJobs();\n expect(repeatableJobs2).to.have.length(0);\n\n const delayedJobs = await queue.getDelayed();\n expect(delayedJobs).to.have.length(0);\n });\n\n it('should return repeatable job key', async () => {\n const repeat = { cron: '*/2 * * * * *' };\n\n const job = await queue.add('remove', { foo: 'bar' }, { repeat });\n\n expect(job.opts.repeat.key).to.be.equal('remove::::*/2 * * * * *');\n });\n\n it('should be able to remove repeatable jobs by key that has a jobId', async () => {\n const repeat = { cron: '*/2 * * * * *' };\n\n await queue.add('remove', { foo: 'bar' }, { jobId: 'qux', repeat });\n\n const repeatableJobs = await queue.getRepeatableJobs();\n expect(repeatableJobs).to.have.length(1);\n await queue.removeRepeatableByKey(repeatableJobs[0].key);\n\n const repeatableJobs2 = await queue.getRepeatableJobs();\n expect(repeatableJobs2).to.have.length(0);\n\n const delayedJobs = await queue.getDelayed();\n expect(delayedJobs).to.have.length(0);\n });\n\n it('should allow removing a customId repeatable job', function(done) {\n const _this = this;\n const date = new Date('2017-02-07 9:24:00');\n this.clock.setSystemTime(date);\n\n const nextTick = 2 * ONE_SECOND + 250;\n const repeat = { cron: '*/2 * * * * *' };\n\n queue.add({ foo: 'bar' }, { repeat: repeat, jobId: 'xxxx' }).then(() => {\n _this.clock.tick(nextTick);\n });\n\n queue.process(() => {\n counter++;\n if (counter == 20) {\n return queue\n .removeRepeatable(_.defaults({ jobId: 'xxxx' }, repeat))\n .then(() => {\n _this.clock.tick(nextTick);\n return queue.getDelayed().then(delayed => {\n expect(delayed).to.be.empty;\n done();\n return null;\n });\n });\n } else if (counter > 20) {\n done(Error('should not repeat more than 20 times'));\n }\n });\n\n let prev;\n let counter = 0;\n queue.on('completed', job => {\n _this.clock.tick(nextTick);\n if (prev) {\n expect(prev.timestamp).to.be.lt(job.timestamp);\n expect(job.timestamp - prev.timestamp).to.be.gte(2000);\n }\n prev = job;\n });\n });\n\n it('should not re-add a repeatable job after it has been removed', function() {\n const _this = this;\n const date = new Date('2017-02-07 9:24:00');\n const nextTick = 2 * ONE_SECOND + 250;\n const repeat = { cron: '*/2 * * * * *' };\n const nextRepeatableJob = queue.nextRepeatableJob;\n this.clock.setSystemTime(date);\n\n const afterRemoved = new Promise(resolve => {\n queue.process(() => {\n queue.nextRepeatableJob = function() {\n const args = arguments;\n // In order to simulate race condition\n // Make removeRepeatables happen any time after a moveToX is called\n return queue\n .removeRepeatable(_.defaults({ jobId: 'xxxx' }, repeat))\n .then(() => {\n // nextRepeatableJob will now re-add the removed repeatable\n return nextRepeatableJob.apply(queue, args);\n })\n .then(result => {\n resolve();\n return result;\n });\n };\n });\n\n queue.add({ foo: 'bar' }, { repeat: repeat, jobId: 'xxxx' }).then(() => {\n _this.clock.tick(nextTick);\n });\n\n queue.on('completed', () => {\n _this.clock.tick(nextTick);\n });\n });\n\n return afterRemoved.then(() => {\n return queue.getRepeatableJobs().then(jobs => {\n // Repeatable job was recreated\n expect(jobs.length).to.eql(0);\n });\n });\n });\n\n it('should allow adding a repeatable job after removing it', () => {\n queue.process((/*job*/) => {\n // dummy\n });\n\n const repeat = {\n cron: '*/5 * * * *'\n };\n\n return queue\n .add(\n 'myTestJob',\n {\n data: '2'\n },\n {\n repeat\n }\n )\n .then(() => {\n return queue.getDelayed();\n })\n .then(delayed => {\n expect(delayed.length).to.be.eql(1);\n })\n .then(() => {\n return queue.removeRepeatable('myTestJob', repeat);\n })\n .then(() => {\n return queue.getDelayed();\n })\n .then(delayed => {\n expect(delayed.length).to.be.eql(0);\n })\n .then(() => {\n return queue.add(\n 'myTestJob',\n {\n data: '2'\n },\n {\n repeat\n }\n );\n })\n .then(() => {\n return queue.getDelayed();\n })\n .then(delayed => {\n expect(delayed.length).to.be.eql(1);\n });\n });\n\n it('should not repeat more than 5 times', function(done) {\n const _this = this;\n const date = new Date('2017-02-07 9:24:00');\n this.clock.setSystemTime(date);\n const nextTick = ONE_SECOND + 500;\n\n queue\n .add(\n 'repeat',\n { foo: 'bar' },\n { repeat: { limit: 5, cron: '*/1 * * * * *' } }\n )\n .then(() => {\n _this.clock.tick(nextTick);\n });\n\n queue.process('repeat', () => {\n // dummy\n });\n\n let counter = 0;\n queue.on('completed', () => {\n _this.clock.tick(nextTick);\n counter++;\n if (counter == 5) {\n utils.sleep(nextTick * 2).then(() => {\n done();\n }, nextTick * 2);\n } else if (counter > 5) {\n done(Error('should not repeat more than 5 times'));\n }\n });\n }).timeout(5000);\n\n it('should processes delayed jobs by priority', function(done) {\n const _this = this;\n const jobAdds = [];\n let currentPriority = 1;\n const nextTick = 1000;\n\n const date = new Date('2017-02-07 9:24:00');\n this.clock.setSystemTime(date);\n\n jobAdds.push(queue.add({ p: 1 }, { priority: 1, delay: nextTick * 2 }));\n jobAdds.push(queue.add({ p: 3 }, { priority: 3, delay: nextTick * 2 }));\n jobAdds.push(queue.add({ p: 2 }, { priority: 2, delay: nextTick * 2 }));\n\n Promise.all(jobAdds).then(() => {\n _this.clock.tick(nextTick * 3);\n\n queue.process((job, jobDone) => {\n try {\n expect(job.id).to.be.ok;\n expect(job.data.p).to.be.eql(currentPriority++);\n } catch (err) {\n done(err);\n }\n jobDone();\n\n if (currentPriority > 3) {\n done();\n }\n });\n }, done);\n });\n\n it('should use \".every\" as a valid interval', function(done) {\n const _this = this;\n const interval = ONE_SECOND * 2;\n const date = new Date('2017-02-07 9:24:00');\n\n // Quantize time\n const time = Math.floor(date.getTime() / interval) * interval;\n this.clock.setSystemTime(time);\n\n const nextTick = ONE_SECOND * 2 + 500;\n\n queue\n .add('repeat m', { type: 'm' }, { repeat: { every: interval } })\n .then(() => {\n return queue.add(\n 'repeat s',\n { type: 's' },\n { repeat: { every: interval } }\n );\n })\n .then(() => {\n _this.clock.tick(nextTick);\n });\n\n queue.process('repeat m', () => {\n // dummy\n });\n\n queue.process('repeat s', () => {\n // dummy\n });\n\n let prevType;\n let counter = 0;\n queue.on('completed', job => {\n _this.clock.tick(nextTick);\n if (prevType) {\n expect(prevType).to.not.be.eql(job.data.type);\n }\n prevType = job.data.type;\n counter++;\n if (counter == 20) {\n done();\n }\n });\n });\n\n it('should throw an error when using .cron and .every simutaneously', done => {\n queue\n .add(\n 'repeat',\n { type: 'm' },\n { repeat: { every: 5000, cron: '*/1 * * * * *' } }\n )\n .then(\n () => {\n throw new Error('The error was not thrown');\n },\n err => {\n expect(err.message).to.be.eql(\n 'Both .cron and .every options are defined for this repeatable job'\n );\n done();\n }\n );\n });\n\n // This tests works well locally but fails in travis for some unknown reason.\n it('should emit a waiting event when adding a repeatable job to the waiting list', function(done) {\n const _this = this;\n const date = new Date('2017-02-07 9:24:00');\n this.clock.setSystemTime(date);\n const nextTick = 2 * ONE_SECOND + 500;\n\n queue.on('waiting', jobId => {\n expect(jobId).to.be.equal(\n 'repeat:93168b0ea97b55fb5a8325e8c66e4300:' +\n (date.getTime() + 2 * ONE_SECOND)\n );\n done();\n });\n\n queue\n .add('repeat', { foo: 'bar' }, { repeat: { cron: '*/2 * * * * *' } })\n .then(() => {\n _this.clock.tick(nextTick);\n });\n\n queue.process('repeat', () => {});\n });\n\n it('should have the right count value', function(done) {\n const _this = this;\n\n queue.add({ foo: 'bar' }, { repeat: { every: 1000 } }).then(() => {\n _this.clock.tick(ONE_SECOND + 10);\n });\n\n queue.process(job => {\n if (job.opts.repeat.count === 1) {\n done();\n } else {\n done(Error('repeatable job got the wrong repeat count'));\n }\n });\n });\n\n it('it should stop repeating after endDate', async function() {\n const every = 100;\n const date = new Date('2017-02-07 9:24:00');\n this.clock.setSystemTime(date);\n\n await queue.add(\n { id: 'my id' },\n {\n repeat: {\n endDate: Date.now() + 1000,\n every: 100\n }\n }\n );\n\n this.clock.tick(every + 1);\n\n let processed = 0;\n queue.process(async () => {\n this.clock.tick(every);\n processed++;\n });\n\n await utils.sleep(1100);\n\n const delayed = await queue.getDelayed();\n\n expect(delayed).to.have.length(0);\n expect(processed).to.be.equal(10);\n });\n});\n" }, { "path": "test/test_sandboxed_process.js", "content": "'use strict';\n\nconst expect = require('chai').expect;\nconst utils = require('./utils');\nconst redis = require('ioredis');\nconst _ = require('lodash');\nconst delay = require('delay');\nconst pReflect = require('p-reflect');\n\ndescribe('sandboxed process', () => {\n let queue;\n let client;\n\n beforeEach(() => {\n client = new redis();\n return client.flushdb().then(() => {\n queue = utils.buildQueue('test process', {\n settings: {\n guardInterval: 300000,\n stalledInterval: 300000\n }\n });\n return queue;\n });\n });\n\n afterEach(() => {\n return queue\n .close()\n .then(() => {\n return client.flushall();\n })\n .then(() => {\n return client.quit();\n });\n });\n\n it('should process and complete', done => {\n const processFile = __dirname + '/fixtures/fixture_processor.js';\n queue.process(processFile);\n\n queue.on('completed', (job, value) => {\n try {\n expect(job.data).to.be.eql({ foo: 'bar' });\n expect(value).to.be.eql(42);\n expect(Object.keys(queue.childPool.retained)).to.have.lengthOf(0);\n expect(queue.childPool.free[processFile]).to.have.lengthOf(1);\n done();\n } catch (err) {\n done(err);\n }\n });\n\n queue.add({ foo: 'bar' });\n });\n\n it('should process with named processor', done => {\n const processFile = __dirname + '/fixtures/fixture_processor.js';\n queue.process('foobar', processFile);\n\n queue.on('completed', (job, value) => {\n try {\n expect(job.data).to.be.eql({ foo: 'bar' });\n expect(value).to.be.eql(42);\n expect(Object.keys(queue.childPool.retained)).to.have.lengthOf(0);\n expect(queue.childPool.free[processFile]).to.have.lengthOf(1);\n done();\n } catch (err) {\n done(err);\n }\n });\n\n queue.add('foobar', { foo: 'bar' });\n });\n\n it('should process with several named processors', done => {\n const processFileFoo = __dirname + '/fixtures/fixture_processor_foo.js';\n const processFileBar = __dirname + '/fixtures/fixture_processor_bar.js';\n\n queue.process('foo', processFileFoo);\n queue.process('bar', processFileBar);\n\n let count = 0;\n queue.on('completed', (job, value) => {\n let data, result, processFile, retainedLength;\n count++;\n if (count == 1) {\n data = { foo: 'bar' };\n result = 'foo';\n processFile = processFileFoo;\n retainedLength = 1;\n } else {\n data = { bar: 'qux' };\n result = 'bar';\n processFile = processFileBar;\n retainedLength = 0;\n }\n\n try {\n expect(job.data).to.be.eql(data);\n expect(value).to.be.eql(result);\n expect(Object.keys(queue.childPool.retained)).to.have.lengthOf(\n retainedLength\n );\n expect(queue.childPool.free[processFile]).to.have.lengthOf(1);\n if (count === 2) {\n done();\n }\n } catch (err) {\n console.error(err);\n done(err);\n }\n });\n\n queue.add('foo', { foo: 'bar' }).then(() => {\n delay(500).then(() => {\n queue.add('bar', { bar: 'qux' });\n });\n });\n\n queue.on('error', err => {\n console.error(err);\n });\n });\n\n it('should process with concurrent processors', done => {\n const after = _.after(4, () => {\n expect(queue.childPool.getAllFree().length).to.eql(4);\n done();\n });\n queue.on('completed', (job, value) => {\n try {\n expect(value).to.be.eql(42);\n expect(\n Object.keys(queue.childPool.retained).length +\n queue.childPool.getAllFree().length\n ).to.eql(4);\n after();\n } catch (err) {\n done(err);\n }\n });\n\n Promise.all([\n queue.add({ foo: 'bar1' }),\n queue.add({ foo: 'bar2' }),\n queue.add({ foo: 'bar3' }),\n queue.add({ foo: 'bar4' })\n ]).then(() => {\n queue.process(4, __dirname + '/fixtures/fixture_processor_slow.js');\n });\n });\n\n it('should reuse process with single processors', done => {\n const after = _.after(4, () => {\n expect(queue.childPool.getAllFree().length).to.eql(1);\n done();\n });\n queue.on('completed', (job, value) => {\n try {\n expect(value).to.be.eql(42);\n expect(\n Object.keys(queue.childPool.retained).length +\n queue.childPool.getAllFree().length\n ).to.eql(1);\n after();\n } catch (err) {\n done(err);\n }\n });\n\n Promise.all([\n queue.add({ foo: 'bar1' }),\n queue.add({ foo: 'bar2' }),\n queue.add({ foo: 'bar3' }),\n queue.add({ foo: 'bar4' })\n ]).then(() => {\n queue.process(__dirname + '/fixtures/fixture_processor_slow.js');\n });\n }).timeout(5000);\n\n it('should process and complete using done', done => {\n queue.process(__dirname + '/fixtures/fixture_processor_callback.js');\n\n queue.on('completed', (job, value) => {\n try {\n expect(job.data).to.be.eql({ foo: 'bar' });\n expect(value).to.be.eql(42);\n expect(Object.keys(queue.childPool.retained)).to.have.lengthOf(0);\n expect(queue.childPool.getAllFree()).to.have.lengthOf(1);\n done();\n } catch (err) {\n done(err);\n }\n });\n\n queue.add({ foo: 'bar' });\n });\n\n it('should process and update progress', done => {\n queue.process(__dirname + '/fixtures/fixture_processor_progress.js');\n\n queue.on('completed', async (job, value) => {\n try {\n expect(job.data).to.be.eql({ foo: 'bar' });\n expect(value).to.be.eql(37);\n expect(job.progress()).to.be.eql(100);\n expect(progresses).to.be.eql([10, 27, 78, 100]);\n expect(Object.keys(queue.childPool.retained)).to.have.lengthOf(0);\n expect(queue.childPool.getAllFree()).to.have.lengthOf(1);\n await queue.getJobLogs(job.id).then(logs =>\n expect(logs).to.be.eql({\n logs: ['10', '27', '78', '100'],\n count: 4\n })\n );\n await queue.getJobLogs(job.id, 2, 2).then(logs =>\n expect(logs).to.be.eql({\n logs: ['78'],\n count: 4\n })\n );\n await queue.getJobLogs(job.id, 0, 1).then(logs =>\n expect(logs).to.be.eql({\n logs: ['10', '27'],\n count: 4\n })\n );\n await queue.getJobLogs(job.id, 1, 2).then(logs =>\n expect(logs).to.be.eql({\n logs: ['27', '78'],\n count: 4\n })\n );\n await queue.getJobLogs(job.id, 2, 2, false).then(logs =>\n expect(logs).to.be.eql({\n logs: ['27'],\n count: 4\n })\n );\n await queue.getJobLogs(job.id, 0, 1, false).then(logs =>\n expect(logs).to.be.eql({\n logs: ['100', '78'],\n count: 4\n })\n );\n await queue.getJobLogs(job.id, 1, 2, false).then(logs =>\n expect(logs).to.be.eql({\n logs: ['78', '27'],\n count: 4\n })\n );\n done();\n } catch (err) {\n done(err);\n }\n });\n\n const progresses = [];\n queue.on('progress', (job, progress) => {\n progresses.push(progress);\n });\n\n queue.add({ foo: 'bar' });\n });\n\n it('should process and update data', done => {\n queue.process(__dirname + '/fixtures/fixture_processor_data.js');\n\n queue.on('completed', (job, value) => {\n try {\n expect(job.data).to.be.eql({ baz: 'qux' });\n expect(value).to.be.eql({ baz: 'qux' });\n expect(Object.keys(queue.childPool.retained)).to.have.lengthOf(0);\n expect(queue.childPool.getAllFree()).to.have.lengthOf(1);\n done();\n } catch (err) {\n done(err);\n }\n });\n\n queue.add({ foo: 'bar' });\n });\n\n it('should process, discard and fail without retry', done => {\n queue.process(__dirname + '/fixtures/fixture_processor_discard.js');\n\n queue.on('failed', (job, err) => {\n try {\n expect(job.data).eql({ foo: 'bar' });\n expect(job.isDiscarded()).to.be.true;\n expect(job.failedReason).eql('Manually discarded processor');\n expect(err.message).eql('Manually discarded processor');\n expect(err.stack).include('fixture_processor_discard.js');\n expect(Object.keys(queue.childPool.retained)).to.have.lengthOf(0);\n expect(queue.childPool.getAllFree()).to.have.lengthOf(1);\n done();\n } catch (err) {\n done(err);\n }\n });\n\n queue.add({ foo: 'bar' });\n });\n\n it('should process and fail', done => {\n queue.process(__dirname + '/fixtures/fixture_processor_fail.js');\n\n queue.on('failed', (job, err) => {\n try {\n expect(job.data).eql({ foo: 'bar' });\n expect(job.failedReason).eql('Manually failed processor');\n expect(err.message).eql('Manually failed processor');\n expect(err.stack).include('fixture_processor_fail.js');\n expect(Object.keys(queue.childPool.retained)).to.have.lengthOf(0);\n expect(queue.childPool.getAllFree()).to.have.lengthOf(1);\n done();\n } catch (err) {\n done(err);\n }\n });\n\n queue.add({ foo: 'bar' });\n });\n\n it('should error if processor file is missing', done => {\n try {\n queue.process(__dirname + '/fixtures/missing_processor.js');\n done(new Error('did not throw error'));\n } catch (err) {\n done();\n }\n });\n\n it('should process and fail using callback', done => {\n queue.process(__dirname + '/fixtures/fixture_processor_callback_fail.js');\n\n queue.on('failed', (job, err) => {\n try {\n expect(job.data).eql({ foo: 'bar' });\n expect(job.failedReason).eql('Manually failed processor');\n expect(err.message).eql('Manually failed processor');\n expect(Object.keys(queue.childPool.retained)).to.have.lengthOf(0);\n expect(queue.childPool.getAllFree()).to.have.lengthOf(1);\n done();\n } catch (err) {\n done(err);\n }\n });\n\n queue.add({ foo: 'bar' });\n });\n\n it('should fail if the process crashes', () => {\n queue.process(__dirname + '/fixtures/fixture_processor_crash.js');\n\n return queue\n .add({})\n .then(job => {\n return pReflect(Promise.resolve(job.finished()));\n })\n .then(inspection => {\n expect(inspection.isRejected).to.be.eql(true);\n expect(inspection.reason.message).to.be.eql('boom!');\n });\n });\n\n it('should fail if the process exits 0', () => {\n queue.process(__dirname + '/fixtures/fixture_processor_crash.js');\n\n return queue\n .add({ exitCode: 0 })\n .then(job => {\n return pReflect(Promise.resolve(job.finished()));\n })\n .then(inspection => {\n expect(inspection.isRejected).to.be.eql(true);\n expect(inspection.reason.message).to.be.eql(\n 'Unexpected exit code: 0 signal: null'\n );\n });\n });\n\n it('should fail if the process exits non-0', () => {\n queue.process(__dirname + '/fixtures/fixture_processor_crash.js');\n\n return queue\n .add({ exitCode: 1 })\n .then(job => {\n return pReflect(Promise.resolve(job.finished()));\n })\n .then(inspection => {\n expect(inspection.isRejected).to.be.eql(true);\n expect(inspection.reason.message).to.be.eql(\n 'Unexpected exit code: 1 signal: null'\n );\n });\n });\n\n it('should remove exited process', done => {\n queue.process(__dirname + '/fixtures/fixture_processor_exit.js');\n\n queue.on('completed', () => {\n try {\n expect(Object.keys(queue.childPool.retained)).to.have.lengthOf(0);\n expect(queue.childPool.getAllFree()).to.have.lengthOf(1);\n delay(500)\n .then(() => {\n expect(Object.keys(queue.childPool.retained)).to.have.lengthOf(0);\n expect(queue.childPool.getAllFree()).to.have.lengthOf(0);\n })\n .then(() => {\n done();\n }, done);\n } catch (err) {\n done(err);\n }\n });\n\n queue.add({ foo: 'bar' });\n });\n\n it('should allow the job to complete and then exit on clean', async function() {\n this.timeout(1500);\n const processFile = __dirname + '/fixtures/fixture_processor_slow.js';\n queue.process(processFile);\n\n // aquire and release a child here so we know it has it's full termination handler setup\n const expectedChild = await queue.childPool.retain(processFile);\n queue.childPool.release(expectedChild);\n const onActive = new Promise(resolve => queue.once('active', resolve));\n const jobAddPromise = queue.add({ foo: 'bar' });\n\n await onActive;\n\n // at this point the job should be active and running on the child\n expect(Object.keys(queue.childPool.retained)).to.have.lengthOf(1);\n expect(queue.childPool.getAllFree()).to.have.lengthOf(0);\n const child = Object.values(queue.childPool.retained)[0];\n expect(child).to.equal(expectedChild);\n expect(child.exitCode).to.equal(null);\n expect(child.finished).to.equal(undefined);\n\n // trigger a clean while we know it's doing work\n await queue.childPool.clean();\n\n // ensure the child did get cleaned up\n expect(expectedChild.killed).to.eql(true);\n expect(Object.keys(queue.childPool.retained)).to.have.lengthOf(0);\n expect(queue.childPool.getAllFree()).to.have.lengthOf(0);\n\n // make sure the job completed successfully\n const job = await jobAddPromise;\n const jobResult = await job.finished();\n expect(jobResult).to.equal(42);\n });\n\n it('should share child pool across all different queues created', async () => {\n const [queueA, queueB] = await Promise.all([\n utils.newQueue('queueA', { settings: { isSharedChildPool: true } }),\n utils.newQueue('queueB', { settings: { isSharedChildPool: true } })\n ]);\n\n const processFile = __dirname + '/fixtures/fixture_processor.js';\n queueA.process(processFile);\n queueB.process(processFile);\n\n await Promise.all([queueA.add(), queueB.add()]);\n\n expect(queueA.childPool).to.be.eql(queueB.childPool);\n });\n\n it(\"should not share childPool across different queues if isSharedChildPool isn't specified\", async () => {\n const [queueA, queueB] = await Promise.all([\n utils.newQueue('queueA', { settings: { isSharedChildPool: false } }),\n utils.newQueue('queueB')\n ]);\n\n const processFile = __dirname + '/fixtures/fixture_processor.js';\n queueA.process(processFile);\n queueB.process(processFile);\n\n await Promise.all([queueA.add(), queueB.add()]);\n\n expect(queueA.childPool).to.not.be.equal(queueB.childPool);\n });\n\n it('should fail if the process file is broken', async () => {\n const processFile = __dirname + '/fixtures/fixture_processor_broken.js';\n queue.process(processFile);\n await queue.add('test', {});\n\n return new Promise(resolve => {\n queue.on('failed', () => {\n resolve();\n });\n });\n });\n});\n" }, { "path": "test/test_when_current_jobs_finished.js", "content": "'use strict';\n\nconst expect = require('chai').expect;\nconst redis = require('ioredis');\nconst utils = require('./utils');\nconst delay = require('delay');\nconst sinon = require('sinon');\n\ndescribe('.whenCurrentJobsFinished', () => {\n let client;\n beforeEach(() => {\n client = new redis();\n return client.flushdb();\n });\n\n afterEach(async () => {\n sinon.restore();\n await utils.cleanupQueues();\n await client.flushdb();\n return client.quit();\n });\n\n it('should handle queue with no processor', async () => {\n const queue = await utils.newQueue();\n expect(await queue.whenCurrentJobsFinished()).to.equal(undefined);\n });\n\n it('should handle queue with no jobs', async () => {\n const queue = await utils.newQueue();\n queue.process(() => Promise.resolve());\n expect(await queue.whenCurrentJobsFinished()).to.equal(undefined);\n });\n\n it('should wait for job to complete', async () => {\n const queue = await utils.newQueue();\n await queue.add({});\n\n let finishJob;\n\n // wait for job to be active\n await new Promise(resolve => {\n queue.process(() => {\n resolve();\n\n return new Promise(resolve => {\n finishJob = resolve;\n });\n });\n });\n\n let isFulfilled = false;\n const finished = queue.whenCurrentJobsFinished().then(() => {\n isFulfilled = true;\n });\n\n await delay(100);\n expect(isFulfilled).to.equal(false);\n\n finishJob();\n expect(await finished).to.equal(\n undefined,\n 'whenCurrentJobsFinished should resolve once jobs are finished'\n );\n });\n\n it('should wait for all jobs to complete', async () => {\n const queue = await utils.newQueue();\n\n // add multiple jobs to queue\n await queue.add({});\n await queue.add({});\n\n let finishJob1;\n let finishJob2;\n\n // wait for all jobs to be active\n await new Promise(resolve => {\n let callCount = 0;\n queue.process(2, () => {\n callCount++;\n if (callCount === 1) {\n return new Promise(resolve => {\n finishJob1 = resolve;\n });\n }\n\n resolve();\n return new Promise(resolve => {\n finishJob2 = resolve;\n });\n });\n });\n\n let isFulfilled = false;\n const finished = queue.whenCurrentJobsFinished().then(() => {\n isFulfilled = true;\n });\n\n finishJob2();\n await delay(100);\n\n expect(isFulfilled).to.equal(\n false,\n 'should not fulfill until all jobs are finished'\n );\n\n finishJob1();\n await delay(100);\n expect(await finished).to.equal(\n undefined,\n 'whenCurrentJobsFinished should resolve once all jobs are finished'\n );\n });\n\n it('should wait for job to fail', async () => {\n const queue = await utils.newQueue();\n await queue.add({});\n\n let rejectJob;\n\n // wait for job to be active\n await new Promise(resolve => {\n queue.process(() => {\n resolve();\n\n return new Promise((resolve, reject) => {\n rejectJob = reject;\n });\n });\n });\n\n let isFulfilled = false;\n const finished = queue.whenCurrentJobsFinished().then(() => {\n isFulfilled = true;\n });\n\n await delay(100);\n expect(isFulfilled).to.equal(false);\n\n rejectJob();\n expect(await finished).to.equal(\n undefined,\n 'whenCurrentJobsFinished should resolve once jobs are finished'\n );\n });\n});\n" }, { "path": "test/test_worker.js", "content": "'use strict';\n\nconst expect = require('chai').expect;\nconst utils = require('./utils');\nconst redis = require('ioredis');\n\ndescribe('workers', () => {\n let queue;\n let client;\n\n beforeEach(() => {\n client = new redis();\n return client.flushdb().then(() => {\n queue = utils.buildQueue('test workers', {\n settings: {\n guardInterval: 300000,\n stalledInterval: 300000\n }\n });\n return queue;\n });\n });\n\n afterEach(() => {\n return queue.close().then(() => {\n return client.quit();\n });\n });\n\n it('should get all workers for this queue', async () => {\n queue.process(() => {});\n\n await queue.bclient.ping();\n\n const workers = await queue.getWorkers();\n expect(workers).to.have.length(1);\n });\n});\n" }, { "path": "test/utils.js", "content": "'use strict';\n\nconst Queue = require('../');\nconst STD_QUEUE_NAME = 'test queue';\nconst _ = require('lodash');\n\nlet queues = [];\n\nconst originalSetTimeout = setTimeout;\n\nfunction simulateDisconnect(queue) {\n queue.client.disconnect();\n queue.eclient.disconnect();\n}\n\nfunction buildQueue(name, options) {\n options = _.extend({ redis: { port: 6379, host: '127.0.0.1' } }, options);\n const queue = new Queue(name || STD_QUEUE_NAME, options);\n queues.push(queue);\n return queue;\n}\n\nfunction newQueue(name, opts) {\n const queue = buildQueue(name, opts);\n return queue.isReady();\n}\n\nfunction cleanupQueue(queue) {\n return queue.empty().then(queue.close.bind(queue));\n}\n\nfunction cleanupQueues() {\n return Promise.all(\n queues.map(queue => {\n const errHandler = function() {};\n queue.on('error', errHandler);\n return queue.close().catch(errHandler);\n })\n ).then(() => {\n queues = [];\n });\n}\n\nfunction sleep(ms) {\n return new Promise(resolve => {\n originalSetTimeout(() => {\n resolve();\n }, ms);\n });\n}\n\nmodule.exports = {\n simulateDisconnect,\n buildQueue,\n cleanupQueue,\n newQueue,\n cleanupQueues,\n sleep\n};\n" } ]