Repository: ali-sdk/ali-ons Branch: master Commit: 85b7d51f8f24 Files: 69 Total size: 244.6 KB Directory structure: gitextract_g5wtfrkh/ ├── .eslintignore ├── .eslintrc ├── .github/ │ └── workflows/ │ ├── ci-actions.yml │ └── release.yml ├── .gitignore ├── AUTHORS ├── CHANGELOG.md ├── History.md ├── LICENSE ├── README.md ├── example/ │ ├── config.js │ ├── consumer.js │ ├── logger.js │ └── producer.js ├── lib/ │ ├── channel.js │ ├── client_config.js │ ├── consumer/ │ │ ├── consume_from_where.js │ │ ├── mq_push_consumer.js │ │ ├── pull_status.js │ │ └── rebalance/ │ │ ├── allocate_message_queue_averagely.js │ │ └── allocate_message_queue_strategy.js │ ├── index.js │ ├── logger.js │ ├── message/ │ │ ├── message.js │ │ ├── message_const.js │ │ └── message_decoder.js │ ├── message_queue.js │ ├── mix_all.js │ ├── mq_client.js │ ├── mq_client_api.js │ ├── process_queue.js │ ├── producer/ │ │ ├── mq_producer.js │ │ ├── send_status.js │ │ └── topic_publish_info.js │ ├── protocol/ │ │ ├── command/ │ │ │ ├── opaque_generator.js │ │ │ └── remoting_command.js │ │ ├── consume_type.js │ │ ├── message_model.js │ │ ├── perm_name.js │ │ ├── request_code.js │ │ └── response_code.js │ ├── remoting_client.js │ ├── store/ │ │ ├── index.js │ │ ├── local_file.js │ │ ├── local_memory.js │ │ ├── read_offset_type.js │ │ └── remote_broker.js │ └── utils/ │ ├── index.js │ ├── message_sys_flag.js │ └── pull_sys_flag.js ├── package.json └── test/ ├── allocate_message_queue_averagely.test.js ├── channel.test.js ├── consumer/ │ └── rebalance/ │ └── allocate_message_queue_averagely.test.js ├── index.test.js ├── index_namesrv.test.js ├── message/ │ └── message_decoder.test.js ├── mq_client.test.js ├── mq_client_api.test.js ├── protocol/ │ └── command/ │ ├── opaque_generator.test.js │ └── remoting_command.test.js ├── remoting_client.test.js ├── store/ │ ├── local_file.test.js │ ├── local_memory.test.js │ └── remote_broker.test.js ├── utils/ │ ├── index.test.js │ ├── message_sys_flag.test.js │ └── pull_sys_flag.test.js └── utils.js ================================================ FILE CONTENTS ================================================ ================================================ FILE: .eslintignore ================================================ test/fixtures test/benchmark node_modules coverage benchmark _lib example ================================================ FILE: .eslintrc ================================================ { "extends": "eslint-config-egg" } ================================================ FILE: .github/workflows/ci-actions.yml ================================================ name: ci-actions on: push: branches: [ master ] pull_request: branches: [ master ] merge_group: jobs: test: runs-on: ubuntu-latest environment: 'ci' strategy: matrix: node-version: [16, 18, 20, 22, 24] env: ALI_SDK_ONS_ID: ${{ secrets.ALI_SDK_ONS_ID }} ALI_SDK_ONS_SECRET: ${{ secrets.ALI_SDK_ONS_SECRET }} steps: - name: Start RocketMQ Services run: | docker network create rocketmq docker run -d --name rocketmq-namesrv --network rocketmq -p 9876:9876 apache/rocketmq:5.3.2 sh mqnamesrv echo "Waiting for RocketMQ services to be ready..." sleep 20 docker logs rocketmq-namesrv echo "brokerIP1=127.0.0.1" > broker.conf broker_ports=(10911 10912) for port in ${broker_ports[@]}; do docker run -d --name rocketmq-broker-${port} --network rocketmq -p ${port}:${port} -e "NAMESRV_ADDR=rocketmq-namesrv:9876" -v ./broker.conf:/home/rocketmq/conf/broker.conf apache/rocketmq:5.3.2 sh mqbroker --enable-proxy -c /home/rocketmq/conf/broker.conf echo "Waiting for RocketMQ broker to be ready..." sleep 20 docker logs rocketmq-broker-${port} docker exec rocketmq-broker-${port} sh mqadmin updatetopic -t TP_alions_test_topic -c DefaultCluster done - uses: actions/checkout@v5 - name: Use Node.js ${{ matrix.node-version }} uses: actions/setup-node@v6 with: node-version: ${{ matrix.node-version }} - name: npm install, build, and test run: | npm install npm run ci - name: Codecov uses: codecov/codecov-action@v2 with: token: ${{ secrets.CODECOV_TOKEN }} ================================================ FILE: .github/workflows/release.yml ================================================ name: Release on: push: branches: [ master ] permissions: contents: write deployments: write issues: write pull-requests: write id-token: write jobs: release: name: NPM uses: ali-sdk/github-actions/.github/workflows/npm-release.yml@master secrets: GIT_TOKEN: ${{ secrets.GIT_TOKEN }} ================================================ FILE: .gitignore ================================================ .DS_Store # Logs logs *.log # Runtime data pids *.pid *.seed # Directory for instrumented libs generated by jscoverage/JSCover lib-cov # Coverage directory used by tools like istanbul coverage # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) .grunt # node-waf configuration .lock-wscript # Compiled binary addons (http://nodejs.org/api/addons.html) build/Release # Dependency directory # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git node_modules example/.config.js .nyc_output/ .idea ================================================ FILE: AUTHORS ================================================ # Ordered by date of first contribution. # Auto-generated by 'contributors' on Fri, 29 Sep 2017 01:42:55 GMT. # https://github.com/xingrz/node-contributors GaoXiaochen 宗羽 ================================================ FILE: CHANGELOG.md ================================================ # Changelog ## 3.12.2 (2025-12-08) * fix: make auto release work again ([5b43dda](https://github.com/ali-sdk/ali-ons/commit/5b43dda)) ## 3.12.1 (2025-12-08) * fix: retry fetching nameservers 3 times until yielding error (#118) ([bdc44eb](https://github.com/ali-sdk/ali-ons/commit/bdc44eb)), closes [#118](https://github.com/ali-sdk/ali-ons/issues/118) * chore: enable auto release (#119) ([669abdc](https://github.com/ali-sdk/ali-ons/commit/669abdc)), closes [#119](https://github.com/ali-sdk/ali-ons/issues/119) ================================================ FILE: History.md ================================================ 3.12.0 / 2023-07-31 ================== **features** * [[`32d4d09`](http://github.com/ali-sdk/ali-ons/commit/32d4d0969fb3f10f691e38a1202f06359516593f)] - feat: 支持shardingKey 客户端实现 (#109) (Orange Mi <>) 3.11.0 / 2023-01-12 ================== **fixes** * [[`15846d4`](http://github.com/ali-sdk/ali-ons/commit/15846d4176bb2e4ef7c4d7399b02affdbcb66d58)] - fix: update name server by throttle (#108) (Gao Yang <<82163514@qq.com>>) 3.10.0 / 2022-06-20 ================== **features** * [[`6bc5a62`](http://github.com/ali-sdk/ali-ons/commit/6bc5a6271afe5c274db36c961cccc36643b1ad3f)] - feat: support consumer filter message by sql92 (#105) (#106) (Hongcai Deng <>) 3.9.0 / 2021-05-08 ================== **features** * [[`483b4fc`](http://github.com/ali-sdk/ali-ons/commit/483b4fc64457554331f5c0654a4aa642ab8a0ccf)] - feat: support set unitName (#101) (zōng yǔ <>) 3.8.2 / 2021-01-28 ================== **fixes** * [[`b6ab961`](http://github.com/ali-sdk/ali-ons/commit/b6ab96159ab584ca27ab3390eb91eb79dddb95e8)] - fix: 🐛 repeat pull on same queue (#100) (Hongcai Deng <>) 3.8.1 / 2020-04-21 ================== **others** * [[`836d9e5`](http://github.com/ali-sdk/ali-ons/commit/836d9e5fae2b9cb0a79f0cc7874b26a7939af791)] - chore: add more debug log (#87) (Hongcai Deng <>) * [[`6851d8e`](http://github.com/ali-sdk/ali-ons/commit/6851d8efe0a2f91cf598440b350060abafdcbff7)] - chore: reduce error (#64) (Hongcai Deng <>) 3.8.0 / 2020-04-14 ================== **features** * [[`207280f`](http://github.com/ali-sdk/ali-ons/commit/207280f8f00f82d98a2fb5bd32a528c3fdc1c68f)] - feat: support reconsume message later (#97) (Gaara <>) 3.7.2 / 2020-03-17 ================== **fixes** * [[`637aee4`](http://github.com/ali-sdk/ali-ons/commit/637aee4424e26ffce2bb1398e2a587b0ce1d6d94)] - fix: random-publish-queue-select-start-point (#36) (Hongcai Deng <>) 3.7.1 / 2020-03-12 ================== **fixes** * [[`cb244b7`](http://github.com/ali-sdk/ali-ons/commit/cb244b7c089f1d30fa1388b4334f27742ada196a)] - fix: use co-gather instead of p-gather (#95) (Yiyu He <>) 3.7.0 / 2019-10-16 ================== **features** * [[`3889a0e`](http://github.com/ali-sdk/ali-ons/commit/3889a0e03637fbb4ce8501abee3bd45d70000e1b)] - feat: support nameAddr as an array (#91) (zōng yǔ <>) 3.6.5 / 2019-05-10 ================== **others** * [[`92892df`](http://github.com/ali-sdk/ali-ons/commit/92892df3ca1a4f33bec36ebd5fcfc8b3a3251a45)] - chore: adjust pulling log format (#82) (zōng yǔ <>) 3.6.4 / 2019-05-10 ================== **fixes** * [[`456cd1f`](http://github.com/ali-sdk/ali-ons/commit/456cd1f70d22161f51749f48737aa038e0b69f6a)] - fix: producer should format topic first (#81) (Hongcai Deng <>) 3.6.3 / 2019-05-10 ================== **fixes** * [[`d477806`](http://github.com/ali-sdk/ali-ons/commit/d47780628ef18be8fda5922c63c71b0bacc4c75c)] - fix: close socket while pulling message timeout (#80) (zōng yǔ <>) **others** * [[`d1ca295`](http://github.com/ali-sdk/ali-ons/commit/d1ca295a9a600da2cb3bee99c206aa03c92025cf)] - update RocketMQ link to https://rocketmq.apache.org (#79) (小雷 <<863837949@qq.com>>) 3.6.2 / 2019-04-16 ================== **fixes** * [[`cb385e3`](http://github.com/ali-sdk/ali-ons/commit/cb385e31c486877b625eb6b27a69245c1d43e65d)] - fix: namespace logic (#78) (zōng yǔ <>) 3.6.1 / 2019-04-15 ================== **fixes** * [[`7e5c6a4`](http://github.com/ali-sdk/ali-ons/commit/7e5c6a4ccf3f488df0f0e41a9a7bcfb0aadefb14)] - fix: not set nameSrv by default (#77) (zōng yǔ <>) 3.6.0 / 2019-04-15 ================== **features** * [[`2cd16be`](http://github.com/ali-sdk/ali-ons/commit/2cd16bed4608ca4235ad9eb5c211b9aad090a411)] - feat: support namespace (#75) (zōng yǔ <>) **fixes** * [[`0176223`](http://github.com/ali-sdk/ali-ons/commit/0176223b3fdd3a2e5a219fb02b03552764951588)] - fix: retry should process by broker (#73) (Hongcai Deng <>) **others** * [[`35f9341`](http://github.com/ali-sdk/ali-ons/commit/35f934174077e6835284ca5cbebb3e171122fd11)] - chore: update test config (#74) (zōng yǔ <>) 3.5.1 / 2019-04-12 ================== 3.5.0 / 2019-04-12 ================== **features** * [[`073058b`](http://github.com/ali-sdk/ali-ons/commit/073058b9d82415f7063758e815f0d31b2b9d5a2b)] - feat: support delay deliver message (#65) (MarvinWilliam <>) **fixes** * [[`8ef9502`](http://github.com/ali-sdk/ali-ons/commit/8ef95027eef89456b17531b89761fa375168bfd6)] - fix: drop message if reconsumeTimes > maxReconsumeTimes (#72) (zōng yǔ <>) **others** * [[`a44d4d0`](http://github.com/ali-sdk/ali-ons/commit/a44d4d05919d68200e012442e7831e5b2217f808)] - chore: upgrade deps (#71) (zōng yǔ <>) 3.4.0 / 2019-03-13 ================== **features** * [[`7dfe610`](http://github.com/ali-sdk/ali-ons/commit/7dfe6105f41d7478d587732e3ed9c14e2b96e1bb)] - feat: support set namesrv (#61) (Hongcai Deng <>) 3.3.0 / 2018-12-14 ================== **features** * [[`9acff9b`](http://github.com/ali-sdk/ali-ons/commit/9acff9b91c0325ed88ad634ae15161f62dceb574)] - feat: support consume back (#56) (Hongcai Deng <>) 3.2.2 / 2018-11-20 ================== **fixes** * [[`39e3782`](http://github.com/ali-sdk/ali-ons/commit/39e37827c8eac0c16e1fb5ecd64792da331673f3)] - fix: parse date format (#53) (Hongcai Deng <>) **others** * [[`e2dc377`](http://github.com/ali-sdk/ali-ons/commit/e2dc377babb3a420e2dfc99a33bf6118979825ff)] - test: fix ci (#54) (zōng yǔ <>) 3.2.1 / 2018-10-18 ================== **fixes** * [[`d6b6fc6`](http://github.com/ali-sdk/ali-ons/commit/d6b6fc61d049c5925c65913c299d0b423573d4b6)] - fix: accessKeyID => accessKeyId (#50) (fengmk2 <>) 3.2.0 / 2018-10-17 ================== **features** * [[`8ec58c8`](http://github.com/ali-sdk/ali-ons/commit/8ec58c8f773633a4f1bb0341306d89afda1972e5)] - feat: unify aliyun keys to accessKeyID and accessKeySecret (fengmk2 <>) **others** * [[`9ecd381`](http://github.com/ali-sdk/ali-ons/commit/9ecd381f7079d7be6f2551a6baabfc51cdefde46)] - f (fengmk2 <>) * [[`742f269`](http://github.com/ali-sdk/ali-ons/commit/742f269534c2e30f0772663ca3690ad000d9c75e)] - f (fengmk2 <>) 3.1.0 / 2018-09-14 ================== **others** * [[`d1de082`](http://github.com/ali-sdk/ali-ons/commit/d1de0823fcc860ae12133a0026ddcfe2105db82d)] - name-server-fault-tolerance (wujia <>) 3.0.0 / 2018-07-03 ================== **features** * [[`5624319`](http://github.com/ali-sdk/ali-ons/commit/56243195ad838b88932be8b5e5c1f2f6a2eb3d4f)] - feat: suppory async (ngot <>) **others** * [[`4b84644`](http://github.com/ali-sdk/ali-ons/commit/4b846446ceec7f175a363ef3dd011f67dc2dcaea)] - test: update ci command (Hongcai Deng <>) * [[`ebb7942`](http://github.com/ali-sdk/ali-ons/commit/ebb7942269662843357ce4070a19628c6d51fe88)] - test: trigger ci (Hongcai Deng <>) * [[`79b56c8`](http://github.com/ali-sdk/ali-ons/commit/79b56c86f03a957cad4442e52ddae8e49389ab57)] - breaking: async (Hongcai Deng <>) * [[`1ee4785`](http://github.com/ali-sdk/ali-ons/commit/1ee47857f4846a4d70b33b60234972651d1c37d3)] - f (ngot <>) * [[`e0a7a77`](http://github.com/ali-sdk/ali-ons/commit/e0a7a77e0830a04c020c7a2762f32a2f210ef426)] - f (ngot <>) * [[`519e164`](http://github.com/ali-sdk/ali-ons/commit/519e1641e224deec08504ccf60ca4fa05e01788b)] - f (ngot <>) * [[`1de2b1a`](http://github.com/ali-sdk/ali-ons/commit/1de2b1a6cdc46fd19faeceda50d89c2f2928144d)] - doc: updaet readme (ngot <>) 2.0.4 / 2018-01-10 ================== **fixes** * [[`6f78013`](http://github.com/ali-sdk/ali-ons/commit/6f780131b713465827588ac3ee866b9e2b2bd2ae)] - fix: fix invokeOneWay issue (gxcsoccer <>) **others** * [[`46d1bfe`](http://github.com/ali-sdk/ali-ons/commit/46d1bfe4cd5af83aa814d2a6dfd490efde5172bc)] - chore: release 2.0.3 (gxcsoccer <>), 2.0.3 / 2017-11-11 ================== * fix: memory leak may occurred cause by Promise.race * doc: fix consumer and producer initialize issue 2.0.2 / 2017-10-09 ================== * fix: persist consumer offset issue 2.0.1 / 2017-09-29 ================== * fix: support subscribe before ready 2.0.0 / 2017-09-29 ================== * feat: add local memory store * refactor: new consumer api * fix: consumer offset update issue 1.0.0 / 2017-03-14 ================== * feat: implement ali-ons base on rocketmq #1 ================================================ FILE: LICENSE ================================================ The MIT License (MIT) Copyright (c) ali-sdk Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: README.md ================================================ # ali-ons [![NPM version][npm-image]][npm-url] [![ci-actions][ci-image]][ci-url] [![npm download][download-image]][download-url] [![Node.js Version](https://img.shields.io/node/v/ali-ons.svg?style=flat)](https://nodejs.org/en/download/) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](https://makeapullrequest.com) ![CodeRabbit Pull Request Reviews](https://img.shields.io/coderabbit/prs/github/ali-sdk/ali-ons) [npm-image]: https://img.shields.io/npm/v/ali-ons.svg?style=flat-square [npm-url]: https://npmjs.org/package/ali-ons [ci-image]: https://github.com/ali-sdk/ali-ons/actions/workflows/ci-actions.yml/badge.svg [ci-url]: https://github.com/ali-sdk/ali-ons/actions/workflows/ci-actions.yml [download-image]: https://img.shields.io/npm/dm/ali-ons.svg?style=flat-square [download-url]: https://npmjs.org/package/ali-ons Aliyun Open Notification Service Client (base on opensource project [RocketMQ](https://rocketmq.apache.org/)) Sub module of [ali-sdk](https://github.com/ali-sdk/ali-sdk). ## Install ```bash npm install ali-ons --save ``` ## Usage consumer ```js 'use strict'; const httpclient = require('urllib'); const Consumer = require('ali-ons').Consumer; const consumer = new Consumer({ httpclient, accessKeyId: 'your-accessKeyId', accessKeySecret: 'your-AccessKeySecret', consumerGroup: 'your-consumer-group', // namespace: '', // aliyun namespace support // isBroadcast: true, }); consumer.subscribe(config.topic, '*', async msg => { console.log(`receive message, msgId: ${msg.msgId}, body: ${msg.body.toString()}`) // return Consumer.ACTION_RETRY; // you can return ACTION_RETRY, then this message will be directly retried }); consumer.on('error', err => console.log(err)); ``` If you want to use sql filter, you can subscribe a topic with a sql expression: ```js consumer.subscribe( config.topic, { expressionType: 'SQL92', subString: 'a is not null' }, async msg => { console.log(`receive message, msgId: ${msg.msgId}, body: ${msg.body.toString()}`) } ); ``` For more information about sql filter, see: [Filter Messages By SQL92](https://rocketmq.apache.org/rocketmq/filter-messages-by-sql92-in-rocketmq/) producer ```js 'use strict'; const httpclient = require('urllib'); const Producer = require('ali-ons').Producer; const Message = require('ali-ons').Message; const producer = new Producer({ httpclient, accessKeyId: 'your-accessKeyId', accessKeySecret: 'your-AccessKeySecret', producerGroup: 'your-producer-group', // namespace: '', // aliyun namespace support }); (async () => { const msg = new Message('your-topic', // topic 'TagA', // tag 'Hello ONS !!! ' // body ); // set Message#keys msg.keys = ['key1']; // delay consume // msg.setStartDeliverTime(Date.now() + 5000); const sendResult = await producer.send(msg); console.log(sendResult); })().catch(err => console.error(err)) ``` ## Secure Keys Please contact to @gxcsoccer to give you accessKey - [ons secure data](https://sharelock.io/1/UM02CJiYyhXiZDOn1nhX0iqPqMIQtdwI_T5BY3F-tHs.d8-ycA/01veKH9kgAuFuKCqlVPzGsyPWJ8mQLaKPJjjcB9tpdbvi9L6XQ/IgqDvAdVDMzV9lK2gQzyAj7q-CNk8-1tWrLmdqMV0oJ5qgky40/HgpZyKKDfOGAcyqQ20RUdRgCLRWqF8LUUko0uDl_L-ATNOsi5z/W2bsvBc8tAoqSwNR7u2Sqe6XkNmD98s3UQOK-6T8--VwTbHzcG/dwkHwie3EkGB-TbiMnbRh7_5A-DaOTCtALP3xvl4G0XKxuOriC/2yfuPp7WRucTAoqx2STO5Hv3MZEhh3IXf7YiOQ8pWDDqjLuQSY/_irqzYyeseY9m106ksMUq3-yS_qkBRIuoyL-hHk9ZRhGppsdA5/Dw4Pjg.fmNP3aFkLnvPuhlPRwNcng) ## License [MIT](LICENSE) ## Contributors [![Contributors](https://contrib.rocks/image?repo=ali-sdk/ali-ons)](https://github.com/ali-sdk/ali-ons/graphs/contributors) Made with [contributors-img](https://contrib.rocks). ================================================ FILE: example/config.js ================================================ 'use strict'; const env = process.env; // export ALI_SDK_ONS_ID=your-accesskey // export ALI_SDK_ONS_SECRET=your-secretkey module.exports = { accessKeyId: env.ALI_SDK_ONS_ID, accessKeySecret: env.ALI_SDK_ONS_SECRET, producerGroup: 'PID_GXCSOCCER', consumerGroup: 'GID_alions', topic: 'TP_alions_test_topic', // https://help.aliyun.com/document_detail/102895.html 阿里云产品更新，支持实例化 nameSrv: 'localhost:9876', // onsAddr: 'http://onsaddr-internet.aliyun.com/rocketmq/nsaddr4client-internet', }; ================================================ FILE: example/consumer.js ================================================ 'use strict'; const httpclient = require('urllib'); const logger = require('./logger'); const Consumer = require('../').Consumer; const config = require('./config'); const consumer = new Consumer(Object.assign(config, { httpclient, logger, maxReconsumeTimes: 1, // isBroadcast: true, // consumeFromWhere: 'CONSUME_FROM_LAST_OFFSET_AND_FROM_MIN_WHEN_BOOT_FIRST', })); consumer.subscribe(config.topic, '*', async function(msg) { console.log(`receive message, msgId: ${msg.msgId}, body: ${msg.body.toString()}`) // throw new Error(123); }); consumer.on('error', err => console.log(err)); ================================================ FILE: example/logger.js ================================================ 'use strict'; module.exports = { info() {}, warn() {}, error(...args) { console.error(...args); }, debug() {}, }; ================================================ FILE: example/producer.js ================================================ 'use strict'; const logger = require('./logger'); const config = require('./config'); const httpclient = require('urllib'); const Producer = require('../').Producer; const Message = require('../').Message; const producer = new Producer(Object.assign({ httpclient, logger }, config)); (async () => { try { await producer.ready(); const msg = new Message(config.topic, // topic 'TagA', // tag 'Hello ONS !!! ' // body ); const sendResult = await producer.send(msg); console.log(sendResult); } catch (err) { console.error(err) } })(); ================================================ FILE: lib/channel.js ================================================ 'use strict'; const is = require('is-type-of'); const crypto = require('crypto'); const Base = require('tcp-base'); const promisify = require('util').promisify; const RemotingCommand = require('./protocol/command/remoting_command'); class Channel extends Base { /** * rocketmq tcp channel object * @param {String} address - server address * @param {Object} options * - {String} accessKey * - {String} secretKey * - {String} onsChannel * @class */ constructor(address, options = {}) { // 10.18.214.201:8080 const arr = address.split(':'); // support alias: accessKeyId and accessKeySecret options.accessKey = options.accessKey || options.accessKeyId; options.secretKey = options.secretKey || options.accessKeySecret; super(Object.assign({ host: arr[0], port: arr[1], headerLength: 4, needHeartbeat: false, }, options)); this.sendPromise = promisify(this.send); } get accessKey() { return this.options.accessKey; } get secretKey() { return this.options.secretKey; } get onsChannel() { return 'ALIYUN'; } /** * Get packet length from header * @param {Buffer} header - packet header * @return {Number} bodyLength */ getBodyLength(header) { return header.readInt32BE(0); } decode(body, header) { const command = RemotingCommand.decode(Buffer.concat([ header, body ])); return { id: command.opaque, isResponse: command.isResponseType, data: command, }; } beforeRequest(command) { if (!this.accessKey || !this.secretKey) { return; } const header = command.customHeader; const map = new Map(); map.set('AccessKey', this.accessKey); map.set('OnsChannel', this.onsChannel); if (header) { for (const field in header) { if (!is.nullOrUndefined(header[field])) { map.set(field, header[field].toString()); } } } let val = ''; const fields = Array.from(map.keys()).sort(); for (const key of fields) { if (key !== 'Signature') { val += map.get(key); } } let total = Buffer.from(val, 'utf8'); const bodyLength = command.body ? command.body.length : 0; if (bodyLength) { total = Buffer.concat([ total, command.body ], total.length + bodyLength); } const hmac = crypto.createHmac('sha1', this.secretKey); const signature = hmac.update(total).digest('base64'); command.extFields.Signature = signature; command.extFields.AccessKey = this.accessKey; command.extFields.OnsChannel = this.onsChannel; } /** * invoke rocketmq api * @param {RemotingCommand} command - remoting command * @param {Number} timeout - response timeout * @return {Promise} response */ invoke(command, timeout) { this.beforeRequest(command); return this.sendPromise({ id: command.opaque, data: command.encode(), timeout, }).catch(err => { // TODO: not sure whether is work ? if (err.name === 'ResponseTimeoutError') { this.close(); } throw err; }); } /** * invoke rocketmq api without need response * @param {RemotingCommand} command - remoting command * @return {Promise} Promise */ invokeOneway(command) { this.beforeRequest(command); return this.sendPromise({ id: command.opaque, data: command.encode(), oneway: true, }); } } module.exports = Channel; ================================================ FILE: lib/client_config.js ================================================ 'use strict'; const Base = require('sdk-base'); const address = require('address'); const MixAll = require('./mix_all'); const defaultOptions = { instanceName: 'DEFAULT', pollNameServerInteval: 30 * 1000, heartbeatBrokerInterval: 30 * 1000, persistConsumerOffsetInterval: 5 * 1000, rebalanceInterval: 10 * 1000, clientIP: address.ip(), unitMode: false, unitName: null, // 阿里云自创建实例，需要 ns 前缀 namespace: '', // 公有云生产环境：http://onsaddr-internal.aliyun.com:8080/rocketmq/nsaddr4client-internal // 公有云公测环境：http://onsaddr-internet.aliyun.com/rocketmq/nsaddr4client-internet // 杭州金融云环境：http://jbponsaddr-internal.aliyun.com:8080/rocketmq/nsaddr4client-internal // 杭州深圳云环境：http://mq4finance-sz.addr.aliyun.com:8080/rocketmq/nsaddr4client-internal onsAddr: 'http://onsaddr-internet.aliyun.com/rocketmq/nsaddr4client-internet', // https://help.aliyun.com/document_detail/102895.html 阿里云产品更新，支持实例化 // nameSrv: 'onsaddr.mq-internet-access.mq-internet.aliyuncs.com:80', onsChannel: 'ALIYUN', // CLOUD, ALIYUN, ALL }; class ClientConfig extends Base { /** * Producer 与 Consumer的公共配置 * @param {Object} options * - {String} instanceName 示例名称 * - {Number} pollNameServerInteval name server 同步的间隔 * - {Number} heartbeatBrokerInterval 心跳的间隔 * - {Number} persistConsumerOffsetInterval 持久化消费进度的间隔 * - {Boolean} unitMode 是否为单元化的订阅组 */ constructor(options) { super(Object.assign({}, defaultOptions, options)); this.instanceName = this.options.instanceName; } get clientId() { return this.unitName ? `${this.options.clientIP}@${this.instanceName}@${this.unitName}` : `${this.options.clientIP}@${this.instanceName}`; } get pollNameServerInteval() { return this.options.pollNameServerInteval; } get heartbeatBrokerInterval() { return this.options.heartbeatBrokerInterval; } get persistConsumerOffsetInterval() { return this.options.persistConsumerOffsetInterval; } get rebalanceInterval() { return this.options.rebalanceInterval; } get unitMode() { return this.options.unitMode; } get unitName() { return this.options.unitName; } get namespace() { return this.options.namespace; } formatTopic(topic) { if (this.namespace && (!topic.startsWith(this.namespace) && !topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX))) { topic = `${this.namespace}%${topic}`; } return topic; } /** * 将实例名修改为进程号 */ changeInstanceNameToPID() { if (this.instanceName === 'DEFAULT') { this.instanceName = process.pid + ''; } } } module.exports = ClientConfig; ================================================ FILE: lib/consumer/consume_from_where.js ================================================ 'use strict'; module.exports = { /** * 一个新的订阅组第一次启动从队列的最后位置开始消费
* 后续再启动接着上次消费的进度开始消费 */ CONSUME_FROM_LAST_OFFSET: 'CONSUME_FROM_LAST_OFFSET', CONSUME_FROM_LAST_OFFSET_AND_FROM_MIN_WHEN_BOOT_FIRST: 'CONSUME_FROM_LAST_OFFSET_AND_FROM_MIN_WHEN_BOOT_FIRST', CONSUME_FROM_MIN_OFFSET: 'CONSUME_FROM_MIN_OFFSET', CONSUME_FROM_MAX_OFFSET: 'CONSUME_FROM_MAX_OFFSET', /** * 一个新的订阅组第一次启动从队列的最前位置开始消费
* 后续再启动接着上次消费的进度开始消费 */ CONSUME_FROM_FIRST_OFFSET: 'CONSUME_FROM_FIRST_OFFSET', /** * 一个新的订阅组第一次启动从指定时间点开始消费
* 后续再启动接着上次消费的进度开始消费
* 时间点设置参见DefaultMQPushConsumer.consumeTimestamp参数 */ CONSUME_FROM_TIMESTAMP: 'CONSUME_FROM_TIMESTAMP', }; ================================================ FILE: lib/consumer/mq_push_consumer.js ================================================ 'use strict'; const assert = require('assert'); const is = require('is-type-of'); const utils = require('../utils'); const logger = require('../logger'); const MixAll = require('../mix_all'); const MQClient = require('../mq_client'); const MQProducer = require('../producer/mq_producer'); const sleep = require('mz-modules/sleep'); const ClientConfig = require('../client_config'); const ProcessQueue = require('../process_queue'); const PullStatus = require('../consumer/pull_status'); const PullSysFlag = require('../utils/pull_sys_flag'); const ConsumeFromWhere = require('./consume_from_where'); const MessageModel = require('../protocol/message_model'); const ConsumeType = require('../protocol/consume_type'); const ReadOffsetType = require('../store/read_offset_type'); const LocalFileOffsetStore = require('../store/local_file'); const LocalMemoryOffsetStore = require('../store/local_memory'); const RemoteBrokerOffsetStore = require('../store/remote_broker'); const AllocateMessageQueueAveragely = require('./rebalance/allocate_message_queue_averagely'); const Message = require('../message/message'); const MessageConst = require('../message/message_const'); const defaultOptions = { logger, persistent: false, // 是否持久化消费进度 isBroadcast: false, // 是否是广播模式（默认集群消费模式） brokerSuspendMaxTimeMillis: 1000 * 15, // 长轮询模式，Consumer连接在Broker挂起最长时间 pullTimeDelayMillsWhenException: 3000, // 拉消息异常时，延迟一段时间再拉 pullTimeDelayMillsWhenFlowControl: 5000, // 进入流控逻辑，延迟一段时间再拉 consumerTimeoutMillisWhenSuspend: 1000 * 30, // 长轮询模式，Consumer超时时间（必须要大于brokerSuspendMaxTimeMillis） consumerGroup: MixAll.DEFAULT_CONSUMER_GROUP, consumeFromWhere: ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET, // Consumer第一次启动时，从哪里开始消费 /** * Consumer第一次启动时，如果回溯消费，默认回溯到哪个时间点，数据格式如下，时间精度秒： * 20131223171201 * 表示2013年12月23日17点12分01秒 * 默认回溯到相对启动时间的半小时前 */ consumeTimestamp: utils.timeMillisToHumanString(Date.now() - 1000 * 60 * 30), pullThresholdForQueue: 500, // 本地队列消息数超过此阀值，开始流控 pullInterval: 0, // 拉取消息的频率, 如果为了降低拉取速度，可以设置大于0的值 consumeMessageBatchMaxSize: 1, // 消费一批消息，最大数 pullBatchSize: 32, // 拉消息，一次拉多少条 parallelConsumeLimit: 1, // 并发消费消息限制 postSubscriptionWhenPull: true, // 是否每次拉消息时，都上传订阅关系 allocateMessageQueueStrategy: new AllocateMessageQueueAveragely(), // 队列分配算法，应用可重写 maxReconsumeTimes: 16, // 最大重试次数 }; class MQPushConsumer extends ClientConfig { constructor(options) { assert(options && options.consumerGroup, '[MQPushConsumer] options.consumerGroup is required'); const mergedOptions = Object.assign({ initMethod: 'init' }, defaultOptions, options); assert(mergedOptions.parallelConsumeLimit <= mergedOptions.pullBatchSize, '[MQPushConsumer] options.parallelConsumeLimit must lte pullBatchSize'); super(mergedOptions); // @example: // pullFromWhichNodeTable => { // '[topic="TEST_TOPIC", brokerName="qdinternet-03", queueId="1"]': 0 // } this._pullFromWhichNodeTable = new Map(); this._subscriptions = new Map(); this._handles = new Map(); this._topicSubscribeInfoTable = new Map(); this._processQueueTable = new Map(); this._inited = false; this._isClosed = false; if (this.messageModel === MessageModel.CLUSTERING) { this.changeInstanceNameToPID(); } this._mqClient = MQClient.getAndCreateMQClient(this); this._offsetStore = this.newOffsetStoreInstance(); this._mqClient.on('error', err => this._handleError(err)); this._offsetStore.on('error', err => this._handleError(err)); } get logger() { return this.options.logger; } get subscriptions() { return this._subscriptions; } get processQueueTable() { return this._processQueueTable; } get parallelConsumeLimit() { return this.options.parallelConsumeLimit; } get consumerGroup() { if (this.namespace) { return `${this.namespace}%${this.options.consumerGroup}`; } return this.options.consumerGroup; } get messageModel() { return this.options.isBroadcast ? MessageModel.BROADCASTING : MessageModel.CLUSTERING; } get consumeType() { return ConsumeType.CONSUME_PASSIVELY; } get consumeFromWhere() { return this.options.consumeFromWhere; } get allocateMessageQueueStrategy() { return this.options.allocateMessageQueueStrategy; } async init() { this._mqClient.registerConsumer(this.consumerGroup, this); await MQProducer.getDefaultProducer(this.options); await this._mqClient.ready(); await this._offsetStore.load(); this.logger.info('[mq:consumer] consumer started'); this._inited = true; // 订阅重试 TOPIC if (this.messageModel === MessageModel.CLUSTERING) { const retryTopic = MixAll.getRetryTopic(this.consumerGroup); this.subscribe(retryTopic, '*', async msg => { const originTopic = msg.retryTopic; const originMsgId = msg.originMessageId; const subscription = this._subscriptions.get(originTopic) || {}; const handler = subscription.handler; if (!MixAll.isRetryTopic(originTopic) && handler) { await handler(msg); } else { this.logger.warn('[MQPushConsumer] retry message no handler, originTopic: %s, originMsgId: %s, msgId: %s', originTopic, originMsgId, msg.msgId); } }); } } /** * close the consumer */ async close() { this._isClosed = true; await this.persistConsumerOffset(); this._pullFromWhichNodeTable.clear(); this._subscriptions.clear(); this._topicSubscribeInfoTable.clear(); this._processQueueTable.clear(); await this._mqClient.unregisterConsumer(this.consumerGroup); await this._mqClient.close(); this.logger.info('[mq:consumer] consumer closed'); this.emit('close'); } newOffsetStoreInstance() { if (this.messageModel === MessageModel.BROADCASTING) { if (this.options.persistent) { return new LocalFileOffsetStore(this._mqClient, this.consumerGroup); } return new LocalMemoryOffsetStore(this._mqClient, this.consumerGroup); } return new RemoteBrokerOffsetStore(this._mqClient, this.consumerGroup); } /** * subscribe * @param {String} topic - topic * @param {String} subExpression - tag * @param {Function} handler - message handler * @return {void} */ subscribe(topic, subExpression, handler) { // 添加 namespace 前缀 topic = this.formatTopic(topic); if (arguments.length === 2) { handler = subExpression; subExpression = null; } assert(is.asyncFunction(handler), '[MQPushConsumer] handler should be a asyncFunction'); assert(!this.subscriptions.has(topic), `[MQPushConsumer] ONLY one handler allowed for topic=${topic}`); const subscriptionData = this.buildSubscriptionData(this.consumerGroup, topic, subExpression); const tagsSet = subscriptionData.tagsSet; const needFilter = !!tagsSet.length; this.subscriptions.set(topic, { handler, subscriptionData, }); (async () => { try { await this.ready(); // 如果 topic 没有路由信息，先更新一下 if (!this._topicSubscribeInfoTable.has(topic)) { await this._mqClient.updateAllTopicRouterInfo(); await this._mqClient.sendHeartbeatToAllBroker(); await this._mqClient.doRebalance(); } // 消息消费循环 while (!this._isClosed && this.subscriptions.has(topic)) { await this._consumeMessageLoop(topic, needFilter, tagsSet, subExpression); } } catch (err) { this._handleError(err); } })(); this.logger.info('[MQPushConsumer] cancel subscribe for topic=%s, subExpression=%s', topic, subExpression); } async _consumeMessageLoop(topic, needFilter, tagsSet, subExpression) { const mqList = this._topicSubscribeInfoTable.get(topic); let hasMsg = false; if (mqList && mqList.length) { for (const mq of mqList) { const item = this._processQueueTable.get(mq.key); if (item) { const pq = item.processQueue; this.logger.debug('[MQPushConsumer] process msg for processQueue=%s, msgCount=%', mq.key, pq.msgCount); while (pq.msgCount) { hasMsg = true; let msgs; if (this.parallelConsumeLimit > pq.msgCount) { msgs = pq.msgList.slice(0, pq.msgCount); } else { msgs = pq.msgList.slice(0, this.parallelConsumeLimit); } // 并发消费任务 const consumeTasks = []; for (const msg of msgs) { const handler = this._subscriptions.get(msg.topic).handler; if (!msg.tags || !needFilter || tagsSet.includes(msg.tags)) { consumeTasks.push(this.consumeSingleMsg(handler, msg, mq, pq)); } else { this.logger.debug('[MQPushConsumer] message filter by tags=, msg.tags=%s', subExpression, msg.tags); } } // 必须全部成功 try { await Promise.all(consumeTasks); } catch (err) { continue; } // 注意这里必须是批量确认 const offset = pq.remove(msgs.length); if (offset >= 0) { this._offsetStore.updateOffset(mq, offset, true); } } } } } if (!hasMsg) { const changedEvent = `topic_${topic}_changed`; this.logger.debug(`[MQPushConsumer] waiting event for ${changedEvent}`); await this.await(changedEvent); } } async consumeSingleMsg(handler, msg, mq, pq) { // 集群消费模式下，如果消费失败，反复重试 while (!this._isClosed) { if (msg.reconsumeTimes > this.options.maxReconsumeTimes) { this.logger.warn('[MQPushConsumer] consume message failed, drop it for reconsumeTimes=%d and maxReconsumeTimes=%d, msgId: %s, originMsgId: %s', msg.reconsumeTimes, this.options.maxReconsumeTimes, msg.msgId, msg.originMessageId); return; } utils.resetRetryTopic(msg, this.consumerGroup); try { const value = await handler(msg, mq, pq); if (value !== MQPushConsumer.ACTION_RETRY) { return; } } catch (err) { err.message = `process mq message failed, topic: ${msg.topic}, msgId: ${msg.msgId}, ${err.message}`; this.emit('error', err); } if (this.messageModel === MessageModel.CLUSTERING) { // 发送重试消息 try { // delayLevel 为 0 代表由服务端控制重试间隔 await this.sendMessageBack(msg, 0, mq.brokerName, this.consumerGroup); return; } catch (err) { this.emit('error', err); this.logger.error( '[MQPushConsumer] send reconsume message failed, fallback to local retry, msgId: %s', msg.msgId ); // 重试消息发送失败，本地重试 await this._sleep(5000); } // 本地重试情况下需要给 reconsumeTimes +1 msg.reconsumeTimes++; } else { this.logger.warn('[MQPushConsumer] BROADCASTING consume message failed, drop it, msgId: %s', msg.msgId); return; } } this.logger.info('[MQPushConsumer] consumer is closed, skip consume msg, msgId=%s', msg.msgId); } /** * construct subscription data * @param {String} consumerGroup - consumer group name * @param {String} topic - topic * @param {String} subString - tag * @return {Object} subscription */ buildSubscriptionData(consumerGroup, topic, subString) { const subscriptionData = { topic, subString, classFilterMode: false, tagsSet: [], codeSet: [], subVersion: Date.now(), }; if (is.nullOrUndefined(subString) || subString === '*' || subString === '') { subscriptionData.subString = '*'; } else if (typeof subString === 'object') { subscriptionData.expressionType = subString.expressionType; subscriptionData.subString = subString.subString; } else { const tags = subString.split('||'); for (let tag of tags) { tag = tag.trim(); if (tag) { subscriptionData.tagsSet.push(tag); subscriptionData.codeSet.push(utils.hashCode(tag)); } } } return subscriptionData; } async persistConsumerOffset() { const mqs = []; for (const key of this._processQueueTable.keys()) { if (this._processQueueTable.get(key)) { mqs.push(this._processQueueTable.get(key).messageQueue); } } await this._offsetStore.persistAll(mqs); } /** * pull message from queue * @param {MessageQueue} messageQueue - message queue * @param {ProcessQueue} processQueue - process queue * @return {void} */ pullMessageQueue(messageQueue, processQueue) { const _self = this; (async () => { while (!_self._isClosed) { const pullRequest = _self._processQueueTable.get(messageQueue.key); if (!pullRequest) { break; } if (pullRequest.processQueue !== processQueue) { // 不是同一个引用，退出循环，避免重复监听 break; } try { await _self.executePullRequestImmediately(messageQueue); await _self._sleep(_self.options.pullInterval); } catch (err) { if (!_self._isClosed) { err.name = 'MQConsumerPullMessageError'; err.message = `[mq:consumer] pull message for queue: ${messageQueue.key}, occurred error: ${err.message}`; _self._handleError(err); await _self._sleep(_self.options.pullTimeDelayMillsWhenException); } } } _self.logger.info('[MQPushConsumer] stop pulling message from queue: %s', messageQueue.key); })(); } /** * execute pull message immediately * @param {MessageQueue} messageQueue - messageQueue * @return {Promise} */ async executePullRequestImmediately(messageQueue) { // close or queue removed if (!this._processQueueTable.has(messageQueue.key)) { return; } const pullRequest = this._processQueueTable.get(messageQueue.key); const processQueue = pullRequest.processQueue; // queue droped if (processQueue.droped) { return; } // flow control const size = processQueue.msgCount; if (size > this.options.pullThresholdForQueue) { await this._sleep(this.options.pullTimeDelayMillsWhenFlowControl); return; } processQueue.lastPullTimestamp = Date.now(); const data = this.subscriptions.get(messageQueue.topic); const subscriptionData = data && data.subscriptionData; if (!subscriptionData) { this.logger.warn('[mq:consumer] execute pull request, but subscriptionData not found, topic: %s, queueId: %s', messageQueue.topic, messageQueue.queueId); await this._sleep(this.options.pullTimeDelayMillsWhenException); return; } let commitOffset = 0; const subExpression = this.options.postSubscriptionWhenPull ? subscriptionData.subString : null; const expressionType = subscriptionData.expressionType; const subVersion = subscriptionData.subVersion; // cluster model if (MessageModel.CLUSTERING === this.messageModel) { const offset = await this._offsetStore.readOffset(pullRequest.messageQueue, ReadOffsetType.READ_FROM_MEMORY); if (offset) { commitOffset = offset; } } this.logger.info('[MQPushConsumer] start to pull message from queue: %s, nextOffset: %s, commitOffset: %s, subExpression: %s, expressionType: %s, subVersion: %s', messageQueue.key, pullRequest.nextOffset, commitOffset, subExpression, expressionType, subVersion); const pullResult = await this.pullKernelImpl(messageQueue, subExpression, expressionType, subVersion, pullRequest.nextOffset, commitOffset); this.updatePullFromWhichNode(messageQueue, pullResult.suggestWhichBrokerId); const originOffset = pullRequest.nextOffset; // update next pull offset pullRequest.nextOffset = pullResult.nextBeginOffset; this.logger.info('[MQPushConsumer] pull message result: %s from queue: %s, requestOffset: %s, nextBeginOffset: %s, minOffset: %s, maxOffset: %s, suggestWhichBrokerId: %s', pullResult.pullStatus, messageQueue.key, originOffset, pullResult.nextBeginOffset, pullResult.minOffset, pullResult.maxOffset, pullResult.suggestWhichBrokerId); switch (pullResult.pullStatus) { case PullStatus.FOUND: { const pullRT = Date.now() - processQueue.lastPullTimestamp; const msgIds = pullResult.msgFoundList.map(v => v.msgId); this.logger.info('[MQPushConsumer] pull message success, found new message size: %d, topic: %s, msgId: [%s], consumerGroup: %s, messageQueue: %s, cost: %dms.', pullResult.msgFoundList.length, messageQueue.topic, msgIds.join(','), this.consumerGroup, messageQueue.key, pullRT); // submit to consumer processQueue.putMessage(pullResult.msgFoundList); this.emit(`topic_${messageQueue.topic}_changed`); break; } case PullStatus.NO_NEW_MSG: case PullStatus.NO_MATCHED_MSG: this.logger.debug('[mq:consumer] no new message for topic: %s at message queue => %s', subscriptionData.topic, messageQueue.key); this.correctTagsOffset(pullRequest); break; case PullStatus.OFFSET_ILLEGAL: this.logger.warn('[mq:consumer] the pull request offset illegal, message queue => %s, the originOffset => %d, pullResult => %j', messageQueue.key, originOffset, pullResult); this._offsetStore.updateOffset(messageQueue, pullRequest.nextOffset); break; default: break; } } async pullKernelImpl(messageQueue, subExpression, expressionType, subVersion, offset, commitOffset) { let sysFlag = PullSysFlag.buildSysFlag( // commitOffset > 0, // commitOffset true, // suspend !!subExpression, // subscription false // class filter ); const result = await this.findBrokerAddress(messageQueue); if (!result) { throw new Error(`The broker[${messageQueue.brokerName}] not exist`); } // Slave不允许实时提交消费进度，可以定时提交 if (result.slave) { sysFlag = PullSysFlag.clearCommitOffsetFlag(sysFlag); } const requestHeader = { consumerGroup: this.consumerGroup, topic: messageQueue.topic, queueId: messageQueue.queueId, queueOffset: offset, maxMsgNums: this.options.pullBatchSize, sysFlag, commitOffset, suspendTimeoutMillis: this.options.brokerSuspendMaxTimeMillis, subscription: subExpression, subVersion, }; // 拉取消息时需要设置表达式类型，未设置默认为 tag if (expressionType) { requestHeader.expressionType = expressionType; } return await this._mqClient.pullMessage(result.brokerAddr, requestHeader, this.options.consumerTimeoutMillisWhenSuspend); } async findBrokerAddress(messageQueue) { let findBrokerResult = this._mqClient.findBrokerAddressInSubscribe( messageQueue.brokerName, this.recalculatePullFromWhichNode(messageQueue), false); if (!findBrokerResult) { await this._mqClient.updateTopicRouteInfoFromNameServer(messageQueue.topic); findBrokerResult = this._mqClient.findBrokerAddressInSubscribe( messageQueue.brokerName, this.recalculatePullFromWhichNode(messageQueue), false); } return findBrokerResult; } recalculatePullFromWhichNode(messageQueue) { // @example: // pullFromWhichNodeTable => { // '[topic="TEST_TOPIC", brokerName="qdinternet-03", queueId="1"]': 0 // } return this._pullFromWhichNodeTable.get(messageQueue.key) || MixAll.MASTER_ID; } correctTagsOffset(pullRequest) { // 仅当已拉下的消息消费完的情况下才更新 offset if (pullRequest.processQueue.msgCount === 0) { this._offsetStore.updateOffset(pullRequest.messageQueue, pullRequest.nextOffset, true); } } updatePullFromWhichNode(messageQueue, brokerId) { this._pullFromWhichNodeTable.set(messageQueue.key, brokerId); } /** * update subscription data * @param {String} topic - topic * @param {Array} info - info * @return {void} */ updateTopicSubscribeInfo(topic, info) { if (this._subscriptions.has(topic)) { this._topicSubscribeInfoTable.set(topic, info); } } /** * whether need update * @param {String} topic - topic * @return {Boolean} need update? */ isSubscribeTopicNeedUpdate(topic) { if (this._subscriptions && this._subscriptions.has(topic)) { return !this._topicSubscribeInfoTable.has(topic); } return false; } /** * rebalance * @return {void} */ async doRebalance() { for (const topic of this.subscriptions.keys()) { await this.rebalanceByTopic(topic); } } async rebalanceByTopic(topic) { this.logger.info('[mq:consumer] rebalanceByTopic: %s, messageModel: %s', topic, this.messageModel); const mqSet = this._topicSubscribeInfoTable.get(topic); // messageQueue list if (!mqSet || !mqSet.length) { this.logger.warn('[mq:consumer] doRebalance, %s, but the topic[%s] not exist.', this.consumerGroup, topic); return; } let changed; let allocateResult = mqSet; if (this.options.isBroadcast) { changed = await this.updateProcessQueueTableInRebalance(topic, mqSet); } else { const cidAll = await this._mqClient.findConsumerIdList(topic, this.consumerGroup); this.logger.info('[mq:consumer] rebalance topic: %s, with consumer ids: %j', topic, cidAll); if (cidAll && cidAll.length) { // 排序 mqSet.sort(compare); cidAll.sort(); allocateResult = this.allocateMessageQueueStrategy.allocate(this.consumerGroup, this._mqClient.clientId, mqSet, cidAll); this.logger.info('[mq:consumer] allocate queue for group: %s, clientId: %s, result: %j', this.consumerGroup, this._mqClient.clientId, allocateResult); changed = await this.updateProcessQueueTableInRebalance(topic, allocateResult); } } if (changed) { this.logger.info('[mq:consumer] do rebalance and message queue changed, topic: %s, mqSet: %j', topic, allocateResult); this.emit(`topic_${topic}_queue_changed`); } } /** * update process queue * @param {String} topic - topic * @param {Array} mqSet - message queue set * @return {Promise} */ async updateProcessQueueTableInRebalance(topic, mqSet) { let changed = false; // delete unnecessary queue for (const key of this._processQueueTable.keys()) { const obj = this._processQueueTable.get(key); const messageQueue = obj.messageQueue; const processQueue = obj.processQueue; if (topic === messageQueue.topic) { // not found in mqSet, that means the process queue is unnecessary. if (!mqSet.some(mq => mq.key === messageQueue.key)) { processQueue.droped = true; await this.removeProcessQueue(messageQueue); changed = true; } else if (processQueue.isPullExpired && this.consumeType === ConsumeType.CONSUME_PASSIVELY) { processQueue.droped = true; await this.removeProcessQueue(messageQueue); changed = true; this.logger.warn('[MQPushConsumer] BUG doRebalance, %s, remove unnecessary mq=%s, because pull is pause, so try to fixed it', this.consumerGroup, messageQueue.key); } } } for (const messageQueue of mqSet) { if (this._processQueueTable.has(messageQueue.key)) { continue; } const nextOffset = await this.computePullFromWhere(messageQueue); // double check if (this._processQueueTable.has(messageQueue.key)) { continue; } if (nextOffset >= 0) { const processQueue = new ProcessQueue(); changed = true; this._processQueueTable.set(messageQueue.key, { messageQueue, processQueue, nextOffset, }); // start to pull this queue; this.pullMessageQueue(messageQueue, processQueue); this.logger.info('[mq:consumer] doRebalance, %s, add a new messageQueue, %j, its nextOffset: %s', this.consumerGroup, messageQueue, nextOffset); } else { this.logger.warn('[mq:consumer] doRebalance, %s, new messageQueue, %j, has invalid nextOffset: %s', this.consumerGroup, messageQueue, nextOffset); } } return changed; } /** * compute consume offset * @param {MessageQueue} messageQueue - message queue * @return {Promise} offset */ async computePullFromWhere(messageQueue) { try { const lastOffset = await this._offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_STORE); this.logger.info('[mq:consumer] read lastOffset => %s from store, topic="%s", brokerName="%s", queueId="%s"', lastOffset, messageQueue.topic, messageQueue.brokerName, messageQueue.queueId); let result = -1; switch (this.consumeFromWhere) { case ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET_AND_FROM_MIN_WHEN_BOOT_FIRST: case ConsumeFromWhere.CONSUME_FROM_MIN_OFFSET: case ConsumeFromWhere.CONSUME_FROM_MAX_OFFSET: case ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET: // 第二次启动，根据上次的消费位点开始消费 if (lastOffset >= 0) { result = lastOffset; } else if (lastOffset === -1) { // 第一次启动，没有记录消费位点 // 重试队列则从队列头部开始 if (messageQueue.topic.indexOf(MixAll.RETRY_GROUP_TOPIC_PREFIX) === 0) { result = 0; } else { // 正常队列则从队列尾部开始 return await this._mqClient.maxOffset(messageQueue); } } break; case ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET: // 第二次启动，根据上次的消费位点开始消费 if (lastOffset >= 0) { result = lastOffset; } else { result = 0; } break; case ConsumeFromWhere.CONSUME_FROM_TIMESTAMP: // 第二次启动，根据上次的消费位点开始消费 if (lastOffset >= 0) { result = lastOffset; } else if (lastOffset === -1) { // 第一次启动，没有记录消费为点 // 重试队列则从队列尾部开始 if (messageQueue.topic.indexOf(MixAll.RETRY_GROUP_TOPIC_PREFIX) === 0) { return await this._mqClient.maxOffset(messageQueue); } // 正常队列则从指定时间点开始 // 时间点需要参数配置 const timestamp = utils.parseDate(this.options.consumeTimestamp).getTime(); return await this._mqClient.searchOffset(messageQueue, timestamp); } break; default: break; } this.logger.info('[mq:consumer] computePullFromWhere() messageQueue => %s should read from offset: %s and lastOffset: %s', messageQueue.key, result, lastOffset); return result; } catch (err) { err.mesasge = 'computePullFromWhere() occurred an exception, ' + err.mesasge; this._handleError(err); return -1; } } /** * 移除消费队列 * @param {MessageQueue} messageQueue - message queue * @return {Promise} */ async removeProcessQueue(messageQueue) { const processQueue = this._processQueueTable.get(messageQueue.key); this._processQueueTable.delete(messageQueue.key); if (processQueue) { processQueue.droped = true; await this.removeUnnecessaryMessageQueue(messageQueue, processQueue); this.logger.info('[mq:consumer] remove unnecessary messageQueue, %s, Droped: %s', messageQueue.key, processQueue.droped); } } /** * remove unnecessary queue * @param {MessageQueue} messageQueue - message queue * @return {Promise} */ async removeUnnecessaryMessageQueue(messageQueue) { await this._offsetStore.persist(messageQueue); this._offsetStore.removeOffset(messageQueue); // todo: consume later ？ } async sendMessageBack(msg, delayLevel, brokerName, consumerGroup) { const brokerAddr = brokerName ? this._mqClient.findBrokerAddressInPublish(brokerName) : msg.storeHost; const thatConsumerGroup = consumerGroup ? consumerGroup : this.consumerGroup; try { await this._mqClient.consumerSendMessageBack( brokerAddr, msg, thatConsumerGroup, delayLevel, 3000, this.options.maxReconsumeTimes); this.logger.info('[MQPushConsumer] consumerSendMessageBack success, topic=%s, consumerGroup=%s, msgId=%s', msg.topic, thatConsumerGroup, msg.msgId); } catch (err) { err.mesasge = 'sendMessageBack() occurred an exception, ' + thatConsumerGroup + ', ' + err.mesasge; this._handleError(err); let newMsg; if (MixAll.isRetryTopic(msg.topic)) { newMsg = msg; } else { newMsg = new Message(MixAll.getRetryTopic(thatConsumerGroup), '', msg.body); newMsg.flag = msg.flag; newMsg.properties = msg.properties; newMsg.originMessageId = msg.originMessageId || msg.msgId; newMsg.retryTopic = msg.topic; // 这里需要加 1，因为如果 maxReconsumeTimes 为 1，那么这条 retry 消息发出去始终不会被重新投递了 newMsg.properties[MessageConst.PROPERTY_MAX_RECONSUME_TIMES] = String(this.options.maxReconsumeTimes + 1); } newMsg.properties[MessageConst.PROPERTY_RECONSUME_TIME] = String(msg.reconsumeTimes + 1); newMsg.delayTimeLevel = 0; await (await MQProducer.getDefaultProducer()).send(newMsg); } } // * viewMessage(msgId) { // const info = MessageDecoder.decodeMessageId(msgId); // return yield this._mqClient.viewMessage(info.address, Number(info.offset.toString()), 3000); // } _handleError(err) { err.message = 'MQPushConsumer occurred an error ' + err.message; this.emit('error', err); } _sleep(timeout) { return sleep(timeout); } } // if subscriber return ACTION_RETRY, message will be directly retried MQPushConsumer.ACTION_RETRY = Symbol('ACTION_RETRY'); module.exports = MQPushConsumer; // Helper // ------------------ function compare(mqA, mqB) { if (mqA.topic === mqB.topic) { if (mqA.brokerName === mqB.brokerName) { return mqA.queueId - mqB.queueId; } return mqA.brokerName > mqB.brokerName ? 1 : -1; } return mqA.topic > mqB.topic ? 1 : -1; } ================================================ FILE: lib/consumer/pull_status.js ================================================ 'use strict'; module.exports = { /** * 找到消息 */ FOUND: 'FOUND', /** * 没有新的消息可以被拉取 */ NO_NEW_MSG: 'NO_NEW_MSG', /** * 经过过滤后，没有匹配的消息 */ NO_MATCHED_MSG: 'NO_MATCHED_MSG', /** * Offset不合法，可能过大或者过小 */ OFFSET_ILLEGAL: 'OFFSET_ILLEGAL', }; ================================================ FILE: lib/consumer/rebalance/allocate_message_queue_averagely.js ================================================ 'use strict'; const debug = require('debug')('mq:allocate'); const AllocateMessageQueueStrategy = require('./allocate_message_queue_strategy'); class AllocateMessageQueueAveragely extends AllocateMessageQueueStrategy { get name() { return 'AVG'; } allocate(consumerGroup, currentCID, mqAll, cidAll) { if (!currentCID || currentCID.length < 1) { throw new Error('currentCID is empty'); } if (!mqAll || !mqAll.length) { throw new Error('mqAll is null or mqAll empty'); } if (!cidAll || !cidAll.length) { throw new Error('cidAll is null or cidAll empty'); } const result = []; const index = cidAll.indexOf(currentCID); if (index === -1) { // 不存在此ConsumerId ,直接返回 debug('[BUG] ConsumerGroup: %s The consumerId: %s not in cidAll: %j', // consumerGroup, // currentCID, // cidAll); return result; } const mqLen = mqAll.length; const cidLen = cidAll.length; const mod = mqLen % cidLen; let averageSize = mqLen <= cidLen ? 1 : mod > 0 && index < mod ? mqLen / cidLen + 1 : mqLen / cidLen; averageSize = Math.floor(averageSize); // 取整 const startIndex = mod > 0 && index < mod ? index * averageSize : index * averageSize + mod; const range = Math.min(averageSize, mqLen - startIndex); for (let i = 0; i < range; i++) { result.push(mqAll[(startIndex + i) % mqLen]); } return result; } } module.exports = AllocateMessageQueueAveragely; ================================================ FILE: lib/consumer/rebalance/allocate_message_queue_strategy.js ================================================ /* istanbul ignore next */ /* eslint valid-jsdoc:0 */ 'use strict'; class AllocateMessageQueueStrategy { /** * 给当前的 ConsumerId 分配队列 * @param {String} consumerGroup - * @param {String} currentCID - 当前 ConsumerId * @param {Array} mqAll - 当前 Topic 的所有队列集合，无重复数据，且有序 * @param {Array} cidAll - 当前订阅组的所有 Consumer 集合，无重复数据，且有序 */ allocate(consumerGroup, currentCID, mqAll, cidAll) { /* eslint no-unused-vars: 0 */ throw new Error('no implementation'); } /** * rebalance 算法的名字 */ get name() { throw new Error('no implementation'); } } module.exports = AllocateMessageQueueStrategy; ================================================ FILE: lib/index.js ================================================ 'use strict'; exports.Message = require('./message/message'); exports.Producer = require('./producer/mq_producer'); exports.Consumer = require('./consumer/mq_push_consumer'); ================================================ FILE: lib/logger.js ================================================ 'use strict'; module.exports = { info() {}, warn(...args) { console.warn(...args); }, error(...args) { console.error(...args); }, debug() {}, }; ================================================ FILE: lib/message/message.js ================================================ 'use strict'; const is = require('is-type-of'); const MessageConst = require('./message_const'); class Message { /** * 创建消息对象 * @param {String} topic - * @param {String} tags - * @param {String|Buffer} body - * @class */ constructor(topic, tags, body) { if (arguments.length === 2) { body = tags; tags = null; } this.storeSize = null; this.bodyCRC = null; this.queueId = null; this.flag = 0; this.queueOffset = null; // long this.commitLogOffset = null; // long this.bornHost = null; this.bornTimestamp = null; // long this.storeTimestamp = null; // long this.storeHost = null; this.reconsumeTimes = 0; this.preparedTransactionOffset = null; // long this.topic = topic; this.properties = {}; this.msgId = null; this.tags = tags; if (body && is.string(body)) { this.body = Buffer.from(body); } else { this.body = body; } } /** * 消息标签，用于过滤 * @property {String} Message#tags */ get tags() { return this.properties && this.properties[MessageConst.PROPERTY_TAGS]; } set tags(val) { this.properties[MessageConst.PROPERTY_TAGS] = val; } /** * 消息关键词 * @property {String} Message#keys */ get keys() { return this.properties && this.properties[MessageConst.PROPERTY_KEYS]; } set keys(val) { this.properties[MessageConst.PROPERTY_KEYS] = val.join(MessageConst.KEY_SEPARATOR).trim(); } /** * 原始消息 Id * @property {String} Message#originMessageId */ get originMessageId() { return this.properties && this.properties[MessageConst.PROPERTY_ORIGIN_MESSAGE_ID]; } set originMessageId(val) { this.properties[MessageConst.PROPERTY_ORIGIN_MESSAGE_ID] = val; } /** * 重试 Topic * @property {String} Message#retryTopic */ get retryTopic() { return this.properties && this.properties[MessageConst.PROPERTY_RETRY_TOPIC]; } set retryTopic(val) { this.properties[MessageConst.PROPERTY_RETRY_TOPIC] = val; } /** * 消息延时投递时间级别，0表示不延时，大于0表示特定延时级别（具体级别在服务器端定义） * @property {Number} Message#delayTimeLevel */ get delayTimeLevel() { const t = this.properties && this.properties[MessageConst.PROPERTY_DELAY_TIME_LEVEL]; if (t) { return parseInt(t, 10); } return 0; } set delayTimeLevel(val) { this.properties[MessageConst.PROPERTY_DELAY_TIME_LEVEL] = val + ''; } /** * 是否等待服务器将消息存储完毕再返回（可能是等待刷盘完成或者等待同步复制到其他服务器） * @property {Boolean} Message#waitStoreMsgOK */ get waitStoreMsgOK() { const result = this.properties && this.properties[MessageConst.PROPERTY_WAIT_STORE_MSG_OK]; return !!result; } set waitStoreMsgOK(val) { this.properties[MessageConst.PROPERTY_WAIT_STORE_MSG_OK] = String(!!val); } /** * userId，用于单元化 * @property {String} Message#buyerId */ get buyerId() { return this.properties && this.properties[MessageConst.PROPERTY_BUYER_ID]; } set buyerId(val) { this.properties[MessageConst.PROPERTY_BUYER_ID] = val; } /** * 延迟执行的时间点，毫秒 * @param {number} delayTime 延迟执行目标时间戳 */ setStartDeliverTime(delayTime) { this.properties[MessageConst.SYSTEM_PROP_KEY_STARTDELIVERTIME] = delayTime; } /** * 获取消息延迟时间 * @return {number} 延迟执行目标时间戳 */ getStartDeliverTime() { return this.properties[MessageConst.SYSTEM_PROP_KEY_STARTDELIVERTIME] || 0; } get shardingKey() { return this.properties && this.properties[MessageConst.PROPERTY_SHARDING_KEY]; } set shardingKey(val) { this.properties[MessageConst.PROPERTY_SHARDING_KEY] = val; } } module.exports = Message; ================================================ FILE: lib/message/message_const.js ================================================ 'use strict'; // 消息关键词，多个Key用KEY_SEPARATOR隔开（查询消息使用） exports.PROPERTY_KEYS = 'KEYS'; // 消息标签，只支持设置一个Tag（服务端消息过滤使用） exports.PROPERTY_TAGS = 'TAGS'; // 是否等待服务器将消息存储完毕再返回（可能是等待刷盘完成或者等待同步复制到其他服务器） exports.PROPERTY_WAIT_STORE_MSG_OK = 'WAIT'; // 消息延时投递时间级别，0表示不延时，大于0表示特定延时级别（具体级别在服务器端定义） exports.PROPERTY_DELAY_TIME_LEVEL = 'DELAY'; // 内部使用 exports.PROPERTY_RETRY_TOPIC = 'RETRY_TOPIC'; exports.PROPERTY_REAL_TOPIC = 'REAL_TOPIC'; exports.PROPERTY_REAL_QUEUE_ID = 'REAL_QID'; exports.PROPERTY_TRANSACTION_PREPARED = 'TRAN_MSG'; exports.PROPERTY_PRODUCER_GROUP = 'PGROUP'; exports.PROPERTY_MIN_OFFSET = 'MIN_OFFSET'; exports.PROPERTY_MAX_OFFSET = 'MAX_OFFSET'; exports.PROPERTY_BUYER_ID = 'BUYER_ID'; exports.PROPERTY_SHARDING_KEY = '__SHARDING_KEY'; exports.PROPERTY_ORIGIN_MESSAGE_ID = 'ORIGIN_MESSAGE_ID'; exports.PROPERTY_TRANSFER_FLAG = 'TRANSFER_FLAG'; exports.PROPERTY_CORRECTION_FLAG = 'CORRECTION_FLAG'; exports.PROPERTY_MQ2_FLAG = 'MQ2_FLAG'; exports.PROPERTY_RECONSUME_TIME = 'RECONSUME_TIME'; exports.PROPERTY_MAX_RECONSUME_TIMES = 'MAX_RECONSUME_TIMES'; exports.KEY_SEPARATOR = ' '; // 延迟消息配置key exports.SYSTEM_PROP_KEY_STARTDELIVERTIME = '__STARTDELIVERTIME'; ================================================ FILE: lib/message/message_decoder.js ================================================ 'use strict'; /* eslint no-bitwise: 0 */ const debug = require('debug')('mq:decoder'); const Long = require('long'); const is = require('is-type-of'); const utils = require('../utils'); const ByteBuffer = require('byte'); const Message = require('./message'); const MessageSysFlag = require('../utils/message_sys_flag'); const MSG_ID_LENGTH = 8 + 8; const NAME_VALUE_SEPARATOR = String.fromCharCode(1); const PROPERTY_SEPARATOR = String.fromCharCode(2); const byteBufferMsgId = ByteBuffer.allocate(MSG_ID_LENGTH); // 将属性转换为字符串 exports.messageProperties2String = function(properties) { let str = ''; if (properties) { for (const key in properties) { str += key + NAME_VALUE_SEPARATOR; str += properties[key] + PROPERTY_SEPARATOR; } } return str; }; // 解析 messageId，从中提取出 address 和 offset 信息 exports.decodeMessageId = function(msgId) { // 地址 const host = string2bytes(msgId.slice(0, 8)); const port = string2bytes(msgId.slice(8, 16)).readInt32BE(0); // offset const data = string2bytes(msgId.slice(16, 32)); const offset = new Long( data.readInt32BE(4), // low, high data.readInt32BE(0) ); return { address: host[0] + '.' + host[1] + '.' + host[2] + '.' + host[3] + ':' + port, offset, }; }; /** * 解析一条消息 * @param {ByteBuffer} byteBuffer - * @param {Boolean} readBody - 是否读取 body 部分 * @param {Boolean} deCompressBody - 是否解压 body * @return {RemotingCommand} command */ const decode = exports.decode = function(byteBuffer, readBody, deCompressBody) { if (is.nullOrUndefined(readBody)) { readBody = true; } if (is.nullOrUndefined(deCompressBody)) { deCompressBody = true; } try { return innerDecode(byteBuffer, readBody, deCompressBody); } catch (err) { debug('Error occurred during innerDecode, err: %s', err.stack); // 指向末尾 byteBuffer.position(byteBuffer.limit()); } return null; }; /** * 执行解码 * @param {ByteBuffer} byteBuffer - * @param {Boolean} readBody - 是否读取 body 部分 * @param {Boolean} deCompressBody - 是否解压 body * @return {RemotingCommand} command */ function innerDecode(byteBuffer, readBody, deCompressBody) { const msgExt = new Message(); // 消息ID byteBufferMsgId.reset(); // 1 TOTALSIZE msgExt.storeSize = byteBuffer.getInt(); // 2 MAGICCODE byteBuffer.getInt(); // 3 BODYCRC msgExt.bodyCRC = byteBuffer.getInt(); // 4 QUEUEID msgExt.queueId = byteBuffer.getInt(); // 5 FLAG msgExt.flag = byteBuffer.getInt(); // 6 QUEUEOFFSET msgExt.queueOffset = byteBuffer.getLong().toNumber(); // 7 PHYSICALOFFSET msgExt.commitLogOffset = byteBuffer.getLong().toNumber(); // 8 SYSFLAG const sysFlag = byteBuffer.getInt(); msgExt.sysFlag = sysFlag; // 9 BORNTIMESTAMP msgExt.bornTimestamp = byteBuffer.getLong().toNumber(); // 10 BORNHOST const host = Buffer.alloc(4); byteBuffer.get(host); let port = byteBuffer.getInt(); msgExt.bornHost = host[0] + '.' + host[1] + '.' + host[2] + '.' + host[3] + ':' + port; // 11 STORETIMESTAMP msgExt.storeTimestamp = byteBuffer.getLong().toNumber(); // 12 STOREHOST host.fill(0); byteBuffer.get(host); port = byteBuffer.getInt(); msgExt.storeHost = host[0] + '.' + host[1] + '.' + host[2] + '.' + host[3] + ':' + port; byteBufferMsgId.put(host); byteBufferMsgId.putInt(port); byteBufferMsgId.putLong(msgExt.commitLogOffset); // 13 RECONSUMETIMES msgExt.reconsumeTimes = byteBuffer.getInt(); // 14 Prepared Transaction Offset msgExt.preparedTransactionOffset = byteBuffer.getLong().toNumber(); // 15 BODY const bodyLen = byteBuffer.getInt(); if (bodyLen > 0) { if (readBody) { let body = Buffer.alloc(bodyLen); byteBuffer.get(body); // uncompress body if (deCompressBody && (sysFlag & MessageSysFlag.CompressedFlag) === MessageSysFlag.CompressedFlag) { body = utils.uncompress(body); } msgExt.body = body; } else { byteBuffer.position(byteBuffer.position() + bodyLen); } } // 16 TOPIC const topicLen = byteBuffer.get(); const topic = Buffer.alloc(topicLen); byteBuffer.get(topic); msgExt.topic = topic.toString(); // 17 properties const propertiesLength = byteBuffer.getShort(); if (propertiesLength > 0) { const properties = Buffer.alloc(propertiesLength); byteBuffer.get(properties); const propertiesString = properties.toString(); msgExt.properties = string2messageProperties(propertiesString); } msgExt.msgId = bytes2string(byteBufferMsgId.copy()); return msgExt; } /** * 解析一批消息 * @param {ByteBuffer} byteBuffer - * @param {Boolean} readBody - 是否读取 body 部分 * @return {RemotingCommand} command */ exports.decodes = function(byteBuffer, readBody) { readBody = readBody || true; const msgExts = []; let msgExt; while (byteBuffer.hasRemaining()) { msgExt = decode(byteBuffer, readBody); if (msgExt) { msgExts.push(msgExt); } else { break; } } return msgExts; }; // Helper // ----------- function string2bytes(hexString) { if (!hexString) { return null; } return Buffer.from(hexString, 'hex'); } function bytes2string(src) { if (!src) { return null; } return src.toString('hex').toUpperCase(); } function string2messageProperties(properties) { const map = { TAGS: null, }; if (properties) { const items = properties.split(PROPERTY_SEPARATOR); for (const item of items) { const index = item.indexOf(NAME_VALUE_SEPARATOR); map[item.slice(0, index)] = item.slice(index + 1); } } return map; } ================================================ FILE: lib/message_queue.js ================================================ 'use strict'; const fmt = require('util').format; class MessageQueue { constructor(topic, brokerName, queueId) { this.topic = topic; this.brokerName = brokerName; this.queueId = queueId; this.key = fmt('[topic="%s", brokerName="%s", queueId="%s"]', this.topic, this.brokerName, this.queueId); } } module.exports = MessageQueue; ================================================ FILE: lib/mix_all.js ================================================ 'use strict'; exports.DEFAULT_TOPIC = 'TBW102'; exports.DEFAULT_PRODUCER_GROUP = 'DEFAULT_PRODUCER'; exports.DEFAULT_CONSUMER_GROUP = 'DEFAULT_CONSUMER'; exports.CLIENT_INNER_PRODUCER_GROUP = 'CLIENT_INNER_PRODUCER'; exports.DEFAULT_CHARSET = 'UTF-8'; exports.MASTER_ID = 0; // 为每个Consumer Group建立一个默认的Topic，前缀 + GroupName，用来保存处理失败需要重试的消息 exports.RETRY_GROUP_TOPIC_PREFIX = '%RETRY%'; // 为每个Consumer Group建立一个默认的Topic，前缀 + GroupName，用来保存重试多次都失败，接下来不再重试的消息 exports.DLQ_GROUP_TOPIC_PREFIX = '%DLQ%'; /** * 获取 RETRY_TOPIC * @param {string} consumerGroup consumerGroup * @return {string} %RETRY%+consumerGroup */ exports.getRetryTopic = consumerGroup => { return exports.RETRY_GROUP_TOPIC_PREFIX + consumerGroup; }; /** * 判断是否为 RETRY_TOPIC * @param {String} topic topic * @return {boolean} ret */ exports.isRetryTopic = topic => { return topic && topic.startsWith(exports.RETRY_GROUP_TOPIC_PREFIX); }; ================================================ FILE: lib/mq_client.js ================================================ 'use strict'; const is = require('is-type-of'); const gather = require('co-gather'); const sleep = require('mz-modules/sleep'); const utility = require('utility'); const MixAll = require('./mix_all'); const MQClientAPI = require('./mq_client_api'); const MessageQueue = require('./message_queue'); const PermName = require('./protocol/perm_name'); const TopicPublishInfo = require('./producer/topic_publish_info'); const instanceTable = new Map(); class MQClient extends MQClientAPI { /** * metaq client * @param {Object} clientConfig - * @class */ constructor(clientConfig) { super(clientConfig.options); this._clientConfig = clientConfig; this._brokerAddrTable = new Map(); this._consumerTable = new Map(); this._producerTable = new Map(); this._topicRouteTable = new Map(); // this.API.on('command', this.handleServerRequest.bind(this)); } /** * @property {String} MQClient#clientId */ get clientId() { return this._clientConfig.clientId; } /** * @property {Number} MQClient#pollNameServerInteval */ get pollNameServerInteval() { return this._clientConfig.pollNameServerInteval; } /** * @property {Number} MQClient#heartbeatBrokerInterval */ get heartbeatBrokerInterval() { return this._clientConfig.heartbeatBrokerInterval; } /** * @property {Number} MQClient#persistConsumerOffsetInterval */ get persistConsumerOffsetInterval() { return this._clientConfig.persistConsumerOffsetInterval; } /** * @property {Number} MQClient#rebalanceInterval */ get rebalanceInterval() { return this._clientConfig.rebalanceInterval; } /** * start the client */ async init() { await super.init(); await this.updateAllTopicRouterInfo(); await this.sendHeartbeatToAllBroker(); await this.doRebalance(); this.startScheduledTask('updateAllTopicRouterInfo', this.pollNameServerInteval); this.startScheduledTask('sendHeartbeatToAllBroker', this.heartbeatBrokerInterval); this.startScheduledTask('doRebalance', this.rebalanceInterval); this.startScheduledTask('persistAllConsumerOffset', this.persistConsumerOffsetInterval); } async close() { if (this._consumerTable.size || this._producerTable.size) { return; } await super.close(); // todo: } /** * start a schedule task * @param {String} name - method name * @param {Number} interval - schedule interval * @param {Number} [delay] - delay time interval * @return {void} */ startScheduledTask(name, interval, delay) { (async () => { await sleep(delay || interval); while (this._inited) { try { this.logger.info('[mq:client] execute `%s` at %s', name, utility.YYYYMMDDHHmmss()); await this[name](); } catch (err) { this.logger.error(err); } await sleep(interval); } })(); } /** * regitser consumer * @param {String} group - consumer group name * @param {Consumer} consumer - consumer instance * @return {void} */ registerConsumer(group, consumer) { if (this._consumerTable.has(group)) { this.logger.warn('[mq:client] the consumer group [%s] exist already.', group); return; } this._consumerTable.set(group, consumer); this.logger.info('[mq:client] new consumer has regitsered, group: %s', group); } /** * unregister consumer * @param {String} group - consumer group name * @return {void} */ async unregisterConsumer(group) { this._consumerTable.delete(group); await this.unregister(null, group); this.logger.info('[mq:client] unregister consumer, group: %s', group); } /** * register producer * @param {String} group - producer group name * @param {Producer} producer - producer * @return {void} */ registerProducer(group, producer) { if (this._producerTable.has(group)) { this.logger.warn('[mq:client] the producer group [%s] exist already.', group); return; } this._producerTable.set(group, producer); this.logger.info('[mq:client] new producer has regitsered, group: %s', group); } /** * unregister producer * @param {String} group - producer group name * @return {void} */ async unregisterProducer(group) { this._producerTable.delete(group); await this.unregister(group, null); this.logger.info('[mq:client] unregister producer, group: %s', group); } /** * notify all broker that producer or consumer is offline * @param {String} producerGroup - producer group name * @param {String} consumerGroup - consumer group name * @return {void} */ async unregister(producerGroup, consumerGroup) { const brokerAddrTable = this._brokerAddrTable; for (const brokerName of brokerAddrTable.keys()) { const oneTable = brokerAddrTable.get(brokerName); if (!oneTable) { continue; } for (const id in oneTable) { const addr = oneTable[id]; if (addr) { await this.unregisterClient(addr, this.clientId, producerGroup, consumerGroup, 3000); } } } } /** * update all router info * @return {void} */ async updateAllTopicRouterInfo() { const topics = []; // consumer for (const groupName of this._consumerTable.keys()) { const consumer = this._consumerTable.get(groupName); if (!consumer) { continue; } for (const topic of consumer.subscriptions.keys()) { topics.push(topic); } } // producer for (const groupName of this._producerTable.keys()) { const producer = this._producerTable.get(groupName); if (!producer) { continue; } for (const topic of producer.publishTopicList) { topics.push(topic); } } this.logger.info('[mq:client] try to update all topic route info. topic: %j', topics); const ret = await gather(topics.map(topic => () => this.updateTopicRouteInfoFromNameServer(topic))); ret.forEach(data => { if (data.isError) { data.error.message = `[mq:client] updateAllTopicRouterInfo occurred error, ${data.error.message}`; this.emit('error', data.error); } }); } /** * update topic route info * @param {String} topic - topic * @param {Boolean} [isDefault] - is default or not * @param {Producer} [defaultMQProducer] - producer */ async updateTopicRouteInfoFromNameServer(topic, isDefault, defaultMQProducer) { this.logger.info('[mq:client] updateTopicRouteInfoFromNameServer() topic: %s, isDefault: %s', topic, !!isDefault); if (isDefault && defaultMQProducer) { const topicRouteData = await this.getDefaultTopicRouteInfoFromNameServer(defaultMQProducer.createTopicKey, 3000); if (topicRouteData) { for (const data of topicRouteData.queueDatas) { const queueNums = defaultMQProducer.defaultTopicQueueNums < data.readQueueNums ? defaultMQProducer.defaultTopicQueueNums : data.readQueueNums; data.readQueueNums = queueNums; data.writeQueueNums = queueNums; } this._refreshTopicRouteInfo(topic, topicRouteData); } } else { const topicRouteData = await this.getTopicRouteInfoFromNameServer(topic, 3000); if (topicRouteData) { this._refreshTopicRouteInfo(topic, topicRouteData); } } } _refreshTopicRouteInfo(topic, topicRouteData) { if (!topicRouteData) { return; } // @example // topicRouteData => { // "brokerDatas": [{ // "brokerAddrs": { // "0": "10.218.145.166:10911" // }, // "brokerName": "taobaodaily-02" // }], // "filterServerTable": {}, // "queueDatas": [{ // "brokerName": "taobaodaily-02", // "perm": 6, // "readQueueNums": 8, // "topicSynFlag": 0, // "writeQueueNums": 8 // }] // } const prev = this._topicRouteTable.get(topic); const needUpdate = this._isRouteDataChanged(prev, topicRouteData) || this._isNeedUpdateTopicRouteInfo(topic); this.logger.info('[mq:client] refresh route data for topic: %s, route data: %j, needUpdate: %s', topic, topicRouteData, needUpdate); if (!needUpdate) { return; } // @example: // this.brokerAddrTable => { // "taobaodaily-02": { // "0": "10.218.145.166:10911" // } // } for (const brokerData of topicRouteData.brokerDatas) { this._brokerAddrTable.set(brokerData.brokerName, brokerData.brokerAddrs); } // update producer route data const publishInfo = this._topicRouteData2TopicPublishInfo(topic, topicRouteData); publishInfo.haveTopicRouterInfo = true; for (const groupName of this._producerTable.keys()) { const producer = this._producerTable.get(groupName); if (producer) { producer.updateTopicPublishInfo(topic, publishInfo); } } // 更新订阅队列信息 const subscribeInfo = this._topicRouteData2TopicSubscribeInfo(topic, topicRouteData); for (const groupName of this._consumerTable.keys()) { const consumer = this._consumerTable.get(groupName); if (consumer) { consumer.updateTopicSubscribeInfo(topic, subscribeInfo); } } this.logger.info('[mq:client] update topicRouteTable for topic: %s, with topicRouteData: %j', topic, topicRouteData); this._topicRouteTable.set(topic, topicRouteData); } _isRouteDataChanged(prev, current) { if (is.nullOrUndefined(prev) || is.nullOrUndefined(current)) { return true; } // todo: performance enhance ? return JSON.stringify(prev) !== JSON.stringify(current); } _isNeedUpdateTopicRouteInfo(topic) { // producer for (const groupName of this._producerTable.keys()) { const producer = this._producerTable.get(groupName); if (producer && producer.isPublishTopicNeedUpdate(topic)) { return true; } } // consumer for (const groupName of this._consumerTable.keys()) { const consumer = this._consumerTable.get(groupName); if (consumer && consumer.isSubscribeTopicNeedUpdate(topic)) { return true; } } return false; } _topicRouteData2TopicPublishInfo(topic, topicRouteData) { const info = new TopicPublishInfo(); // 顺序消息 if (topicRouteData.orderTopicConf && topicRouteData.orderTopicConf.length) { const brokers = topicRouteData.orderTopicConf.split(';'); for (const broker of brokers) { const item = broker.split(':'); const nums = parseInt(item[1], 10); for (let i = 0; i < nums; i++) { info.messageQueueList.push(new MessageQueue(topic, item[0], i)); } } info.orderTopic = true; } else { // 非顺序消息 for (const queueData of topicRouteData.queueDatas) { if (PermName.isWriteable(queueData.perm)) { const brokerData = topicRouteData.brokerDatas.find(data => { return data.brokerName === queueData.brokerName; }); if (!brokerData || !brokerData.brokerAddrs[MixAll.MASTER_ID]) { continue; } for (let i = 0, nums = queueData.writeQueueNums; i < nums; i++) { info.messageQueueList.push(new MessageQueue(topic, queueData.brokerName, i)); } } } info.orderTopic = false; } return info; } _topicRouteData2TopicSubscribeInfo(topic, topicRouteData) { const messageQueueList = []; for (const queueData of topicRouteData.queueDatas) { if (PermName.isReadable(queueData.perm)) { for (let i = 0, nums = queueData.readQueueNums; i < nums; i++) { messageQueueList.push(new MessageQueue(topic, queueData.brokerName, i)); } } } return messageQueueList; } /** * send heartbeat to all brokers * @return {void} */ async sendHeartbeatToAllBroker() { this._cleanOfflineBroker(); const heartbeatData = this._prepareHeartbeatData(); const consumerEmpty = heartbeatData.consumerDataSet.length === 0; const producerEmpty = heartbeatData.producerDataSet.length === 0; if (consumerEmpty && producerEmpty) { this.logger.info('[mq:client] sending hearbeat, but no consumer and no producer'); return; } const brokers = []; for (const brokerName of this._brokerAddrTable.keys()) { const oneTable = this._brokerAddrTable.get(brokerName); if (!oneTable) { continue; } for (const id in oneTable) { const addr = oneTable[id]; if (!addr) { continue; } // 说明只有Producer，则不向Slave发心跳 if (consumerEmpty && Number(id) !== MixAll.MASTER_ID) { continue; } brokers.push(addr); } } const ret = await gather(brokers.map(addr => () => this.sendHearbeat(addr, heartbeatData, 3000))); this.logger.info('[mq:client] send heartbeat: %j to : %j, and result: %j', heartbeatData, brokers, ret); } _prepareHeartbeatData() { const heartbeatData = { clientID: this.clientId, consumerDataSet: [], producerDataSet: [], }; // Consumer for (const groupName of this._consumerTable.keys()) { const consumer = this._consumerTable.get(groupName); if (consumer) { const subscriptionDataSet = []; for (const topic of consumer.subscriptions.keys()) { const data = consumer.subscriptions.get(topic); if (data && data.subscriptionData) { subscriptionDataSet.push(data.subscriptionData); } } heartbeatData.consumerDataSet.push({ groupName: consumer.consumerGroup, consumeType: consumer.consumeType, messageModel: consumer.messageModel, consumeFromWhere: consumer.consumeFromWhere, subscriptionDataSet, unitMode: consumer.unitMode, }); } } // Producer for (const groupName of this._producerTable.keys()) { const producer = this._producerTable.get(groupName); if (producer) { heartbeatData.producerDataSet.push({ groupName, }); } } return heartbeatData; } _cleanOfflineBroker() { for (const brokerName of this._brokerAddrTable.keys()) { const oneTable = this._brokerAddrTable.get(brokerName); let exists = false; for (const brokerId in oneTable) { const addr = oneTable[brokerId]; if (this._isBrokerAddrExistInTopicRouteTable(addr)) { exists = true; } else { delete oneTable[brokerId]; this.logger.info('[mq:client] the broker addr[%s %s] is offline, remove it', brokerName, addr); } } if (!exists) { this._brokerAddrTable.delete(brokerName); this.logger.info('[mq:client] the broker[%s] name\'s host is offline, remove it', brokerName); } } } _isBrokerAddrExistInTopicRouteTable(addr) { for (const topic of this._topicRouteTable.keys()) { const topicRouteData = this._topicRouteTable.get(topic); for (const brokerData of topicRouteData.brokerDatas) { for (const brokerId in brokerData.brokerAddrs) { if (brokerData.brokerAddrs[brokerId] === addr) { return true; } } } } return false; } /** * rebalance * @return {void} */ async doRebalance() { for (const groupName of this._consumerTable.keys()) { const consumer = this._consumerTable.get(groupName); if (consumer) { await consumer.doRebalance(); } } } /** * get all consumer list of topic * @param {String} topic - topic * @param {String} group - consumer group * @return {Array} consumer list */ async findConsumerIdList(topic, group) { let brokerAddr = this.findBrokerAddrByTopic(topic); if (!brokerAddr) { await this.updateTopicRouteInfoFromNameServer(topic); brokerAddr = this.findBrokerAddrByTopic(topic); if (!brokerAddr) { throw new Error(`The broker of topic[${topic}] not exist`); } else { return await this.getConsumerIdListByGroup(brokerAddr, group, 3000); } } else { return await this.getConsumerIdListByGroup(brokerAddr, group, 3000); } } // get broker address by topic findBrokerAddrByTopic(topic) { const topicRouteData = this._topicRouteTable.get(topic); if (topicRouteData) { const brokerDatas = topicRouteData.brokerDatas || []; const broker = brokerDatas[0]; if (broker) { // master first, not found try slave const addr = broker.brokerAddrs[MixAll.MASTER_ID]; if (!addr) { for (const id in broker.brokerAddrs) { return broker.brokerAddrs[id]; } } return addr; } } return null; } /** * find broker address * @param {String} brokerName - broker name * @return {Object} broker info */ findBrokerAddressInAdmin(brokerName) { const map = this._brokerAddrTable.get(brokerName); if (!map) { return null; } if (map[MixAll.MASTER_ID]) { return { brokerAddr: map[MixAll.MASTER_ID], slave: false, }; } for (const id in map) { if (map[id]) { return { brokerAddr: map[id], slave: true, }; } } return null; } findBrokerAddressInSubscribe(brokerName, brokerId, onlyThisBroker) { let brokerAddr = null; let slave = false; let found = false; const map = this._brokerAddrTable.get(brokerName); if (map) { brokerAddr = map[brokerId]; slave = Number(brokerId) !== MixAll.MASTER_ID; found = !is.nullOrUndefined(brokerAddr); // 尝试寻找其他Broker if (!found && !onlyThisBroker) { for (const id in map) { brokerAddr = map[id]; slave = Number(id) !== MixAll.MASTER_ID; found = true; break; } } } if (found) { return { brokerAddr, slave, }; } return null; } /** * get current offset of message queue * @param {MessageQueue} messageQueue - message queue * @return {Number} offset */ async maxOffset(messageQueue) { let brokerAddr = this.findBrokerAddressInPublish(messageQueue.brokerName); if (!brokerAddr) { await this.updateTopicRouteInfoFromNameServer(messageQueue.topic); brokerAddr = this.findBrokerAddressInPublish(messageQueue.brokerName); if (!brokerAddr) { throw new Error(`The broker[${messageQueue.brokerName}] not exist`); } else { return await this.getMaxOffset(brokerAddr, messageQueue.topic, messageQueue.queueId, 3000); } } else { return await this.getMaxOffset(brokerAddr, messageQueue.topic, messageQueue.queueId, 3000); } } // should be master findBrokerAddressInPublish(brokerName) { const map = this._brokerAddrTable.get(brokerName); return map && map[MixAll.MASTER_ID]; } async persistAllConsumerOffset() { for (const groupName of this._consumerTable.keys()) { const consumer = this._consumerTable.get(groupName); if (consumer) { await consumer.persistConsumerOffset(); } } } async searchOffset(messageQueue, timestamp) { let brokerAddr = this.findBrokerAddressInPublish(messageQueue.brokerName); if (!brokerAddr) { await this.updateTopicRouteInfoFromNameServer(messageQueue.topic); brokerAddr = this.findBrokerAddressInPublish(messageQueue.brokerName); if (!brokerAddr) { throw new Error('The broker[' + messageQueue.brokerName + '] not exist'); } else { return await super.searchOffset(brokerAddr, messageQueue.topic, messageQueue.queueId, timestamp, 3000); } } else { return await super.searchOffset(brokerAddr, messageQueue.topic, messageQueue.queueId, timestamp, 3000); } } static getAndCreateMQClient(clientConfig) { const clientId = clientConfig.clientId; let instance = instanceTable.get(clientId); if (!instance) { instance = new MQClient(clientConfig); instanceTable.set(clientId, instance); instance.once('close', () => { instanceTable.delete(clientId); }); } return instance; } } module.exports = MQClient; ================================================ FILE: lib/mq_client_api.js ================================================ 'use strict'; const JSON2 = require('JSON2'); const bytes = require('bytes'); const fmt = require('util').format; const ByteBuffer = require('byte'); const MessageQueue = require('./message_queue'); const RemotingClient = require('./remoting_client'); const PullStatus = require('./consumer/pull_status'); const SendStatus = require('./producer/send_status'); const RequestCode = require('./protocol/request_code'); const ResponseCode = require('./protocol/response_code'); const MessageConst = require('./message/message_const'); const MessageDecoder = require('./message/message_decoder'); const RemotingCommand = require('./protocol/command/remoting_command'); // const localIp = require('address').ip(); // const NAMESPACE_ORDER_TOPIC_CONFIG = 'ORDER_TOPIC_CONFIG'; const NAMESPACE_PROJECT_CONFIG = 'PROJECT_CONFIG'; const VIRTUAL_APPGROUP_PREFIX = '%%PROJECT_%s%%'; const byteBuffer = ByteBuffer.allocate(bytes('1m')); /* eslint no-fallthrough:0 */ class MQClientAPI extends RemotingClient { /** * Metaq api wrapper * @param {Object} options * - {HttpClient} httpclient - http request client * - {Object} [logger] - log module * - {Number} [responseTimeout] - tcp response timeout * @class */ constructor(options) { super(options); // virtual env project group this.projectGroupPrefix = null; } /** * start the client * @return {void} */ async init() { await super.init(); // this.projectGroupPrefix = await this.getProjectGroupByIp(localIp, 3000); } /** * get project group prefix by ip address * @param {String} ip - ip address * @param {Number} [timeoutMillis] - timeout in milliseconds * @return {String} prefix */ async getProjectGroupByIp(ip, timeoutMillis) { try { return await this.getKVConfigByValue(NAMESPACE_PROJECT_CONFIG, ip, timeoutMillis); } catch (err) { err.message = `[mq:api] Can not get project config from server, ${err.message}`; this.logger.error(err); return null; } } /** * get key-value config * @param {String} namespace - config namespace * @param {String} value - config key * @param {Number} [timeoutMillis] - timeout in milliseconds * @return {String} config value */ async getKVConfigByValue(namespace, value, timeoutMillis) { const requestHeader = { namespace, key: value, }; const request = RemotingCommand.createRequestCommand(RequestCode.GET_KV_CONFIG_BY_VALUE, requestHeader); const response = await this.invokeForNameSrvAtLeastOnce(request, timeoutMillis); switch (response.code) { case ResponseCode.SUCCESS: { const responseHeader = response.decodeCommandCustomHeader(); return responseHeader && responseHeader.value; } default: this._defaultHandler(request, response); break; } } /** * get route info of topic * @param {String} topic - topic * @param {Number} [timeoutMillis] - timeout in milliseconds * @return {Object} router info */ async getDefaultTopicRouteInfoFromNameServer(topic, timeoutMillis) { const requestHeader = { topic, }; const request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINTO_BY_TOPIC, requestHeader); const response = await this.invokeForNameSrvAtLeastOnce(request, timeoutMillis); switch (response.code) { case ResponseCode.SUCCESS: { const body = response.body; if (body) { this.logger.info('[mq:client_api] get Topic [%s] RouteInfoFromNameServer: %s', topic, body.toString()); // JSON.parse dose not work here const routerInfoData = JSON2.parse(body.toString()); // sort routerInfoData.queueDatas.sort(compare); routerInfoData.brokerDatas.sort(compare); return routerInfoData; } break; } case ResponseCode.TOPIC_NOT_EXIST: this.logger.info('[mq:client_api] get Topic [%s] RouteInfoFromNameServer is not exist value', topic); default: this._defaultHandler(request, response); break; } } /** * notify broker that client is offline * @param {String} addr - brokder address * @param {String} clientId - clientId * @param {String} producerGroup - producer group name * @param {String} consumerGroup - consumer group name * @param {Number} [timeoutMillis] - timeout in milliseconds * @return {void} */ async unregisterClient(addr, clientId, producerGroup, consumerGroup, timeoutMillis) { producerGroup = this._buildWithProjectGroup(producerGroup); consumerGroup = this._buildWithProjectGroup(consumerGroup); const requestHeader = { clientID: clientId, producerGroup, consumerGroup, }; const request = RemotingCommand.createRequestCommand(RequestCode.UNREGISTER_CLIENT, requestHeader); const response = await this.invoke(addr, request, timeoutMillis); switch (response.code) { case ResponseCode.SUCCESS: break; default: this._defaultHandler(request, response); break; } } /** * get route info from name server * @param {String} topic - topic * @param {Number} [timeoutMillis] - timeout in milliseconds * @return {Object} route info */ async getTopicRouteInfoFromNameServer(topic, timeoutMillis) { topic = this._buildWithProjectGroup(topic); const request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINTO_BY_TOPIC, { topic, }); const response = await this.invokeForNameSrvAtLeastOnce(request, timeoutMillis); switch (response.code) { case ResponseCode.SUCCESS: { const body = response.body; if (body) { const routerInfoData = JSON2.parse(body.toString()); // sort routerInfoData.queueDatas.sort(compare); routerInfoData.brokerDatas.sort(compare); return routerInfoData; } break; } case ResponseCode.TOPIC_NOT_EXIST: this.logger.warn('[mq:client_api] get Topic [%s] RouteInfoFromNameServer is not exist value', topic); default: this._defaultHandler(request, response); break; } } /** * send heartbeat * @param {String} addr - broker address * @param {Object} heartbeatData - heartbeat data * @param {Number} [timeout] - timeout in milliseconds * @return {void} */ async sendHearbeat(addr, heartbeatData, timeout) { if (this.projectGroupPrefix) { for (const consumerData of heartbeatData.consumerDataSet) { consumerData.groupName = this._buildWithProjectGroup(consumerData.groupName); for (const subscriptionData of consumerData.subscriptionDataSet) { subscriptionData.topic = this._buildWithProjectGroup(subscriptionData.topic); } } for (const producerData of heartbeatData.producerDataSet) { producerData.groupName = this._buildWithProjectGroup(producerData.groupName); } } const request = RemotingCommand.createRequestCommand(RequestCode.HEART_BEAT, null); request.body = Buffer.from(JSON.stringify(heartbeatData)); const response = await this.invoke(addr, request, timeout); if (response.code !== ResponseCode.SUCCESS) { this._defaultHandler(request, response); } } /** * update consumer offset * @param {String} brokerAddr - broker address * @param {Object} requestHeader - request header * @return {void} */ async updateConsumerOffsetOneway(brokerAddr, requestHeader) { requestHeader.consumerGroup = this._buildWithProjectGroup(requestHeader.consumerGroup); requestHeader.topic = this._buildWithProjectGroup(requestHeader.topic); const request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_CONSUMER_OFFSET, requestHeader); await this.invokeOneway(brokerAddr, request); } /** * query consume offset * @param {String} brokerAddr - broker address * @param {Object} requestHeader - request header * @param {Number} [timeoutMillis] - timeout in milliseconds * @return {Number} offset */ async queryConsumerOffset(brokerAddr, requestHeader, timeoutMillis) { requestHeader.consumerGroup = this._buildWithProjectGroup(requestHeader.consumerGroup); requestHeader.topic = this._buildWithProjectGroup(requestHeader.topic); const request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUMER_OFFSET, requestHeader); const response = await this.invoke(brokerAddr, request, timeoutMillis); switch (response.code) { case ResponseCode.SUCCESS: { const responseHeader = response.decodeCommandCustomHeader(); return Number(responseHeader.offset.toString()); } default: this._defaultHandler(request, response); break; } } /** * get current max offset of queue * @param {String} addr - broker address * @param {String} topic - topic * @param {Number} queueId - queue id * @param {Number} [timeoutMillis] - timeout in milliseconds * @return {Number} offset */ async getMaxOffset(addr, topic, queueId, timeoutMillis) { topic = this._buildWithProjectGroup(topic); const requestHeader = { topic, queueId, }; const request = RemotingCommand.createRequestCommand(RequestCode.GET_MAX_OFFSET, requestHeader); const response = await this.invoke(addr, request, timeoutMillis); switch (response.code) { case ResponseCode.SUCCESS: { const responseHeader = response.decodeCommandCustomHeader(); // todo: return responseHeader && Number(responseHeader.offset); } default: this._defaultHandler(request, response); break; } } /** * search consume offset by timestamp * @param {String} addr - broker address * @param {String} topic - topic * @param {Number} queueId - queue id * @param {String} timestamp - timestamp used to query * @param {Number} [timeoutMillis] - timeout in milliseconds * @return {Number} offset */ async searchOffset(addr, topic, queueId, timestamp, timeoutMillis) { topic = this._buildWithProjectGroup(topic); const requestHeader = { topic, queueId, timestamp, }; const request = RemotingCommand.createRequestCommand(RequestCode.SEARCH_OFFSET_BY_TIMESTAMP, requestHeader); const response = await this.invoke(addr, request, timeoutMillis); switch (response.code) { case ResponseCode.SUCCESS: { const responseHeader = response.decodeCommandCustomHeader(); // todo: return responseHeader && Number(responseHeader.offset); } default: this._defaultHandler(request, response); break; } } /** * get all consumer's id in same group * @param {String} addr - broker address * @param {String} consumerGroup - consumer group * @param {Number} [timeoutMillis] - timeout in milliseconds * @return {Array} consumer list */ async getConsumerIdListByGroup(addr, consumerGroup, timeoutMillis) { consumerGroup = this._buildWithProjectGroup(consumerGroup); const requestHeader = { consumerGroup, }; const request = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_LIST_BY_GROUP, requestHeader); const response = await this.invoke(addr, request, timeoutMillis); switch (response.code) { case ResponseCode.SUCCESS: if (response.body) { const body = JSON2.parse(response.body.toString()); return body.consumerIdList; } break; default: this._defaultHandler(request, response); break; } } /** * pull message from broker * @param {String} brokerAddr - broker address * @param {Object} requestHeader - request header * @param {Number} [timeoutMillis] - timeout in milliseconds * @return {Object} pull result */ async pullMessage(brokerAddr, requestHeader, timeoutMillis) { requestHeader.consumerGroup = this._buildWithProjectGroup(requestHeader.consumerGroup); requestHeader.topic = this._buildWithProjectGroup(requestHeader.topic); const request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, requestHeader); const response = await this.invoke(brokerAddr, request, timeoutMillis); let pullStatus = PullStatus.NO_NEW_MSG; switch (response.code) { case ResponseCode.SUCCESS: pullStatus = PullStatus.FOUND; break; case ResponseCode.PULL_NOT_FOUND: pullStatus = PullStatus.NO_NEW_MSG; break; case ResponseCode.PULL_RETRY_IMMEDIATELY: pullStatus = PullStatus.NO_MATCHED_MSG; break; case ResponseCode.PULL_OFFSET_MOVED: pullStatus = PullStatus.OFFSET_ILLEGAL; break; default: this._defaultHandler(request, response); break; } const responseHeader = response.decodeCommandCustomHeader(); let msgList = []; if (pullStatus === PullStatus.FOUND) { byteBuffer.reset(); byteBuffer.put(response.body).flip(); msgList = MessageDecoder.decodes(byteBuffer); for (const msg of msgList) { msg.topic = this._clearProjectGroup(msg.topic); msg.properties[MessageConst.PROPERTY_MIN_OFFSET] = responseHeader.minOffset.toString(); msg.properties[MessageConst.PROPERTY_MAX_OFFSET] = responseHeader.maxOffset.toString(); } } return { pullStatus, nextBeginOffset: Number(responseHeader.nextBeginOffset), minOffset: Number(responseHeader.minOffset), maxOffset: Number(responseHeader.maxOffset), msgFoundList: msgList, suggestWhichBrokerId: responseHeader.suggestWhichBrokerId, }; } /** * create topic * @param {String} addr - broker address * @param {String} defaultTopic - default topic: TBW102 * @param {Object} topicConfig - new topic config * @param {Number} [timeoutMillis] - timeout in milliseconds * @return {void} */ async createTopic(addr, defaultTopic, topicConfig, timeoutMillis) { const topicWithProjectGroup = this._buildWithProjectGroup(topicConfig.topicName); const requestHeader = { topic: topicWithProjectGroup, defaultTopic, readQueueNums: topicConfig.readQueueNums, writeQueueNums: topicConfig.writeQueueNums, perm: topicConfig.perm, topicFilterType: topicConfig.topicFilterType, topicSysFlag: topicConfig.topicSysFlag, order: topicConfig.order, }; const request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_TOPIC, requestHeader); const response = await this.invoke(addr, request, timeoutMillis); switch (response.code) { case ResponseCode.SUCCESS: return; default: this._defaultHandler(request, response); break; } } /** * send message * @param {String} brokerAddr - broker address * @param {String} brokerName - broker name * @param {Message} msg - msg object * @param {Object} requestHeader - request header * @param {Number} [timeoutMillis] - timeout in milliseconds * @return {Object} sendResult */ async sendMessage(brokerAddr, brokerName, msg, requestHeader, timeoutMillis) { msg.topic = this._buildWithProjectGroup(msg.topic); requestHeader.producerGroup = this._buildWithProjectGroup(requestHeader.producerGroup); requestHeader.topic = this._buildWithProjectGroup(requestHeader.topic); const requestHeaderV2 = { a: requestHeader.producerGroup, b: requestHeader.topic, c: requestHeader.defaultTopic, d: requestHeader.defaultTopicQueueNums, e: requestHeader.queueId, f: requestHeader.sysFlag, g: requestHeader.bornTimestamp, h: requestHeader.flag, i: requestHeader.properties, j: requestHeader.reconsumeTimes, k: requestHeader.unitMode, l: requestHeader.maxReconsumeTimes, }; const request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, requestHeaderV2); request.body = msg.body; const response = await this.invoke(brokerAddr, request, timeoutMillis); let sendStatus = SendStatus.SEND_OK; switch (response.code) { case ResponseCode.FLUSH_DISK_TIMEOUT: sendStatus = SendStatus.FLUSH_DISK_TIMEOUT; break; case ResponseCode.FLUSH_SLAVE_TIMEOUT: sendStatus = SendStatus.FLUSH_SLAVE_TIMEOUT; break; case ResponseCode.SLAVE_NOT_AVAILABLE: sendStatus = SendStatus.SLAVE_NOT_AVAILABLE; break; case ResponseCode.SUCCESS: sendStatus = SendStatus.SEND_OK; break; default: this._defaultHandler(request, response); break; } const responseHeader = response.decodeCommandCustomHeader(); const messageQueue = new MessageQueue(msg.topic, brokerName, responseHeader.queueId); messageQueue.topic = this._clearProjectGroup(messageQueue.topic); return { sendStatus, msgId: responseHeader.msgId, messageQueue, queueOffset: Number(responseHeader.queueOffset), }; } /** * consumer send message back * @param {String} brokerAddr - broker address * @param {Message} msg - message object * @param {String} consumerGroup - consumer group * @param {Number} delayLevel - delay level * @param {Number} timeoutMillis - timeout in millis * @param {Number} maxConsumeRetryTimes - max retry times */ async consumerSendMessageBack(brokerAddr, msg, consumerGroup, delayLevel, timeoutMillis, maxConsumeRetryTimes) { const requestHeader = { offset: msg.commitLogOffset, group: consumerGroup, delayLevel, originMsgId: msg.msgId, originTopic: msg.topic, unitMode: false, maxReconsumeTimes: maxConsumeRetryTimes, }; const request = RemotingCommand.createRequestCommand(RequestCode.CONSUMER_SEND_MSG_BACK, requestHeader); const response = await this.invoke(brokerAddr, request, timeoutMillis); switch (response.code) { case ResponseCode.SUCCESS: return; default: this._defaultHandler(request, response); break; } } // * viewMessage(brokerAddr, phyoffset, timeoutMillis) { // const requestHeader = { // offset: phyoffset, // }; // const request = RemotingCommand.createRequestCommand(RequestCode.VIEW_MESSAGE_BY_ID, requestHeader); // const response = await this.invoke(brokerAddr, request, timeoutMillis); // switch (response.code) { // case ResponseCode.SUCCESS: // { // const byteBuffer = ByteBuffer.wrap(response.body); // const messageExt = MessageDecoder.decode(byteBuffer); // // 清除虚拟运行环境相关的projectGroupPrefix // if (this.projectGroupPrefix) { // messageExt.topic = this._clearProjectGroup(messageExt.topic, this.projectGroupPrefix); // } // return messageExt; // } // default: // this._defaultHandler(request, response); // break; // } // } // default handler _defaultHandler(request, response) { const err = new Error(response.remark); err.name = 'MQClientException'; err.code = response.code; throw err; } _buildWithProjectGroup(origin) { if (this.projectGroupPrefix) { const prefix = fmt(VIRTUAL_APPGROUP_PREFIX, this.projectGroupPrefix); if (!origin.endsWith(prefix)) { return origin + prefix; } return origin; } return origin; } _clearProjectGroup(origin) { const prefix = fmt(VIRTUAL_APPGROUP_PREFIX, this.projectGroupPrefix); if (prefix && origin.endsWith(prefix)) { return origin.slice(0, origin.lastIndexOf(prefix)); } return origin; } } module.exports = MQClientAPI; // Helper // --------------- function compare(routerA, routerB) { if (routerA.brokerName > routerB.brokerName) { return 1; } else if (routerA.brokerName < routerB.brokerName) { return -1; } return 0; } ================================================ FILE: lib/process_queue.js ================================================ 'use strict'; const Base = require('sdk-base'); const bsInsert = require('binary-search-insert'); const comparator = (a, b) => a.queueOffset - b.queueOffset; const pullMaxIdleTime = 120000; class ProcessQueue extends Base { constructor(options = {}) { super(options); this.msgList = []; this.droped = false; this.lastPullTimestamp = Date.now(); this.lastConsumeTimestamp = Date.now(); this.locked = false; this.lastLockTimestamp = Date.now(); } get maxSpan() { const msgCount = this.msgCount; if (msgCount) { return this.msgList[msgCount - 1].queueOffset - this.msgList[0].queueOffset; } return 0; } get msgCount() { return this.msgList.length; } get isPullExpired() { return Date.now() - this.lastPullTimestamp > pullMaxIdleTime; } putMessage(msgs) { for (const msg of msgs) { bsInsert(this.msgList, comparator, msg); } this.queueOffsetMax = this.msgList[this.msgCount - 1].queueOffset; } remove(count = 1) { this.msgList.splice(0, count); return this.msgCount ? this.msgList[0].queueOffset : this.queueOffsetMax + 1; } clear() { this.msgList = []; } } module.exports = ProcessQueue; ================================================ FILE: lib/producer/mq_producer.js ================================================ 'use strict'; /* eslint no-bitwise: 0 */ const utils = require('../utils'); const logger = require('../logger'); const MixAll = require('../mix_all'); const MQClient = require('../mq_client'); const SendStatus = require('./send_status'); const ClientConfig = require('../client_config'); // const PermName = require('../protocol/perm_name'); const MessageConst = require('../message/message_const'); const TopicPublishInfo = require('./topic_publish_info'); const ResponseCode = require('../protocol/response_code'); const MessageSysFlag = require('../utils/message_sys_flag'); const MessageDecoder = require('../message/message_decoder'); const defaultOptions = { logger, producerGroup: MixAll.DEFAULT_PRODUCER_GROUP, createTopicKey: MixAll.DEFAULT_TOPIC, defaultTopicQueueNums: 4, sendMsgTimeout: 3000, compressMsgBodyOverHowmuch: 1024 * 4, retryTimesWhenSendFailed: 3, retryAnotherBrokerWhenNotStoreOK: false, maxMessageSize: 1024 * 128, connectTimeout: 10000, }; class MQProducer extends ClientConfig { /** * metaq message producer * @param {Object} options - * @class */ constructor(options) { super(Object.assign({ initMethod: 'init' }, defaultOptions, options)); this._topicPublishInfoTable = new Map(); this._inited = false; if (this.producerGroup !== MixAll.CLIENT_INNER_PRODUCER_GROUP) { this.changeInstanceNameToPID(); } this._mqClient = MQClient.getAndCreateMQClient(this); this._mqClient.on('error', err => this._error(err)); } get logger() { return this.options.logger; } get producerGroup() { if (this.namespace) { return `${this.namespace}%${this.options.producerGroup}`; } return this.options.producerGroup; } get createTopicKey() { return this.options.createTopicKey; } get defaultTopicQueueNums() { return this.options.defaultTopicQueueNums; } /** * topic list * @property {Array} MQProducer#publishTopicList */ get publishTopicList() { return this._topicPublishInfoTable.keys(); } /** * start the producer * @return {void} */ async init() { // this._topicPublishInfoTable.set(this.createTopicKey, new TopicPublishInfo()); await MQProducer.getDefaultProducer(this.options); this._mqClient.registerProducer(this.producerGroup, this); await this._mqClient.ready(); this.logger.info('[mq:producer] producer started'); this._inited = true; } /** * close the producer */ async close() { this._topicPublishInfoTable.clear(); await this._mqClient.unregisterProducer(this.producerGroup); await this._mqClient.close(); this.removeAllListeners(); this._inited = false; this.logger.info('[mq:producer] producer close'); } /** * default error handler * @param {Error} err - error * @return {void} */ _error(err) { setImmediate(() => { err.message = 'MQPushConsumer occurred an error' + err.message; this.emit('error', err); }); } /** * update topic info * @param {String} topic - topic * @param {Object} info - route info * @return {void} */ updateTopicPublishInfo(topic, info) { if (topic && info) { const prev = this._topicPublishInfoTable.get(topic); this._topicPublishInfoTable.set(topic, info); if (prev) { info.sendWhichQueue = prev.sendWhichQueue; this.logger.warn('[mq:producer] updateTopicPublishInfo() prev is not null, %j', prev); } } } /** * is publish topic info need update * @param {String} topic - topic * @return {Boolean} need update? */ isPublishTopicNeedUpdate(topic) { const info = this._topicPublishInfoTable.get(topic); return !info || !info.ok; } /** * create topic * @param {String} key - TBW102 * @param {String} newTopic - * @param {Number} queueNum - queue number * @param {Number} topicSysFlag - 0 * @return {void} */ // * createTopic(key, newTopic, queueNum, topicSysFlag) { // await this.ready(); // topicSysFlag = topicSysFlag || 0; // const topicRouteData = await this._mqClient.getTopicRouteInfoFromNameServer(key, 1000 * 3); // const brokerDataList = topicRouteData.brokerDatas; // if (brokerDataList && brokerDataList.length) { // // 排序原因：即使没有配置顺序消息模式，默认队列的顺序同配置的一致。 // brokerDataList.sort(compare); // await brokerDataList.map(brokerData => { // const addr = brokerData.brokerAddrs[MixAll.MASTER_ID]; // if (addr) { // const topicConfig = { // topicName: newTopic, // readQueueNums: queueNum, // writeQueueNums: queueNum, // topicSysFlag, // perm: PermName.PERM_READ | PermName.PERM_WRITE, // topicFilterType: 'SINGLE_TAG', // order: false, // }; // this.logger.info('[mq:producer] execute createTopic at broker: %s, topic: %s', addr, newTopic); // return this._mqClient.createTopic(addr, key, topicConfig, 1000 * 3); // } // return null; // }); // } else { // throw new Error('Not found broker, maybe key is wrong'); // } // } /** * send message * @param {Message} msg - message object * @return {Object} sendResult */ async send(msg) { await this.ready(); msg.topic = this.formatTopic(msg.topic); const topicPublishInfo = await this.tryToFindTopicPublishInfo(msg.topic); if (!topicPublishInfo) { throw new Error(`no publish router data for topic: ${msg.topic}`); } const maxTimeout = this.options.sendMsgTimeout + 1000; const timesTotal = 1 + this.options.retryTimesWhenSendFailed; const beginTimestamp = Date.now(); const brokersSent = []; let times = 0; let lastBrokerName; let sendResult; for (; times < timesTotal && Date.now() - beginTimestamp < maxTimeout; times++) { const messageQueue = topicPublishInfo.selectOneMessageQueue(lastBrokerName, msg); if (!messageQueue) { continue; } lastBrokerName = messageQueue.brokerName; brokersSent[times] = lastBrokerName; try { sendResult = await this.sendKernelImpl(msg, messageQueue); if (sendResult.sendStatus === SendStatus.SEND_OK || !this.options.retryAnotherBrokerWhenNotStoreOK) { break; } } catch (err) { if (err.name === 'MQClientException') { switch (err.code) { case ResponseCode.TOPIC_NOT_EXIST: case ResponseCode.SERVICE_NOT_AVAILABLE: case ResponseCode.SYSTEM_ERROR: case ResponseCode.NO_PERMISSION: case ResponseCode.NO_BUYER_ID: case ResponseCode.NOT_IN_CURRENT_UNIT: break; default: throw err; } } } } if (!sendResult) { throw new Error(`Send [${times}] times, still failed, cost [${Date.now() - beginTimestamp}]ms, Topic: ${msg.topic}, BrokersSent: ${JSON.stringify(brokersSent)}`); } return sendResult; } /** * try to find topic info from name server * @param {String} topic - topic * @return {Object} route info */ async tryToFindTopicPublishInfo(topic) { let topicPublishInfo = this._topicPublishInfoTable.get(topic); if (!topicPublishInfo || !topicPublishInfo.ok) { this._topicPublishInfoTable.set(topic, new TopicPublishInfo()); await this._mqClient.updateTopicRouteInfoFromNameServer(topic); topicPublishInfo = this._topicPublishInfoTable.get(topic); } if (topicPublishInfo.haveTopicRouterInfo || topicPublishInfo && topicPublishInfo.ok) { return topicPublishInfo; } await this._mqClient.updateTopicRouteInfoFromNameServer(topic, true, this); return this._topicPublishInfoTable.get(topic); } async sendKernelImpl(msg, messageQueue) { let brokerAddr = this._mqClient.findBrokerAddressInPublish(messageQueue.brokerName); if (!brokerAddr) { await this.tryToFindTopicPublishInfo(messageQueue.topic); brokerAddr = this._mqClient.findBrokerAddressInPublish(messageQueue.brokerName); if (!brokerAddr) { const err = new Error(`The broker[${messageQueue.brokerName}] not exist`); err.name = 'MQClientException'; throw err; } } let sysFlag = 0; if (this.tryToCompressMessage(msg)) { sysFlag |= MessageSysFlag.CompressedFlag; } const tranMsg = msg.properties[MessageConst.PROPERTY_TRANSACTION_PREPARED]; if (tranMsg) { sysFlag |= MessageSysFlag.TransactionPreparedType; } const requestHeader = { producerGroup: this.producerGroup, topic: msg.topic, defaultTopic: this.createTopicKey, defaultTopicQueueNums: this.defaultTopicQueueNums, queueId: messageQueue.queueId, sysFlag, bornTimestamp: Date.now(), flag: msg.flag, properties: MessageDecoder.messageProperties2String(msg.properties), reconsumeTimes: 0, unitMode: this.unitMode, }; if (requestHeader.topic.indexOf(MixAll.RETRY_GROUP_TOPIC_PREFIX) === 0) { const reconsumeTimes = msg.properties[MessageConst.PROPERTY_RECONSUME_TIME]; if (reconsumeTimes) { requestHeader.reconsumeTimes = parseInt(reconsumeTimes, 10); delete msg.properties[MessageConst.PROPERTY_RECONSUME_TIME]; } const maxReconsumeTimes = msg.properties[MessageConst.PROPERTY_MAX_RECONSUME_TIMES]; if (maxReconsumeTimes) { requestHeader.maxReconsumeTimes = parseInt(maxReconsumeTimes, 10); delete msg.properties[MessageConst.PROPERTY_MAX_RECONSUME_TIMES]; } } return await this._mqClient.sendMessage(brokerAddr, messageQueue.brokerName, msg, requestHeader, this.options.sendMsgTimeout); } /** * 尝试压缩消息 * @param {Message} msg - 消息对象 * @return {Boolean} 是否压缩 */ tryToCompressMessage(msg) { const body = msg.body; if (body) { if (body.length >= this.options.compressMsgBodyOverHowmuch) { try { const data = utils.compress(body); if (data) { msg.body = data; return true; } } catch (err) { err.name = 'MetaQCompressError'; this._error(err); } } } return false; } /** * 获取默认 Producer 实例 * @param {Object} options 配置 * @return {MQProducer} 默认 Producer 实例 */ static async getDefaultProducer(options = {}) { if (!MQProducer.defaultProducer) { const opts = Object.assign({}, options); opts.producerGroup = MixAll.CLIENT_INNER_PRODUCER_GROUP; MQProducer.defaultProducer = new MQProducer(opts); await MQProducer.defaultProducer.ready(); return MQProducer.defaultProducer; } return MQProducer.defaultProducer; } } module.exports = MQProducer; // // Helper // // --------------- // function compare(routerA, routerB) { // if (routerA.brokerName > routerB.brokerName) { // return 1; // } else if (routerA.brokerName < routerB.brokerName) { // return -1; // } // return 0; // } ================================================ FILE: lib/producer/send_status.js ================================================ 'use strict'; module.exports = { // 消息发送成功 SEND_OK: 'SEND_OK', // 消息发送成功，但是服务器刷盘超时，消息已经进入服务器队列，只有此时服务器宕机，消息才会丢失 FLUSH_DISK_TIMEOUT: 'FLUSH_DISK_TIMEOUT', // 消息发送成功，但是服务器同步到Slave时超时，消息已经进入服务器队列，只有此时服务器宕机，消息才会丢失 FLUSH_SLAVE_TIMEOUT: 'FLUSH_SLAVE_TIMEOUT', // 消息发送成功，但是此时slave不可用，消息已经进入服务器队列，只有此时服务器宕机，消息才会丢失 SLAVE_NOT_AVAILABLE: 'SLAVE_NOT_AVAILABLE', }; ================================================ FILE: lib/producer/topic_publish_info.js ================================================ 'use strict'; const crypto = require('crypto'); class TopicPublishInfo { constructor() { this.orderTopic = false; this.haveTopicRouterInfo = false; this.messageQueueList = []; this.sendWhichQueue = Math.floor(Math.random() * Math.floor(Math.pow(2, 31))); // 打散 queue 选择起点 } /** * 是否可用 */ get ok() { return this.messageQueueList && this.messageQueueList.length; } /** * 如果lastBrokerName不为null，则寻找与其不同的MessageQueue * @param {String} lastBrokerName - last broker name * @param {Object} message - message * @return {Object} queue */ selectOneMessageQueue(lastBrokerName, message) { let index, pos, mq; const shardingKey = message && message.shardingKey; let queueId = ''; if (shardingKey) { // 这里不应该是所有可用队列数字，应该是queue总数 // 但这里的 messageQueueList 是所有可写队列， golang和java客户端均为在queue总对恶劣中寻找位置 // 通过寻找所有可写queueId总数计算应该写入的queueId const queueIds = []; this.messageQueueList.forEach(queue => { if (!queueIds.includes(queue.queueId)) queueIds.push(queue.queueId); }); // 参考golang和java的sdk，没有规范确定的shardingKey找queueId的实现标准 // 若引入其他hash算法就显得比较笨重, 可以使用 nodejs 原生hash方法生成可计算的hash值 // Reference: https://github.com/apache/rocketmq/blob/c11ed78eeb73c23da4cf9a36a6bad493a1279210/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java#L382 // Reference: https://github.com/apache/rocketmq-client-go/blob/896a8a3453be4bcfcb5bfb5bb9139c4df6d4cdab/producer/selector.go#L112 const queueIndex = crypto .createHash('md5') .update(shardingKey) .digest() .readUInt32BE() % queueIds.length; queueId = queueIds[queueIndex] || ''; } if (!Number.isSafeInteger(this.sendWhichQueue)) { this.sendWhichQueue = 0; // 超出安全范围，重置为 0 } index = this.sendWhichQueue++; for (let i = 0, len = this.messageQueueList.length; i < len; i++) { pos = Math.abs(index++) % len; mq = this.messageQueueList[pos]; if (lastBrokerName && mq.brokerName === lastBrokerName) { continue; } if (queueId && mq.queueId !== queueId) { continue; } return mq; } return null; } } module.exports = TopicPublishInfo; ================================================ FILE: lib/protocol/command/opaque_generator.js ================================================ 'use strict'; const MAX_INT_31 = Math.pow(2, 31) - 10; let opaque = 0; exports.getNextOpaque = () => { if (opaque >= MAX_INT_31) { this.resetOpaque(); } return ++opaque; }; exports.resetOpaque = () => { opaque = 0; }; exports.getCurrentOpaque = () => opaque; ================================================ FILE: lib/protocol/command/remoting_command.js ================================================ 'use strict'; /* eslint no-bitwise: 0*/ const bytes = require('bytes'); const is = require('is-type-of'); const ByteBuffer = require('byte'); const OpaqueGenerator = require('./opaque_generator'); const REQUEST_COMMAND = 'REQUEST_COMMAND'; const RESPONSE_COMMAND = 'RESPONSE_COMMAND'; // 0, REQUEST_COMMAND // 1, RESPONSE_COMMAND const RPC_TYPE = 0; // 0, RPC // 1, Oneway const RPC_ONEWAY = 1; // V3_4_9 支持设置重试的 maxReconsumeTimes const MQ_VERSION = 121; const buf = new ByteBuffer({ size: bytes('1m') }); class RemotingCommand { /** * metaq 命令 * @param {Object} options * - code {Number} 命令代号 * - customHeader {Object} 自定义头部 * - flag {Number} 标识命令的属性 * - opaque {Number} 关联请求、响应的字段 * - remark {String} * - extFields {Object} 扩展字段 */ constructor(options) { this.code = options.code; this.customHeader = options.customHeader; this.flag = options.flag || 0; this.opaque = options.opaque || OpaqueGenerator.getNextOpaque(); this.remark = options.remark; this.extFields = options.extFields || {}; this.body = null; } get language() { return 'JAVA'; } get version() { return MQ_VERSION; } get type() { return this.isResponseType ? RESPONSE_COMMAND : REQUEST_COMMAND; } get isResponseType() { const bits = 1 << RPC_TYPE; return (this.flag & bits) === bits; } get isOnewayRPC() { const bits = 1 << RPC_ONEWAY; return (this.flag & bits) === bits; } markResponseType() { const bits = 1 << RPC_TYPE; this.flag |= bits; } markOnewayRPC() { const bits = 1 << RPC_ONEWAY; this.flag |= bits; } makeCustomHeaderToNet() { const customHeader = this.customHeader; if (!customHeader) { return; } this.extFields = this.extFields || {}; for (const key in customHeader) { const field = customHeader[key]; if (is.nullOrUndefined(field) || is.function(field)) { continue; } this.extFields[key] = field; } } decodeCommandCustomHeader() { return JSON.parse(JSON.stringify(this.extFields)); // 深拷贝 } buildHeader() { this.makeCustomHeaderToNet(); return Buffer.from(JSON.stringify({ code: this.code, language: this.language, opaque: this.opaque, flag: this.flag, version: this.version, remark: this.remark, extFields: this.extFields, }), 'utf8'); } encode() { buf.reset(); let length = 4; const headerData = this.buildHeader(); length += headerData.length; if (this.body) { length += this.body.length; } buf.putInt(length); buf.putInt(headerData.length); buf.put(headerData); if (this.body) { buf.put(this.body); } return buf.copy(); } /** * 创建 request 命令 * @param {Number} code - 命令代号 * @param {Object} customHeader - * @return {RemotingCommand} command */ static createRequestCommand(code, customHeader) { return new RemotingCommand({ code, customHeader, }); } /** * 创建 request 命令 * @param {Number} code - 命令代号 * @param {Number} opaque - * @param {String} remark - * @return {RemotingCommand} command */ static createResponseCommand(code, opaque, remark) { const command = new RemotingCommand({ code, opaque, remark: remark || 'not set any response code', }); command.markResponseType(); return command; } /** * 反序列化报文 * @param {Buffer} packet - * @return {RemotingCommand} command */ static decode(packet) { const customHeaderLength = packet.readInt32BE(4); const customHeaderData = packet.slice(8, 8 + customHeaderLength); let body; if (packet.length - 8 - customHeaderLength > 0) { body = packet.slice(8 + customHeaderLength); } const command = new RemotingCommand(JSON.parse(customHeaderData)); command.body = body; return command; } } module.exports = RemotingCommand; ================================================ FILE: lib/protocol/consume_type.js ================================================ 'use strict'; module.exports = { CONSUME_ACTIVELY: 'CONSUME_ACTIVELY', CONSUME_PASSIVELY: 'CONSUME_PASSIVELY', }; ================================================ FILE: lib/protocol/message_model.js ================================================ 'use strict'; module.exports = { BROADCASTING: 'BROADCASTING', CLUSTERING: 'CLUSTERING', }; ================================================ FILE: lib/protocol/perm_name.js ================================================ 'use strict'; /* eslint no-bitwise: 0 */ // var PERM_PRIORITY = 0x1 << 3; const PERM_READ = 0x1 << 2; const PERM_WRITE = 0x1 << 1; const PERM_INHERIT = 0x1 << 0; exports.PERM_READ = PERM_READ; exports.PERM_WRITE = PERM_WRITE; const isReadable = exports.isReadable = function(perm) { return (perm & PERM_READ) === PERM_READ; }; const isWriteable = exports.isWriteable = function(perm) { return (perm & PERM_WRITE) === PERM_WRITE; }; const isInherited = exports.isInherited = function(perm) { return (perm & PERM_INHERIT) === PERM_INHERIT; }; exports.perm2String = function(perm) { let str = ''; str += isReadable(perm) ? 'R' : '-'; str += isWriteable(perm) ? 'W' : '-'; str += isInherited(perm) ? 'X' : '-'; return str; }; ================================================ FILE: lib/protocol/request_code.js ================================================ 'use strict'; // Broker 发送消息 exports.SEND_MESSAGE = 10; // Broker 订阅消息 exports.PULL_MESSAGE = 11; // Broker 查询消息 exports.QUERY_MESSAGE = 12; // Broker 查询Broker Offset exports.QUERY_BROKER_OFFSET = 13; // Broker 查询Consumer Offset exports.QUERY_CONSUMER_OFFSET = 14; // Broker 更新Consumer Offset exports.UPDATE_CONSUMER_OFFSET = 15; // Broker 更新或者增加一个Topic exports.UPDATE_AND_CREATE_TOPIC = 17; // Broker 获取所有Topic的配置（Slave和Namesrv都会向Master请求此配置） exports.GET_ALL_TOPIC_CONFIG = 21; // Broker 获取所有Topic配置（Slave和Namesrv都会向Master请求此配置） exports.GET_TOPIC_CONFIG_LIST = 22; // Broker 获取所有Topic名称列表 exports.GET_TOPIC_NAME_LIST = 23; // Broker 更新Broker上的配置 exports.UPDATE_BROKER_CONFIG = 25; // Broker 获取Broker上的配置 exports.GET_BROKER_CONFIG = 26; // Broker 触发Broker删除文件 exports.TRIGGER_DELETE_FILES = 27; // Broker 获取Broker运行时信息 exports.GET_BROKER_RUNTIME_INFO = 28; // Broker 根据时间查询队列的Offset exports.SEARCH_OFFSET_BY_TIMESTAMP = 29; // Broker 查询队列最大Offset exports.GET_MAX_OFFSET = 30; // Broker 查询队列最小Offset exports.GET_MIN_OFFSET = 31; // Broker 查询队列最早消息对应时间 exports.GET_EARLIEST_MSG_STORETIME = 32; // Broker 根据消息ID来查询消息 exports.VIEW_MESSAGE_BY_ID = 33; // Broker Client向Client发送心跳，并注册自身 exports.HEART_BEAT = 34; // Broker Client注销 exports.UNREGISTER_CLIENT = 35; // Broker Consumer将处理不了的消息发回服务器 exports.CONSUMER_SEND_MSG_BACK = 36; // Broker Commit或者Rollback事务 exports.END_TRANSACTION = 37; // Broker 获取ConsumerId列表通过GroupName exports.GET_CONSUMER_LIST_BY_GROUP = 38; // Broker 主动向Producer回查事务状态 exports.CHECK_TRANSACTION_STATE = 39; // Broker Broker通知Consumer列表变化 exports.NOTIFY_CONSUMER_IDS_CHANGED = 40; // Broker Consumer向Master锁定队列 exports.LOCK_BATCH_MQ = 41; // Broker Consumer向Master解锁队列 exports.UNLOCK_BATCH_MQ = 42; // Broker 获取所有Consumer Offset exports.GET_ALL_CONSUMER_OFFSET = 43; // Broker 获取所有定时进度 exports.GET_ALL_DELAY_OFFSET = 45; // Namesrv 向Namesrv追加KV配置 exports.PUT_KV_CONFIG = 100; // Namesrv 从Namesrv获取KV配置 exports.GET_KV_CONFIG = 101; // Namesrv 从Namesrv获取KV配置 exports.DELETE_KV_CONFIG = 102; // Namesrv 注册一个Broker，数据都是持久化的，如果存在则覆盖配置 exports.REGISTER_BROKER = 103; // Namesrv 卸载一个Broker，数据都是持久化的 exports.UNREGISTER_BROKER = 104; // Namesrv 根据Topic获取Broker Name、队列数(包含读队列与写队列) exports.GET_ROUTEINTO_BY_TOPIC = 105; // Namesrv 获取注册到Name Server的所有Broker集群信息 exports.GET_BROKER_CLUSTER_INFO = 106; exports.UPDATE_AND_CREATE_SUBSCRIPTIONGROUP = 200; exports.GET_ALL_SUBSCRIPTIONGROUP_CONFIG = 201; exports.GET_TOPIC_STATS_INFO = 202; exports.GET_CONSUMER_CONNECTION_LIST = 203; exports.GET_PRODUCER_CONNECTION_LIST = 204; exports.WIPE_WRITE_PERM_OF_BROKER = 205; // 从Name Server获取完整Topic列表 exports.GET_ALL_TOPIC_LIST_FROM_NAMESERVER = 206; // 从Broker删除订阅组 exports.DELETE_SUBSCRIPTIONGROUP = 207; // 从Broker获取消费状态（进度） exports.GET_CONSUME_STATS = 208; // Suspend Consumer消费过程 exports.SUSPEND_CONSUMER = 209; // Resume Consumer消费过程 exports.RESUME_CONSUMER = 210; // 重置Consumer Offset exports.RESET_CONSUMER_OFFSET_IN_CONSUMER = 211; // 重置Consumer Offset exports.RESET_CONSUMER_OFFSET_IN_BROKER = 212; // 调整Consumer线程池数量 exports.ADJUST_CONSUMER_THREAD_POOL = 213; // 查询消息被哪些消费组消费 exports.WHO_CONSUME_THE_MESSAGE = 214; // 从Broker删除Topic配置 exports.DELETE_TOPIC_IN_BROKER = 215; // 从Namesrv删除Topic配置 exports.DELETE_TOPIC_IN_NAMESRV = 216; // Namesrv 通过 project 获取所有的 server ip 信息 exports.GET_KV_CONFIG_BY_VALUE = 217; // Namesrv 删除指定 project group 下的所有 server ip 信息 exports.DELETE_KV_CONFIG_BY_VALUE = 218; // 通过NameSpace获取所有的KV List exports.GET_KVLIST_BY_NAMESPACE = 219; // offset 重置 exports.RESET_CONSUMER_CLIENT_OFFSET = 220; // 客户端订阅消息 exports.GET_CONSUMER_STATUS_FROM_CLIENT = 221; // 通知 broker 调用 offset 重置处理 exports.INVOKE_BROKER_TO_RESET_OFFSET = 222; // 通知 broker 调用客户端订阅消息处理 exports.INVOKE_BROKER_TO_GET_CONSUMER_STATUS = 223; // Broker 查询topic被谁消费 // 2014-03-21 Add By shijia exports.QUERY_TOPIC_CONSUME_BY_WHO = 300; // 获取指定集群下的所有 topic // 2014-03-26 exports.GET_TOPICS_BY_CLUSTER = 224; // 向Broker注册Filter Server // 2014-04-06 Add By shijia exports.REGISTER_FILTER_SERVER = 301; // 向Filter Server注册Class // 2014-04-06 Add By shijia exports.REGISTER_MESSAGE_FILTER_CLASS = 302; // 根据 topic 和 group 获取消息的时间跨度 exports.QUERY_CONSUME_TIME_SPAN = 303; // 获取所有系统内置 Topic 列表 exports.GET_SYSTEM_TOPIC_LIST_FROM_NS = 304; exports.GET_SYSTEM_TOPIC_LIST_FROM_BROKER = 305; // 清理失效队列 exports.CLEAN_EXPIRED_CONSUMEQUEUE = 306; // 通过Broker查询Consumer内存数据 // 2014-07-19 Add By shijia exports.GET_CONSUMER_RUNNING_INFO = 307; // 查找被修正 offset (转发组件） exports.QUERY_CORRECTION_OFFSET = 308; // 通过Broker直接向某个Consumer发送一条消息，并立刻消费，返回结果给broker，再返回给调用方 // 2014-08-11 Add By shijia exports.CONSUME_MESSAGE_DIRECTLY = 309; // Broker 发送消息，优化网络数据包 exports.SEND_MESSAGE_V2 = 310; // 单元化相关 topic exports.GET_UNIT_TOPIC_LIST = 311; // 获取含有单元化订阅组的 Topic 列表 exports.GET_HAS_UNIT_SUB_TOPIC_LIST = 312; // 获取含有单元化订阅组的非单元化 Topic 列表 exports.GET_HAS_UNIT_SUB_UNUNIT_TOPIC_LIST = 313; // 克隆某一个组的消费进度到新的组 exports.CLONE_GROUP_OFFSET = 314; // 查看Broker上的各种统计信息 exports.VIEW_BROKER_STATS_DATA = 315; ================================================ FILE: lib/protocol/response_code.js ================================================ 'use strict'; // 成功 exports.SUCCESS = 0; // 发生了未捕获异常 exports.SYSTEM_ERROR = 1; // 由于线程池拥堵，系统繁忙 exports.SYSTEM_BUSY = 2; // 请求代码不支持 exports.REQUEST_CODE_NOT_SUPPORTED = 3; // Broker 刷盘超时 exports.FLUSH_DISK_TIMEOUT = 10; // Broker 同步双写，Slave不可用 exports.SLAVE_NOT_AVAILABLE = 11; // Broker 同步双写，等待Slave应答超时 exports.FLUSH_SLAVE_TIMEOUT = 12; // Broker 消息非法 exports.MESSAGE_ILLEGAL = 13; // Broker, Namesrv 服务不可用，可能是正在关闭或者权限问题 exports.SERVICE_NOT_AVAILABLE = 14; // Broker, Namesrv 版本号不支持 exports.VERSION_NOT_SUPPORTED = 15; // Broker, Namesrv 无权限执行此操作，可能是发、收、或者其他操作 exports.NO_PERMISSION = 16; // Broker, Topic不存在 exports.TOPIC_NOT_EXIST = 17; // Broker, Topic已经存在，创建Topic exports.TOPIC_EXIST_ALREADY = 18; // Broker 拉消息未找到（请求的Offset等于最大Offset，最大Offset无对应消息） exports.PULL_NOT_FOUND = 19; // Broker 可能被过滤，或者误通知等 exports.PULL_RETRY_IMMEDIATELY = 20; // Broker 拉消息请求的Offset不合法，太小或太大 exports.PULL_OFFSET_MOVED = 21; // Broker 查询消息未找到 exports.QUERY_NOT_FOUND = 22; // Broker 订阅关系解析失败 exports.SUBSCRIPTION_PARSE_FAILED = 23; // Broker 订阅关系不存在 exports.SUBSCRIPTION_NOT_EXIST = 24; // Broker 订阅关系不是最新的 exports.SUBSCRIPTION_NOT_LATEST = 25; // Broker 订阅组不存在 exports.SUBSCRIPTION_GROUP_NOT_EXIST = 26; // Producer 事务应该被提交 exports.TRANSACTION_SHOULD_COMMIT = 200; // Producer 事务应该被回滚 exports.TRANSACTION_SHOULD_ROLLBACK = 201; // Producer 事务状态未知 exports.TRANSACTION_STATE_UNKNOW = 202; // Producer ProducerGroup错误 exports.TRANSACTION_STATE_GROUP_WRONG = 203; // 单元化消息，需要设置 buyerId exports.NO_BUYER_ID = 204; // 单元化消息，非本单元消息 exports.NOT_IN_CURRENT_UNIT = 205; // Consumer不在线 exports.CONSUMER_NOT_ONLINE = 206; // Consumer消费消息超时 exports.CONSUME_MSG_TIMEOUT = 207; ================================================ FILE: lib/remoting_client.js ================================================ 'use strict'; const assert = require('assert'); const Base = require('sdk-base'); const logger = require('./logger'); const Channel = require('./channel'); const defaultOptions = { logger, responseTimeout: 30000, }; class RemotingClient extends Base { /** * rocketmq remoting client * @param {Object} options * - {HttpClient} httpclient - http request client * - {Object} [logger] - log module * - {Number} [responseTimeout] - tcp response timeout * @class */ constructor(options) { assert(options.onsAddr || options.nameSrv, '[RemotingClient] options.onsAddr or options.nameSrv one of them is required'); assert(options.httpclient, '[RemotingClient] options.httpclient is required'); super(Object.assign({ initMethod: 'init' }, defaultOptions, options)); this._index = 0; this._inited = false; this._namesrvAddrList = []; this._channels = new Map(); } /** * @property {HttpClient} RemotingClient#httpclient */ get httpclient() { return this.options.httpclient; } /** * @property {Object} RemotingClient#logger */ get logger() { return this.options.logger; } /** * @property {Number} RemotingClient#responseTimeout */ get responseTimeout() { return this.options.responseTimeout; } get unitName() { return this.options.unitName; } /** * start the client * @return {void} */ async init() { // get name server address at first await this.updateNameServerAddressList(); this._inited = true; this.logger.info('[mq:remoting_client] remoting client started'); } /** * close the client * @return {void} */ async close() { if (!this._inited) { return; } // wait all channel close await Promise.all(Array.from(this._channels.keys()).map(addr => { return new Promise(resolve => { const channel = this._channels.get(addr); if (channel && channel.isOK) { channel.once('close', resolve); channel.close(); } else { resolve(); } this._channels.delete(addr); }); })); this._inited = false; this.emit('close'); this.removeAllListeners(); this.logger.info('[mq:remoting_client] remoting client is closed'); } /** * default error handler * @param {Error} err - error object * @return {void} */ error(err) { this.emit('error', err); } async handleClose(addr, channel) { if (this._channels.get(addr) && this._channels.get(addr).clientId === channel.clientId) { this._channels.delete(addr); } await this.updateNameServerAddressListThrottle(); } async updateNameServerAddressListThrottle() { if (this._updatingNameServerAddressList) { // if it is in the process of updating and there is a new update task // it needs to be recorded at this time and update again at the end to avoid missing the last update. this._needToUpdatingNameServerAddressList = true; return; } this._updatingNameServerAddressList = true; let updateError; try { await this.updateNameServerAddressList(); } catch (e) { updateError = e; } this._updatingNameServerAddressList = false; if (this._needToUpdatingNameServerAddressList) { this._needToUpdatingNameServerAddressList = false; this.updateNameServerAddressListThrottle(); } if (updateError) { throw updateError; } } /** * fetch name server address list * @return {Promise} */ async updateNameServerAddressList() { if (this.options.nameSrv) { this._namesrvAddrList = Array.isArray(this.options.nameSrv) ? this.options.nameSrv : [ this.options.nameSrv ]; return; } let url = this.options.onsAddr; if (this.unitName) { url = url + '-' + this.unitName + '?nofix=1'; } let retries = 3; let ret; while (retries-- > 0) { ret = await this.httpclient.request(url, { timeout: this.options.connectTimeout || 10000, }); if (ret.status === 200) { const addrs = ret.data.toString().trim(); const newList = addrs.split(';'); if (newList.length) { this._namesrvAddrList = newList; } this.logger.info('[mq:remoting_client] fetch name server addresses successfully, address: %s', addrs); return; } } throw new Error('[mq:remoting_client] fetch name server addresses failed, ret.statusCode: ' + ret.status); } /** * invoke command * @param {String} addr - server address * @param {RemotingCommand} command - remoting command * @param {Number} [timeout] - response timeout * @return {Promise} response */ async invoke(addr, command, timeout) { if (!this._inited) { await this.ready(); } return await this.getAndCreateChannel(addr) .invoke(command, timeout || this.responseTimeout); } /** * invoke command for namesrv, return success at least once success * @param {RemotingCommand} command - remoting command * @param {Number} [timeout] - response timeout * @return {Object} response */ async invokeForNameSrvAtLeastOnce(command, timeout) { let response = {}; let count = this._namesrvAddrList.length; let err = null; while (count--) { try { response = await this.invoke(null, command, timeout); // 有一个成功即无需抛 invoke 错误 err = null; break; } catch (e) { err = e; this.logger.warn(err); } } if (err) { throw err; } return response; } /** * invoke command without response * @param {String} addr - server address * @param {RemotingCommand} command - remoting command * @return {Promise} */ async invokeOneway(addr, command) { if (!this._inited) { await this.ready(); } await this.getAndCreateChannel(addr).invokeOneway(command); } /** * get request channel from address * @param {String} [addr] - server address * @return {Promise} request channel */ getAndCreateChannel(addr) { if (!addr) { this._index = ++this._index % this._namesrvAddrList.length; addr = this._namesrvAddrList[this._index]; } let channel = this._channels.get(addr); if (channel && channel.isOK) { return channel; } channel = new Channel(addr, this.options); this._channels.set(addr, channel); channel.once('close', this.handleClose.bind(this, addr, channel)); channel.on('error', err => this.error(err)); channel.on('request', (request, address) => this.emit('request', request, address)); return channel; } } module.exports = RemotingClient; ================================================ FILE: lib/store/index.js ================================================ 'use strict'; exports.LocalFileOffsetStore = require('./local_file'); exports.LocalMemoryOffsetStore = require('./local_memory'); exports.RemoteBrokerOffsetStore = require('./remote_broker'); ================================================ FILE: lib/store/local_file.js ================================================ 'use strict'; const fs = require('fs'); const path = require('path'); const osenv = require('osenv'); const mkdirp = require('mkdirp'); const is = require('is-type-of'); const Base = require('sdk-base'); const ReadOffsetType = require('./read_offset_type'); const localOffsetStoreDir = path.join(osenv.home(), '.rocketmq_offsets_node'); class LocalFileOffsetStore extends Base { /** * 消费进度存储到Consumer本地 * @param {MQClient} mqClient - * @param {String} groupName 分组名 * @class */ constructor(mqClient, groupName) { super(); this.mqClient = mqClient; this.groupName = groupName; this.offsetTable = new Map(); this.storePath = path.join(localOffsetStoreDir, mqClient.clientId, groupName, 'offsets.json'); mkdirp.sync(path.dirname(this.storePath)); } get logger() { return this.mqClient.logger; } /** * 加载Offset */ async load() { const data = await this.readLocalOffset(); if (data && data.offsetTable) { for (const key in data.offsetTable) { if (is.number(data.offsetTable[key])) { this.offsetTable.set(key, data.offsetTable[key]); } } } } /** * 更新消费进度，存储到内存 * @param {MessageQueue} messageQueue - * @param {Number} offset 新的进度 * @param {Boolean} increaseOnly 是否强制覆盖 * @return {void} */ updateOffset(messageQueue, offset, increaseOnly) { increaseOnly = increaseOnly || false; if (!messageQueue) { return; } let prev = -1; if (this.offsetTable.has(messageQueue.key)) { prev = this.offsetTable.get(messageQueue.key); } if (prev >= offset && increaseOnly) { return; } this.offsetTable.set(messageQueue.key, offset); this.logger.info('[mq:LocalFileOffsetStore] update offset for messageQueue: %s, current offset: %d, prev offset: %s, increaseOnly: %s', messageQueue.key, offset, prev, increaseOnly); } /** * 从本地缓存读取消费进度 * @param {MessageQueue} messageQueue - * @param {String} type 消费类型（从哪里开始消费） * @return {void} */ async readOffset(messageQueue, type) { if (!messageQueue) { return -1; } switch (type) { case ReadOffsetType.MEMORY_FIRST_THEN_STORE: case ReadOffsetType.READ_FROM_MEMORY: { const offset = this.offsetTable.get(messageQueue.key); if (!is.nullOrUndefined(offset)) { return offset; } break; } case ReadOffsetType.READ_FROM_STORE: { const data = await this.readLocalOffset(); if (data && data.offsetTable) { const offset = data.offsetTable[messageQueue.key]; if (is.number(offset)) { this.updateOffset(messageQueue, offset, false); return offset; } } break; } default: break; } return -1; } /** * 持久化全部消费进度，可能持久化本地或者远端Broker * @param {Array} mqs - * @return {void} */ async persistAll(mqs) { if (mqs && mqs.length) { const data = { offsetTable: {}, }; const newTable = new Map(); for (const messageQueue of mqs) { const offset = this.offsetTable.get(messageQueue.key) || -1; data.offsetTable[messageQueue.key] = offset; newTable.set(messageQueue.key, offset); } this.offsetTable = newTable; fs.writeFileSync(this.storePath, JSON.stringify(data)); } } /* eslint-disable no-empty-function */ /** * 持久化消费进度 * @return {void} */ async persist() {} /* eslint-enable no-empty-function */ /** * 删除不必要的MessageQueue offset * @param {Object} messageQueue 消息队列 * @return {void} */ removeOffset(messageQueue) { if (messageQueue) { this.offsetTable.delete(messageQueue.key); } } /** * 从本地文件读消费进度 * @return {void} */ async readLocalOffset() { if (!fs.existsSync(this.storePath)) { return null; } const content = fs.readFileSync(this.storePath); if (!content) { return null; } try { return JSON.parse(content.toString()); } catch (e) { e.message = 'readLocalOffset() failed' + e.message; this.emit('error', e); return null; } } } module.exports = LocalFileOffsetStore; ================================================ FILE: lib/store/local_memory.js ================================================ 'use strict'; const is = require('is-type-of'); const Base = require('sdk-base'); const ReadOffsetType = require('./read_offset_type'); class LocalMemoryOffsetStore extends Base { /** * 消费进度存储到Consumer本地 * @param {MQClient} mqClient - * @param {String} groupName 分组名 * @class */ constructor(mqClient, groupName) { super(); this.mqClient = mqClient; this.groupName = groupName; this.offsetTable = new Map(); } get logger() { return this.mqClient.logger; } /** * 更新消费进度，存储到内存 * @param {MessageQueue} messageQueue - * @param {Number} offset 新的进度 * @param {Boolean} increaseOnly 是否强制覆盖 * @return {void} */ updateOffset(messageQueue, offset, increaseOnly) { increaseOnly = increaseOnly || false; if (!messageQueue) { return; } let prev = -1; if (this.offsetTable.has(messageQueue.key)) { prev = this.offsetTable.get(messageQueue.key); } if (prev >= offset && increaseOnly) { return; } this.offsetTable.set(messageQueue.key, offset); this.logger.info('[mq:LocalMemoryOffsetStore] update offset for messageQueue: %s, current offset: %d, prev offset: %s, increaseOnly: %s', messageQueue.key, offset, prev, increaseOnly); } /** * 从本地缓存读取消费进度 * @param {MessageQueue} messageQueue - * @param {String} type 消费类型（从哪里开始消费） * @return {void} */ async readOffset(messageQueue, type) { if (!messageQueue) { return -1; } switch (type) { case ReadOffsetType.MEMORY_FIRST_THEN_STORE: case ReadOffsetType.READ_FROM_MEMORY: case ReadOffsetType.READ_FROM_STORE: { const offset = this.offsetTable.get(messageQueue.key); if (!is.nullOrUndefined(offset)) { return offset; } break; } default: break; } return -1; } /* eslint-disable no-empty-function */ async load() {} async persistAll() {} async persist() {} /* eslint-enable no-empty-function */ /** * 删除不必要的MessageQueue offset * @param {Object} messageQueue 消息队列 * @return {void} */ removeOffset(messageQueue) { if (messageQueue) { this.offsetTable.delete(messageQueue.key); } } } module.exports = LocalMemoryOffsetStore; ================================================ FILE: lib/store/read_offset_type.js ================================================ 'use strict'; module.exports = { // 只从Memory读取 READ_FROM_MEMORY: 'READ_FROM_MEMORY', // 只从存储层读取（本地或者远端） READ_FROM_STORE: 'READ_FROM_STORE', // 先从内存读，内存不存在再从存储层读 MEMORY_FIRST_THEN_STORE: 'MEMORY_FIRST_THEN_STORE', }; ================================================ FILE: lib/store/remote_broker.js ================================================ 'use strict'; const is = require('is-type-of'); const Base = require('sdk-base'); const gather = require('co-gather'); const ReadOffsetType = require('./read_offset_type'); class RemoteBrokerOffsetStore extends Base { /** * consume offset store at remote server * @param {MQClient} mqClient - mq client * @param {String} groupName - group name * @class */ constructor(mqClient, groupName) { super(); this.mqClient = mqClient; this.groupName = groupName; this.offsetTable = new Map(); } get logger() { return this.mqClient.logger; } /* eslint-disable no-empty-function */ /** * load the offset * @return {void} */ async load() {} /* eslint-enable no-empty-function */ /** * update consume offset * @param {MessageQueue} messageQueue - message queue * @param {Number} offset - new offset * @param {Boolean} increaseOnly - force override * @return {void} */ updateOffset(messageQueue, offset, increaseOnly) { increaseOnly = increaseOnly || false; if (!messageQueue) { return; } let prev = -1; if (this.offsetTable.has(messageQueue.key)) { prev = this.offsetTable.get(messageQueue.key); } if (prev >= offset && increaseOnly) { return; } this.offsetTable.set(messageQueue.key, offset); this.logger.info('[mq:RemoteBrokerOffsetStore] update offset for messageQueue: %s, current offset: %d, prev offset: %s, increaseOnly: %s', messageQueue.key, offset, prev, increaseOnly); } /** * read consume offset * @param {MessageQueue} messageQueue - message queue * @param {String} type - consume from where * @return {Promise} offset */ async readOffset(messageQueue, type) { if (!messageQueue) { return -1; } try { switch (type) { case ReadOffsetType.MEMORY_FIRST_THEN_STORE: case ReadOffsetType.READ_FROM_MEMORY: { const offset = this.offsetTable.get(messageQueue.key); if (!is.nullOrUndefined(offset)) { return offset; } break; } case ReadOffsetType.READ_FROM_STORE: { const brokerOffset = await this.fetchConsumeOffsetFromBroker(messageQueue); this.updateOffset(messageQueue, brokerOffset, false); return brokerOffset; } default: break; } } catch (err) { err.message = `RemoteBrokerOffsetStore.readOffset failed, topic: ${messageQueue.topic}, group: ${this.groupName}, queueId: ${messageQueue.queueId}, type: ${type} ` + err.message; this.emit('error', err); } return -1; } /** * persist all consume offset * @param {Array} mqs - all queues * @return {void} */ async persistAll(mqs) { if (!mqs || !mqs.length) { return; } const newTable = new Map(); const tasks = []; let info = 'persist all consume offset\n'; for (const messageQueue of mqs) { const offset = this.offsetTable.get(messageQueue.key) || -1; newTable.set(messageQueue.key, offset); info += `\t- topic="${messageQueue.topic}", brokerName="${messageQueue.brokerName}", queueId="${messageQueue.queueId}", consumer offset => ${offset}\n`; tasks.push(() => this.updateConsumeOffsetToBroker(messageQueue, offset)); } this.logger.info('[mq:RemoteBrokerOffsetStore] %s', info); this.offsetTable = newTable; const ret = await gather(tasks); const errors = ret.filter(item => item.isError); for (const err of errors) { this.emit('error', err); } } /** * persist consume offset * @param {MessageQueue} messageQueue - message queue * @return {void} */ async persist(messageQueue) { const offset = this.offsetTable.get(messageQueue.key); this.logger.info('[mq:RemoteBrokerOffsetStore] persist consume offset of [%s], offset: %d', messageQueue.key, offset); if (is.number(offset)) { await this.updateConsumeOffsetToBroker(messageQueue, offset); } } /** * remove message queue * @param {MessageQueue} messageQueue - message queue * @return {void} */ removeOffset(messageQueue) { if (messageQueue) { this.offsetTable.delete(messageQueue.key); } } /** * fetch consume offset from broker * @param {MessageQueue} messageQueue - message queue * @return {Promise} brokerOffset */ async fetchConsumeOffsetFromBroker(messageQueue) { const requestHeader = { topic: messageQueue.topic, consumerGroup: this.groupName, queueId: messageQueue.queueId, }; const findBrokerResult = await this.findBrokerAddressInAdmin(messageQueue); return await this.mqClient.queryConsumerOffset(findBrokerResult.brokerAddr, requestHeader, 1000 * 5); } /** * update consume offset to broker * @param {MessageQueue} messageQueue - message queue * @param {Number} offset - offset * @return {void} */ async updateConsumeOffsetToBroker(messageQueue, offset) { const requestHeader = { topic: messageQueue.topic, consumerGroup: this.groupName, queueId: messageQueue.queueId, commitOffset: offset, }; const findBrokerResult = await this.findBrokerAddressInAdmin(messageQueue); // use oneway, cause this action may cost time. await this.mqClient.updateConsumerOffsetOneway(findBrokerResult.brokerAddr, requestHeader); } async findBrokerAddressInAdmin(messageQueue) { let findBrokerResult = this.mqClient.findBrokerAddressInAdmin(messageQueue.brokerName); if (!findBrokerResult) { await this.mqClient.updateTopicRouteInfoFromNameServer(messageQueue.topic); findBrokerResult = this.mqClient.findBrokerAddressInAdmin(messageQueue.brokerName); if (!findBrokerResult) { throw new Error(`The broker[${messageQueue.brokerName}] not exist`); } } return findBrokerResult; } } module.exports = RemoteBrokerOffsetStore; ================================================ FILE: lib/utils/index.js ================================================ 'use strict'; /* eslint no-bitwise:0 */ const zlib = require('zlib'); const MixAll = require('../mix_all'); const is = require('is-type-of'); // 压缩 exports.compress = function(bs) { return zlib.deflateSync(bs); }; // 解压 exports.uncompress = function(bs) { return zlib.inflateSync(bs); }; // 解析日期（注意时区为当地时区） exports.parseDate = function(str) { return new Date( Number(str.slice(0, 4)), Number(str.slice(4, 6)) - 1, Number(str.slice(6, 8)), Number(str.slice(8, 10)), Number(str.slice(10, 12)), Number(str.slice(12))); }; // 返回日期时间格式，精度到秒 // 格式如下：20131223051900 exports.timeMillisToHumanString = function(time) { const date = is.date(time) ? time : new Date(time); return '' + date.getFullYear() + zeroize(date.getMonth() + 1) + zeroize(date.getDate()) + zeroize(date.getHours()) + zeroize(date.getMinutes()) + zeroize(date.getSeconds()); }; // 获取字符串的哈希值 exports.hashCode = function(str) { // s[0]*31^(n-1) + s[1]*31^(n-2) + ... + s[n-1] let hash = 0; const len = str.length; for (let i = 0; i < len; i++) { hash = 31 * hash + str.charCodeAt(i) << 0; } return hash & 4294967295; }; // 重置重试消息的 Topic exports.resetRetryTopic = function(msg, consumerGroup) { const groupTopic = MixAll.getRetryTopic(consumerGroup); if (msg.topic === groupTopic) { msg.topic = msg.retryTopic; } }; function zeroize(value, length) { if (!length) { length = 2; } value = String(value); let zeros = ''; for (let i = 0; i < length - value.length; i++) { zeros += '0'; } return zeros + value; } ================================================ FILE: lib/utils/message_sys_flag.js ================================================ 'use strict'; /* eslint no-bitwise: 0 */ exports.CompressedFlag = 1 << 0; exports.MultiTagsFlag = 1 << 1; /** * 7 6 5 4 3 2 1 0 * SysFlag 事务相关，从左属，2与3 */ exports.TransactionNotType = 0 << 2; exports.TransactionPreparedType = 1 << 2; exports.TransactionCommitType = 2 << 2; exports.TransactionRollbackType = 3 << 2; exports.getTransactionValue = function(flag) { return flag & exports.TransactionRollbackType; }; exports.resetTransactionValue = function(flag, type) { return flag & ~exports.TransactionRollbackType | type; }; exports.clearCompressedFlag = function(flag) { return flag & ~exports.CompressedFlag; }; ================================================ FILE: lib/utils/pull_sys_flag.js ================================================ 'use strict'; /* eslint no-bitwise: 0 */ const FLAG_COMMIT_OFFSET = 1 << 0; const FLAG_SUSPEND = 1 << 1; const FLAG_SUBSCRIPTION = 1 << 2; const FLAG_CLASS_FILTER = 1 << 3; exports.buildSysFlag = function(commitOffset, suspend, subscription, classFilter) { let flag = 0; if (commitOffset) { flag |= FLAG_COMMIT_OFFSET; } if (suspend) { flag |= FLAG_SUSPEND; } if (subscription) { flag |= FLAG_SUBSCRIPTION; } if (classFilter) { flag |= FLAG_CLASS_FILTER; } return flag; }; exports.clearCommitOffsetFlag = function(sysFlag) { return sysFlag & ~FLAG_COMMIT_OFFSET; }; exports.hasCommitOffsetFlag = function(sysFlag) { return (sysFlag & FLAG_COMMIT_OFFSET) === FLAG_COMMIT_OFFSET; }; exports.hasSuspendFlag = function(sysFlag) { return (sysFlag & FLAG_SUSPEND) === FLAG_SUSPEND; }; exports.hasSubscriptionFlag = function(sysFlag) { return (sysFlag & FLAG_SUBSCRIPTION) === FLAG_SUBSCRIPTION; }; exports.hasClassFilterFlag = function(sysFlag) { return (sysFlag & FLAG_CLASS_FILTER) === FLAG_CLASS_FILTER; }; ================================================ FILE: package.json ================================================ { "name": "ali-ons", "version": "3.12.2", "description": "Aliyun Open Notification Service Client", "main": "lib/index.js", "files": [ "lib" ], "scripts": { "pkgfiles": "egg-bin pkgfiles --check", "lint": "eslint . --ext .js", "test": "npm run lint && npm run test-local", "test-local": "TEST_TIMEOUT=120000 egg-bin test", "cov": "TEST_TIMEOUT=120000 egg-bin cov", "ci": "npm run pkgfiles && npm run lint && npm run cov" }, "repository": { "type": "git", "url": "git+https://github.com/ali-sdk/ali-ons.git" }, "keywords": [ "ons", "aliyun", "ali-sdk" ], "author": "gxcsoccer ", "license": "MIT", "bugs": { "url": "https://github.com/ali-sdk/ali-ons/issues" }, "homepage": "https://github.com/ali-sdk/ali-ons#readme", "dependencies": { "address": "^1.1.2", "binary-search-insert": "^1.0.3", "byte": "^2.0.0", "bytes": "^3.1.0", "co-gather": "^1.0.1", "debug": "^4.3.1", "is-type-of": "^1.2.1", "JSON2": "^0.1.0", "long": "^4.0.0", "mkdirp": "^1.0.4", "mz-modules": "^2.1.0", "osenv": "^0.1.5", "sdk-base": "^3.6.0", "tcp-base": "^3.1.0", "utility": "^1.17.0" }, "devDependencies": { "egg-bin": "^4.16.1", "eslint": "^7.25.0", "eslint-config-egg": "^9.0.0", "mm": "^3.2.0", "nyc": "^17.1.0", "urllib": "^2.37.1" }, "engines": { "node": ">= 8.0.0" } } ================================================ FILE: test/allocate_message_queue_averagely.test.js ================================================ 'use strict'; const assert = require('assert'); const AllocateMessageQueueAveragely = require('../lib/consumer/rebalance/allocate_message_queue_averagely'); describe('test/allocate_message_queue_averagely.test.js', function() { const allocator = new AllocateMessageQueueAveragely(); it('should allocate queue averagely', function() { assert(allocator.name === 'AVG'); let result = allocator.allocate('test', '1', [ 1, 2, 3, 4, 5, 6, 7, 8 ], [ '1', '2' ]); assert.deepEqual(result, [ 1, 2, 3, 4 ]); result = allocator.allocate('test', '2', [ 1, 2, 3, 4, 5, 6, 7, 8 ], [ '1', '2' ]); assert.deepEqual(result, [ 5, 6, 7, 8 ]); }); it('should allocate ok, indivisible', function() { let result = allocator.allocate('test', '1', [ 1, 2, 3, 4, 5, 6, 7, 8 ], [ '1', '2', '3' ]); assert.deepEqual(result, [ 1, 2, 3 ]); result = allocator.allocate('test', '2', [ 1, 2, 3, 4, 5, 6, 7, 8 ], [ '1', '2', '3' ]); assert.deepEqual(result, [ 4, 5, 6 ]); result = allocator.allocate('test', '3', [ 1, 2, 3, 4, 5, 6, 7, 8 ], [ '1', '2', '3' ]); assert.deepEqual(result, [ 7, 8 ]); }); it('should allocate ok, indivisible 2', function() { let result = allocator.allocate('test', '1', [ 1, 2, 3, 4, 5 ], [ '1', '2' ]); assert.deepEqual(result, [ 1, 2, 3 ]); result = allocator.allocate('test', '2', [ 1, 2, 3, 4, 5 ], [ '1', '2' ]); assert.deepEqual(result, [ 4, 5 ]); }); it('should allocate ok, indivisible 3', function() { let result = allocator.allocate('test', '1', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3' ]); assert.deepEqual(result, [ 1, 2 ]); result = allocator.allocate('test', '2', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3' ]); assert.deepEqual(result, [ 3, 4 ]); result = allocator.allocate('test', '3', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3' ]); assert.deepEqual(result, [ 5 ]); }); it('should allocate ok, indivisible 4', function() { let result = allocator.allocate('test', '1', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4' ]); assert.deepEqual(result, [ 1, 2 ]); result = allocator.allocate('test', '2', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4' ]); assert.deepEqual(result, [ 3 ]); result = allocator.allocate('test', '3', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4' ]); assert.deepEqual(result, [ 4 ]); result = allocator.allocate('test', '4', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4' ]); assert.deepEqual(result, [ 5 ]); }); it('should allocate ok, indivisible 5', function() { let result = allocator.allocate('test', '1', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5' ]); assert.deepEqual(result, [ 1 ]); result = allocator.allocate('test', '2', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5' ]); assert.deepEqual(result, [ 2 ]); result = allocator.allocate('test', '3', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5' ]); assert.deepEqual(result, [ 3 ]); result = allocator.allocate('test', '4', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5' ]); assert.deepEqual(result, [ 4 ]); result = allocator.allocate('test', '5', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5' ]); assert.deepEqual(result, [ 5 ]); }); it('should allocate ok, indivisible 6', function() { let result = allocator.allocate('test', '1', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5', '6' ]); assert.deepEqual(result, [ 1 ]); result = allocator.allocate('test', '2', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5', '6' ]); assert.deepEqual(result, [ 2 ]); result = allocator.allocate('test', '3', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5', '6' ]); assert.deepEqual(result, [ 3 ]); result = allocator.allocate('test', '4', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5', '6' ]); assert.deepEqual(result, [ 4 ]); result = allocator.allocate('test', '5', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5', '6' ]); assert.deepEqual(result, [ 5 ]); result = allocator.allocate('test', '6', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5', '6' ]); assert.deepEqual(result, []); }); it('should throw error if currentId is null', function() { assert.throws(() => { allocator.allocate('test', null, [ 1, 2, 3, 4, 5, 6, 7, 8 ], [ '1', '2' ]); }, /currentCID is empty/); }); it('should throw error if mqAll is empty', function() { assert.throws(() => { allocator.allocate('test', '1', [], [ '1', '2' ]); }, /mqAll is null or mqAll empty/); }); it('should throw error if cidAll is empty', function() { assert.throws(() => { allocator.allocate('test', '1', [ 1, 2, 3, 4, 5, 6, 7, 8 ], []); }, /cidAll is null or cidAll empty/); }); it('should get empty array if currentId not exists in cidAll', function() { const result = allocator.allocate('test', 'not exists', [ 1, 2, 3, 4, 5, 6, 7, 8 ], [ '1', '2' ]); assert.deepEqual(result, []); }); }); ================================================ FILE: test/channel.test.js ================================================ 'use strict'; const mm = require('mm'); const net = require('net'); const assert = require('assert'); const urllib = require('urllib'); const sleep = require('mz-modules/sleep'); const Channel = require('../lib/channel'); const config = require('../example/config'); const RequestCode = require('../lib/protocol/request_code'); const ResponseCode = require('../lib/protocol/response_code'); const RemotingCommand = require('../lib/protocol/command/remoting_command'); const localIp = require('address').ip(); const { nameSrv } = require('../example/config'); const TOPIC = config.topic; describe('test/channel.test.js', function() { const onsAddr = 'http://onsaddr-internet.aliyun.com/rocketmq/nsaddr4client-internet'; let address; before(function(done) { urllib.request(onsAddr, function(err, data, result) { if (err) { return done(err); } if (result && result.status === 200) { address = data.toString().trim().split(';')[0]; done(); } else { done(new Error('no addresses')); } }); }); afterEach(mm.restore); it('should connect ok', () => { const channel = new Channel(nameSrv); return channel.ready(); }); it('should close ok', done => { const channel = new Channel(nameSrv); channel.ready(err => { if (err) { done(err); return; } assert(channel._socket); channel.close(); }); channel.on('close', () => { assert(!channel._socket); done(); }); }); it('should invoke ok', async () => { const channel = new Channel(nameSrv, config); await channel.ready(); const res = await channel.invoke(new RemotingCommand({ code: RequestCode.GET_ROUTEINTO_BY_TOPIC, customHeader: { topic: TOPIC, }, }), 5000); assert(res); assert(res.body); channel.close(); }); it('should invoke ok though channel not ready', async () => { const channel = new Channel(address); const res = await channel.invoke(new RemotingCommand({ code: RequestCode.GET_ROUTEINTO_BY_TOPIC, customHeader: { topic: TOPIC, }, }), 5000); assert(res); assert(res.remark && res.remark.includes('__accessKey is blank')); channel.close(); }); it('should invoke oneway ok', async () => { const channel = new Channel(nameSrv); await channel.invokeOneway(new RemotingCommand({ code: RequestCode.GET_KV_CONFIG_BY_VALUE, customHeader: { namespace: 'PROJECT_CONFIG', key: localIp, }, })); await channel.ready(); }); it('should close if timeout', async () => { const server = net.createServer(); server.listen(); const { port } = server.address(); await sleep(100); const channel = new Channel(`127.0.0.1:${port}`); await assert.rejects(async () => { await channel.invoke(new RemotingCommand({ code: RequestCode.GET_ROUTEINTO_BY_TOPIC, customHeader: { topic: TOPIC, }, }), 5000); }, { name: 'ResponseTimeoutError', }); await sleep(100); assert(!channel._socket); server.close(); }); // it('should emit error if connect failed', function(done) { // done = pedding(done, 2); // const channel = new Channel('100.100.100.100:9876'); // channel.once('close', function() { // console.log('channel closed') // done(); // }); // channel.once('error', function(err) { // console.log('channel error', err); // should.exist(err); // done(); // }); // }); it('should get error response', async () => { const channel = new Channel('100.100.100.100:9876'); try { await channel.invoke(new RemotingCommand({ code: RequestCode.GET_KV_CONFIG_BY_VALUE, customHeader: { namespace: 'PROJECT_CONFIG', key: localIp, }, })); } catch (err) { console.log(err); } channel.close(); }); // it.only('should throw error if command is invalid', function*() { // const channel = new Channel(nameSrv); // let isError = false; // try { // yield channel.invoke('invalid command', 5000); // } catch (err) { // isError = true; // err.name.should.equal('MQInvalidCommandError'); // err.message.should.equal('Invalid command'); // } // isError.should.be.ok(); // channel.close(); // channel.close(); // }); it('should clearupInvokes after connection close', async () => { const channel = new Channel(nameSrv); await channel.ready(); channel.close(); let isError = false; try { await channel.invoke(new RemotingCommand({ code: RequestCode.GET_ROUTEINTO_BY_TOPIC, customHeader: { topic: 'NOT_EXISTS', }, }), 5000); } catch (err) { isError = true; assert(err.message === `The socket was closed. (address: ${nameSrv})`); } assert(isError); }); // it('should throw error if invoke after close', function(done) { // const channel = new Channel(nameSrv); // channel.on('close', function() { // channel.invoke(new RemotingCommand({ // code: RequestCode.GET_ROUTEINTO_BY_TOPIC, // customHeader: { // topic: 'NOT_EXISTS' // } // }), 5000, function(err, res) { // should.exist(err); // err.message.should.equal(`The socket is already closed`); // done(); // }); // }); // channel.close(); // }); it('should invoke response ok', async () => { const channel = new Channel(nameSrv); await channel.ready(); await channel.invokeOneway(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, 1, '')); }); it('should throw error if invoke timeout', async () => { const channel = new Channel(nameSrv); let isError = false; try { await channel.invoke(new RemotingCommand({ code: RequestCode.GET_KV_CONFIG_BY_VALUE, customHeader: { namespace: 'PROJECT_CONFIG', key: localIp, }, }), 1); } catch (err) { isError = true; assert(err.name === 'ResponseTimeoutError'); assert(err.message === `Server no response in 1ms, address#${nameSrv}`); } channel.close(); assert(isError); }); // it('should throw error if connection close by remote server', function(done) { // const channel = new Channel(nameSrv); // yield channel.ready(); // channel.invoke(new RemotingCommand({ // code: RequestCode.GET_KV_CONFIG_BY_VALUE, // customHeader: { // namespace: 'PROJECT_CONFIG', // key: localIp, // }, // }), 1, function(err, res) { // should.exist(err); // channel.invokes.size.should.equal(0); // err.name.should.equal('MQSocketClosedError'); // err.message.should.containEql(`${address} conn closed by remote server`); // done(); // }); // channel.close(); // }); }); ================================================ FILE: test/consumer/rebalance/allocate_message_queue_averagely.test.js ================================================ 'use strict'; const assert = require('assert'); const AllocateMessageQueueAveragely = require('../../../lib/consumer/rebalance/allocate_message_queue_averagely'); describe('test/allocate_message_queue_averagely.test.js', function() { const allocator = new AllocateMessageQueueAveragely(); it('should allocate queue averagely', function() { assert(allocator.name === 'AVG'); let result = allocator.allocate('test', '1', [ 1, 2, 3, 4, 5, 6, 7, 8 ], [ '1', '2' ]); assert.deepEqual(result, [ 1, 2, 3, 4 ]); result = allocator.allocate('test', '2', [ 1, 2, 3, 4, 5, 6, 7, 8 ], [ '1', '2' ]); assert.deepEqual(result, [ 5, 6, 7, 8 ]); }); it('should allocate ok, indivisible', function() { let result = allocator.allocate('test', '1', [ 1, 2, 3, 4, 5, 6, 7, 8 ], [ '1', '2', '3' ]); assert.deepEqual(result, [ 1, 2, 3 ]); result = allocator.allocate('test', '2', [ 1, 2, 3, 4, 5, 6, 7, 8 ], [ '1', '2', '3' ]); assert.deepEqual(result, [ 4, 5, 6 ]); result = allocator.allocate('test', '3', [ 1, 2, 3, 4, 5, 6, 7, 8 ], [ '1', '2', '3' ]); assert.deepEqual(result, [ 7, 8 ]); }); it('should allocate ok, indivisible 2', function() { let result = allocator.allocate('test', '1', [ 1, 2, 3, 4, 5 ], [ '1', '2' ]); assert.deepEqual(result, [ 1, 2, 3 ]); result = allocator.allocate('test', '2', [ 1, 2, 3, 4, 5 ], [ '1', '2' ]); assert.deepEqual(result, [ 4, 5 ]); }); it('should allocate ok, indivisible 3', function() { let result = allocator.allocate('test', '1', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3' ]); assert.deepEqual(result, [ 1, 2 ]); result = allocator.allocate('test', '2', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3' ]); assert.deepEqual(result, [ 3, 4 ]); result = allocator.allocate('test', '3', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3' ]); assert.deepEqual(result, [ 5 ]); }); it('should allocate ok, indivisible 4', function() { let result = allocator.allocate('test', '1', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4' ]); assert.deepEqual(result, [ 1, 2 ]); result = allocator.allocate('test', '2', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4' ]); assert.deepEqual(result, [ 3 ]); result = allocator.allocate('test', '3', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4' ]); assert.deepEqual(result, [ 4 ]); result = allocator.allocate('test', '4', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4' ]); assert.deepEqual(result, [ 5 ]); }); it('should allocate ok, indivisible 5', function() { let result = allocator.allocate('test', '1', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5' ]); assert.deepEqual(result, [ 1 ]); result = allocator.allocate('test', '2', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5' ]); assert.deepEqual(result, [ 2 ]); result = allocator.allocate('test', '3', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5' ]); assert.deepEqual(result, [ 3 ]); result = allocator.allocate('test', '4', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5' ]); assert.deepEqual(result, [ 4 ]); result = allocator.allocate('test', '5', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5' ]); assert.deepEqual(result, [ 5 ]); }); it('should allocate ok, indivisible 6', function() { let result = allocator.allocate('test', '1', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5', '6' ]); assert.deepEqual(result, [ 1 ]); result = allocator.allocate('test', '2', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5', '6' ]); assert.deepEqual(result, [ 2 ]); result = allocator.allocate('test', '3', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5', '6' ]); assert.deepEqual(result, [ 3 ]); result = allocator.allocate('test', '4', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5', '6' ]); assert.deepEqual(result, [ 4 ]); result = allocator.allocate('test', '5', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5', '6' ]); assert.deepEqual(result, [ 5 ]); result = allocator.allocate('test', '6', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5', '6' ]); assert.deepEqual(result, []); }); it('should throw error if currentId is null', function() { assert.throws(() => { allocator.allocate('test', null, [ 1, 2, 3, 4, 5, 6, 7, 8 ], [ '1', '2' ]); }, /currentCID is empty/); }); it('should throw error if mqAll is empty', function() { assert.throws(() => { allocator.allocate('test', '1', [], [ '1', '2' ]); }, /mqAll is null or mqAll empty/); }); it('should throw error if cidAll is empty', function() { assert.throws(() => { allocator.allocate('test', '1', [ 1, 2, 3, 4, 5, 6, 7, 8 ], []); }, /cidAll is null or cidAll empty/); }); it('should get empty array if currentId not exists in cidAll', function() { const result = allocator.allocate('test', 'not exists', [ 1, 2, 3, 4, 5, 6, 7, 8 ], [ '1', '2' ]); assert.deepEqual(result, []); }); }); ================================================ FILE: test/index.test.js ================================================ 'use strict'; const mm = require('mm'); const path = require('path'); const osenv = require('osenv'); const assert = require('assert'); const httpclient = require('urllib'); const utils = require('../lib/utils'); const Message = require('../').Message; const Consumer = require('../').Consumer; const Producer = require('../').Producer; const sleep = require('mz-modules/sleep'); const rimraf = require('mz-modules/rimraf'); const config = require('../example/config'); const MixAll = require('../lib/mix_all'); const TOPIC = config.topic; const localOffsetStoreDir = path.join(osenv.home(), '.rocketmq_offsets_node'); describe('test/index.test.js', () => { describe('API', () => { let producer; let consumer; before(async () => { producer = new Producer(Object.assign({ httpclient, retryAnotherBrokerWhenNotStoreOK: true, compressMsgBodyOverHowmuch: 10, }, config)); await producer.ready(); consumer = new Consumer(Object.assign({ httpclient, rebalanceInterval: 1000, isBroadcast: true, }, config)); await consumer.ready(); consumer.subscribe(TOPIC, async msg => { console.log('message receive ------------> ', msg.body.toString()); }); }); after(async () => { await producer.close(); await consumer.close(); }); afterEach(mm.restore); // it.skip('it should create topic ok', function*() { // yield producer.createTopic('TBW102', 'XXX', 8); // }); it('should select another broker if one failed', async () => { mm(producer, 'sendKernelImpl', async () => { mm.restore(); return { sendStatus: 'FLUSH_DISK_TIMEOUT', }; }); const msg = new Message(TOPIC, // topic 'TagA', // tag 'Hello TagA !!! ' // body ); let sendResult = await producer.send(msg); assert(sendResult && sendResult.msgId); mm(producer, 'sendKernelImpl', async () => { mm.restore(); const err = new Error('mock err'); err.name = 'MQClientException'; err.code = 205; throw err; }); sendResult = await producer.send(msg); assert(sendResult && sendResult.msgId); }); it('should tryToFindTopicPublishInfo if brokerAddr not found', async () => { const msg = new Message(TOPIC, // topic 'TagA', // tag 'Hello TagA !!! ' // body ); let sendResult = await producer.send(msg); assert(sendResult && sendResult.msgId); mm(producer._mqClient, 'findBrokerAddressInPublish', () => { mm.restore(); return null; }); sendResult = await producer.send(msg); assert(sendResult && sendResult.msgId); mm(producer._mqClient, 'findBrokerAddressInPublish', () => { return null; }); let isError = false; try { await producer.send(msg); } catch (err) { isError = true; assert(/The broker\[.+\] not exist/i.test(err.message)); assert(err.name === 'MQClientException'); } assert(isError); mm.restore(); isError = false; try { await producer.send(new Message('%RETRY%' + TOPIC, // topic 'TagA', // tag 'Hello TagA !!! ' // body )); } catch (err) { isError = true; assert(err.name === 'MQClientException'); } assert(isError); }); it('should emit error if compress failed', done => { mm(utils, 'compress', function() { throw new Error('mock error'); }); const msg = new Message(TOPIC, // topic 'TagA', // tag 'Hello TagA !!! sds' // body ); producer.on('error', err => { assert(err.name === 'MetaQCompressError'); done(); }); (async () => { try { const sendResult = await producer.send(msg); assert(sendResult && sendResult.msgId); console.log('over'); } catch (err) { console.log(err); } })(); }); it('should updateProcessQueueTableInRebalance ok', async () => { await sleep(3000); await consumer.rebalanceByTopic(TOPIC); const size = consumer.processQueueTable.size; assert(size > 0); const key = Array.from(consumer.processQueueTable.keys())[0]; const obj = consumer.processQueueTable.get(key); const processQueue = obj.processQueue; processQueue.lastPullTimestamp = 10000; await consumer.rebalanceByTopic(TOPIC); assert(consumer.processQueueTable.size === size); await consumer.updateProcessQueueTableInRebalance(TOPIC, []); assert(consumer.processQueueTable.size === 0); await consumer.updateProcessQueueTableInRebalance(MixAll.getRetryTopic(consumer.consumerGroup), []); assert(consumer.processQueueTable.size === 0); }); it('should computePullFromWhere ok', async () => { mm(consumer._offsetStore, 'readOffset', async () => { return -1; }); mm(consumer, 'consumeFromWhere', 'CONSUME_FROM_LAST_OFFSET'); let offset = await consumer.computePullFromWhere({ topic: '%RETRY%__', }); assert(offset === 0); mm(consumer, 'consumeFromWhere', 'CONSUME_FROM_TIMESTAMP'); mm(consumer._mqClient, 'maxOffset', async () => { return 1000; }); offset = await consumer.computePullFromWhere({ topic: '%RETRY%__', }); assert(offset === 1000); mm.error(consumer._mqClient, 'maxOffset'); offset = await consumer.computePullFromWhere({ topic: '%RETRY%__', }); assert(offset === -1); mm(consumer._offsetStore, 'readOffset', async () => { return 100; }); offset = await consumer.computePullFromWhere({ topic: 'TP', }); assert(offset === 100); mm(consumer, 'consumeFromWhere', 'CONSUME_FROM_FIRST_OFFSET'); offset = await consumer.computePullFromWhere({ topic: 'TP', }); assert(offset === 100); mm(consumer, 'consumeFromWhere', 'NOT_EXISTS'); offset = await consumer.computePullFromWhere({ topic: 'TP', }); assert(offset === -1); }); it('should support namespace', () => { mm(producer, 'namespace', 'xxx'); mm(consumer, 'namespace', 'xxx'); assert(consumer.consumerGroup === `xxx%${config.consumerGroup}`); assert(producer.producerGroup === `xxx%${config.producerGroup}`); assert(consumer.formatTopic(`%RETRY%${consumer.consumerGroup}`) === `%RETRY%${consumer.consumerGroup}`); assert(producer.formatTopic('TEST_TOPIC') === 'xxx%TEST_TOPIC'); assert(consumer.formatTopic('TEST_TOPIC') === 'xxx%TEST_TOPIC'); }); }); // 广播消费 describe('broadcast', () => { [ 'CONSUME_FROM_LAST_OFFSET_AND_FROM_MIN_WHEN_BOOT_FIRST', 'CONSUME_FROM_MIN_OFFSET', 'CONSUME_FROM_MAX_OFFSET', 'CONSUME_FROM_LAST_OFFSET', 'CONSUME_FROM_FIRST_OFFSET', 'CONSUME_FROM_TIMESTAMP', ].forEach(consumeFromWhere => { let consumer; let producer; describe(`consumeFromWhere => ${consumeFromWhere}`, () => { before(() => { return rimraf(localOffsetStoreDir); }); before(async () => { consumer = new Consumer(Object.assign({ httpclient, consumeFromWhere, isBroadcast: true, persistent: true, rebalanceInterval: 2000, }, config)); producer = new Producer(Object.assign({ httpclient, }, config)); await consumer.ready(); await producer.ready(); }); after(async () => { await producer.close(); const originFn = consumer._offsetStore.persistAll; mm(consumer._offsetStore, 'persistAll', async mqs => { assert(mqs && mqs.length); return await originFn.call(consumer._offsetStore, mqs); }); await consumer.close(); }); afterEach(mm.restore); it('should subscribe message ok', async () => { let msgId; const received = new Set(); consumer.subscribe(TOPIC, 'TagA', async msg => { console.log('message receive ------------> ', msg.msgId, msg.body.toString()); assert(msg.tags !== 'TagB'); received.add(msg.msgId); if (msgId && received.has(msgId)) { assert(msg.body.toString() === 'Hello MetaQ !!! '); consumer.emit('TagA'); } }); await sleep(5000); let msg = new Message(TOPIC, // topic 'TagB', // tag 'Hello MetaQ !!! ' // body ); let sendResult = await producer.send(msg); assert(sendResult && sendResult.msgId); msg = new Message(TOPIC, // topic 'TagA', // tag 'Hello MetaQ !!! ' // body ); sendResult = await producer.send(msg); assert(sendResult && sendResult.msgId); msgId = sendResult.msgId; console.log('send message success,', sendResult.msgId); if (!received.has(msgId)) { await consumer.await('TagA'); } }); it.skip('should viewMessage ok', async () => { const msg = new Message(TOPIC, // topic 'TagA', // tag 'Hello MetaQ !!! ' // body ); const sendResult = await producer.send(msg); assert(sendResult && sendResult.msgId); await sleep(3000); const message = await consumer.viewMessage(sendResult.msgId); assert(message.body.toString() === 'Hello MetaQ !!! '); }); }); }); }); // 集群消费 describe('cluster', () => { let consumer; let producer; beforeEach(async () => { consumer = new Consumer(Object.assign({ httpclient, isBroadcast: false, }, config)); await consumer.ready(); producer = new Producer(Object.assign({ httpclient, }, config)); await producer.ready(); }); afterEach(async () => { await consumer.close(); await producer.close(); }); afterEach(mm.restore); it('should subscribe message ok', async () => { await sleep(3000); const msg = new Message(TOPIC, // topic 'TagA', // tag 'Hello MetaQ !!! ' // body ); const sendResult = await producer.send(msg); assert(sendResult && sendResult.msgId); const msgId = sendResult.msgId; console.log(sendResult); await new Promise(r => { consumer.subscribe(TOPIC, '*', async msg => { if (msg.msgId === msgId) { assert(msg.body.toString() === 'Hello MetaQ !!! '); r(); } }); }); }); it.skip('should subscribe message with SQL92 expression type ok', async () => { // 公有云未开启 SQL 过滤 await sleep(3000); const msg = new Message(TOPIC, // topic 'TagA', // tag 'Hello MetaQ !!! ' // body ); msg.properties.a = '1'; const sendResult = await producer.send(msg); assert(sendResult && sendResult.msgId); const msgId = sendResult.msgId; console.log(sendResult); await new Promise(r => { consumer.subscribe(TOPIC, { expressionType: 'SQL92', subString: 'a IS NOT NULL', }, async msg => { if (msg.msgId === msgId) { assert(msg.body.toString() === 'Hello MetaQ !!! '); r(); } }); }); }); }); describe('process exception', () => { let consumer; let producer; const logger = { info() {}, warn() {}, error(...args) { console.error(...args); }, debug() {}, }; beforeEach(async () => { consumer = new Consumer(Object.assign({ httpclient, logger, rebalanceInterval: 2000, maxReconsumeTimes: 2, }, config)); producer = new Producer(Object.assign({ httpclient, logger, }, config)); await consumer.ready(); await producer.ready(); }); afterEach(async () => { await producer.close(); await consumer.close(); }); it('should not correctTags if process queue not empty', () => { let done = false; mm(consumer._offsetStore, 'updateOffset', () => { done = true; }); consumer.correctTagsOffset({ processQueue: { msgCount: 0, }, }); assert(done); done = true; mm.restore(); mm(consumer._offsetStore, 'updateOffset', () => { done = false; }); consumer.correctTagsOffset({ processQueue: { msgCount: 1, }, }); assert(done); mm.restore(); }); it('should retry(consume later) if process failed', async () => { let msgId; consumer.subscribe(TOPIC, '*', async msg => { console.warn('message receive ------------> ', msg.body.toString()); if (msg.msgId === msgId || msg.originMessageId === msgId) { assert(msg.body.toString() === 'Hello MetaQ !!! '); if (msg.reconsumeTimes === 0) { throw new Error('mock error'); } consumer.emit('*'); } }); await sleep(5000); const msg = new Message(TOPIC, // topic 'TagA', // tag 'Hello MetaQ !!! ' // body ); const sendResult = await producer.send(msg); assert(sendResult && sendResult.msgId); msgId = sendResult.msgId; await consumer.await('*'); }); it('broker should drop message if reconsumeTimes gt maxReconsumeTimes', async () => { let msgId; let reconsumeTimes = 0; const randomNumber = Math.floor(Math.random() * 100) + 1; const msgBody = 'Hello MetaQ !!!' + randomNumber; consumer.subscribe(TOPIC, '*', async msg => { console.warn('message receive ------------> ', msg.body.toString(), msg.reconsumeTimes); if (msg.msgId === msgId || msg.originMessageId && (msg.originMessageId === msgId)) { assert(msg.body.toString() === msgBody); reconsumeTimes = msg.reconsumeTimes; throw new Error('mock error'); } }); await sleep(5000); const msg = new Message(TOPIC, // topic 'TagA', // tag msgBody // body ); const sendResult = await producer.send(msg); assert(sendResult && sendResult.msgId); msgId = sendResult.msgId; await sleep(60000); assert(reconsumeTimes === 2); }); it('should retry(retry message) if process failed', async () => { let msgId; mm(consumer._mqClient, 'consumerSendMessageBack', async () => { throw new Error('mock error'); }); consumer.subscribe(TOPIC, '*', async msg => { console.warn('message receive ------------> ', msg.msgId, msg.originMessageId, msg.body.toString()); if (msg.msgId === msgId || msg.originMessageId === msgId) { assert(msg.body.toString() === 'Hello MetaQ !!! '); if (msg.reconsumeTimes === 0) { throw new Error('mock error'); } consumer.emit('*'); } }); await sleep(5000); const msg = new Message(TOPIC, // topic 'TagA', // tag 'Hello MetaQ !!! ' // body ); const sendResult = await producer.send(msg); assert(sendResult && sendResult.msgId); msgId = sendResult.msgId; console.log('msgId -->', msgId); await consumer.await('*'); }); it('should retry if process return ACTION_RETRY', async () => { let msgId; consumer.subscribe(TOPIC, '*', async msg => { console.warn('message receive ------------> ', msg.body.toString()); if (msg.msgId === msgId || msg.originMessageId === msgId) { assert(msg.body.toString() === 'retry'); if (msg.reconsumeTimes === 0) { return Consumer.ACTION_RETRY; } if (msg.reconsumeTimes === 1) { consumer.emit('*'); } } }); await sleep(5000); const msg = new Message(TOPIC, // topic 'TagRetry', // tag 'retry' // body ); const sendResult = await producer.send(msg); assert(sendResult && sendResult.msgId); msgId = sendResult.msgId; await consumer.await('*'); }); it('should retry(fall to local) if process failed', async () => { let msgId; mm(consumer, 'sendMessageBack', async () => { throw new Error('mock error'); }); consumer.subscribe(TOPIC, '*', async msg => { console.warn('message receive ------------> ', msg.body.toString()); if (msg.msgId === msgId) { assert(msg.body.toString() === 'Hello MetaQ !!! '); if (msg.reconsumeTimes === 0) { throw new Error('mock error'); } consumer.emit('*'); } }); await sleep(5000); const msg = new Message(TOPIC, // topic 'TagA', // tag 'Hello MetaQ !!! ' // body ); const sendResult = await producer.send(msg); assert(sendResult && sendResult.msgId); msgId = sendResult.msgId; await consumer.await('*'); }); }); describe('flow control', () => { let consumer; let producer; before(async () => { await rimraf(localOffsetStoreDir); consumer = new Consumer(Object.assign({ httpclient, isBroadcast: true, rebalanceInterval: 2000, pullThresholdForQueue: 1, consumeFromWhere: 'CONSUME_FROM_FIRST_OFFSET', pullTimeDelayMillsWhenFlowControl: 5000, }, config)); producer = new Producer(Object.assign({ httpclient, }, config)); await consumer.ready(); await producer.ready(); }); after(async () => { await producer.close(); await consumer.close(); }); it('should retry if process failed', async () => { let count = 20; while (count--) { const msg = new Message(TOPIC, // topic 'TagA', // tag 'Hello MetaQ !!! ' // body ); const sendResult = await producer.send(msg); assert(sendResult && sendResult.msgId); } consumer.subscribe(TOPIC, '*', async (msg, mq, pq) => { console.log('message receive ------------> ', msg.body.toString()); const msgCount = pq.msgCount; await sleep(10000); // 不再拉取 try { console.info('----------------> origin msgCount %d, current msgCount %d', msgCount, pq.msgCount); assert(msgCount === 1 || pq.msgCount === msgCount); consumer.emit('over'); } catch (err) { consumer.emit('error', err); } }); await Promise.race([ consumer.await('over'), consumer.await('error'), ]); }); }); describe('delay consume message', () => { let producer; let consumer; let consumeTime = 0; // 允许的误差时间 const deviationTime = 4000; before(async () => { producer = new Producer(Object.assign({ httpclient, }, config)); await producer.ready(); consumer = new Consumer(Object.assign({ httpclient, }, config)); await consumer.ready(); }); after(async () => { await producer.close(); await consumer.close(); }); it.skip('should receive message with specified time', async () => { const delayTime = 10000; const body = 'hello delay message at ' + Date.now(); const msg = new Message(config.topic, 'TagDelay', body); const produceTime = Date.now(); msg.setStartDeliverTime(produceTime + delayTime); const result = await producer.send(msg); console.log(result); consumer.subscribe(config.topic, 'TagDelay', async msg => { console.log('message receive ------------> ', msg.msgId, msg.body.toString()); if (body === msg.body.toString()) { consumeTime = Date.now(); consumer.emit('consumed'); } }); await consumer.await('consumed'); assert(consumeTime - produceTime <= delayTime + deviationTime && consumeTime - produceTime >= delayTime - deviationTime); }); }); }); ================================================ FILE: test/index_namesrv.test.js ================================================ 'use strict'; const mm = require('mm'); const path = require('path'); const osenv = require('osenv'); const assert = require('assert'); const httpclient = require('urllib'); const utils = require('../lib/utils'); const Message = require('../').Message; const Consumer = require('../').Consumer; const Producer = require('../').Producer; const sleep = require('mz-modules/sleep'); const rimraf = require('mz-modules/rimraf'); const rawConfig = require('../example/config'); const config = Object.assign({ nameSrv: '112.124.141.191:80', }, rawConfig); const MixAll = require('../lib/mix_all'); const TOPIC = config.topic; const localOffsetStoreDir = path.join(osenv.home(), '.rocketmq_offsets_node'); describe('test/index_namesrv.test.js', () => { describe('API', () => { let producer; let consumer; before(async () => { producer = new Producer(Object.assign({ httpclient, retryAnotherBrokerWhenNotStoreOK: true, compressMsgBodyOverHowmuch: 10, }, config)); await producer.ready(); consumer = new Consumer(Object.assign({ httpclient, rebalanceInterval: 1000, isBroadcast: true, }, config)); await consumer.ready(); consumer.subscribe(TOPIC, async msg => { console.log('message receive ------------> ', msg.body.toString()); }); }); after(async () => { await producer.close(); await consumer.close(); }); afterEach(mm.restore); // it.skip('it should create topic ok', function*() { // yield producer.createTopic('TBW102', 'XXX', 8); // }); it('should select another broker if one failed', async () => { mm(producer, 'sendKernelImpl', async () => { mm.restore(); return { sendStatus: 'FLUSH_DISK_TIMEOUT', }; }); const msg = new Message(TOPIC, // topic 'TagA', // tag 'Hello TagA !!! ' // body ); let sendResult = await producer.send(msg); assert(sendResult && sendResult.msgId); mm(producer, 'sendKernelImpl', async () => { mm.restore(); const err = new Error('mock err'); err.name = 'MQClientException'; err.code = 205; throw err; }); sendResult = await producer.send(msg); assert(sendResult && sendResult.msgId); }); it('should tryToFindTopicPublishInfo if brokerAddr not found', async () => { const msg = new Message(TOPIC, // topic 'TagA', // tag 'Hello TagA !!! ' // body ); let sendResult = await producer.send(msg); assert(sendResult && sendResult.msgId); mm(producer._mqClient, 'findBrokerAddressInPublish', () => { mm.restore(); return null; }); sendResult = await producer.send(msg); assert(sendResult && sendResult.msgId); mm(producer._mqClient, 'findBrokerAddressInPublish', () => { return null; }); let isError = false; try { await producer.send(msg); } catch (err) { isError = true; assert(/The broker\[.+\] not exist/i.test(err.message)); assert(err.name === 'MQClientException'); } assert(isError); mm.restore(); isError = false; try { await producer.send(new Message('%RETRY%' + TOPIC, // topic 'TagA', // tag 'Hello TagA !!! ' // body )); } catch (err) { isError = true; assert(err.name === 'MQClientException'); } assert(isError); }); it('should emit error if compress failed', done => { mm(utils, 'compress', function() { throw new Error('mock error'); }); const msg = new Message(TOPIC, // topic 'TagA', // tag 'Hello TagA !!! sds' // body ); producer.on('error', err => { assert(err.name === 'MetaQCompressError'); done(); }); (async () => { try { const sendResult = await producer.send(msg); assert(sendResult && sendResult.msgId); console.log('over'); } catch (err) { console.log(err); } })(); }); it('should updateProcessQueueTableInRebalance ok', async () => { await sleep(3000); await consumer.rebalanceByTopic(TOPIC); const size = consumer.processQueueTable.size; assert(size > 0); const key = Array.from(consumer.processQueueTable.keys())[0]; const obj = consumer.processQueueTable.get(key); const processQueue = obj.processQueue; processQueue.lastPullTimestamp = 10000; await consumer.rebalanceByTopic(TOPIC); assert(consumer.processQueueTable.size === size); await consumer.updateProcessQueueTableInRebalance(TOPIC, []); assert(consumer.processQueueTable.size === 0); await consumer.updateProcessQueueTableInRebalance(MixAll.getRetryTopic(consumer.consumerGroup), []); assert(consumer.processQueueTable.size === 0); }); it('should computePullFromWhere ok', async () => { mm(consumer._offsetStore, 'readOffset', async () => { return -1; }); mm(consumer, 'consumeFromWhere', 'CONSUME_FROM_LAST_OFFSET'); let offset = await consumer.computePullFromWhere({ topic: '%RETRY%__', }); assert(offset === 0); mm(consumer, 'consumeFromWhere', 'CONSUME_FROM_TIMESTAMP'); mm(consumer._mqClient, 'maxOffset', async () => { return 1000; }); offset = await consumer.computePullFromWhere({ topic: '%RETRY%__', }); assert(offset === 1000); mm.error(consumer._mqClient, 'maxOffset'); offset = await consumer.computePullFromWhere({ topic: '%RETRY%__', }); assert(offset === -1); mm(consumer._offsetStore, 'readOffset', async () => { return 100; }); offset = await consumer.computePullFromWhere({ topic: 'TP', }); assert(offset === 100); mm(consumer, 'consumeFromWhere', 'CONSUME_FROM_FIRST_OFFSET'); offset = await consumer.computePullFromWhere({ topic: 'TP', }); assert(offset === 100); mm(consumer, 'consumeFromWhere', 'NOT_EXISTS'); offset = await consumer.computePullFromWhere({ topic: 'TP', }); assert(offset === -1); }); }); // 广播消费 describe('broadcast', () => { [ 'CONSUME_FROM_LAST_OFFSET_AND_FROM_MIN_WHEN_BOOT_FIRST', 'CONSUME_FROM_MIN_OFFSET', 'CONSUME_FROM_MAX_OFFSET', 'CONSUME_FROM_LAST_OFFSET', 'CONSUME_FROM_FIRST_OFFSET', 'CONSUME_FROM_TIMESTAMP', ].forEach(consumeFromWhere => { let consumer; let producer; describe(`consumeFromWhere => ${consumeFromWhere}`, () => { before(() => { return rimraf(localOffsetStoreDir); }); before(async () => { consumer = new Consumer(Object.assign({ httpclient, consumeFromWhere, isBroadcast: true, persistent: true, rebalanceInterval: 2000, }, config)); producer = new Producer(Object.assign({ httpclient, }, config)); await consumer.ready(); await producer.ready(); }); after(async () => { await producer.close(); const originFn = consumer._offsetStore.persistAll; mm(consumer._offsetStore, 'persistAll', async mqs => { assert(mqs && mqs.length); return await originFn.call(consumer._offsetStore, mqs); }); await consumer.close(); }); afterEach(mm.restore); it('should subscribe message ok', async () => { let msgId; const received = new Set(); consumer.subscribe(TOPIC, 'TagA', async msg => { console.log('message receive ------------> ', msg.msgId, msg.body.toString()); assert(msg.tags !== 'TagB'); received.add(msg.msgId); if (msgId && received.has(msgId)) { assert(msg.body.toString() === 'Hello MetaQ !!! '); consumer.emit('TagA'); } }); await sleep(5000); let msg = new Message(TOPIC, // topic 'TagB', // tag 'Hello MetaQ !!! ' // body ); let sendResult = await producer.send(msg); assert(sendResult && sendResult.msgId); msg = new Message(TOPIC, // topic 'TagA', // tag 'Hello MetaQ !!! ' // body ); sendResult = await producer.send(msg); assert(sendResult && sendResult.msgId); msgId = sendResult.msgId; console.log('send message success,', sendResult.msgId); if (!received.has(msgId)) { await consumer.await('TagA'); } }); it.skip('should viewMessage ok', async () => { const msg = new Message(TOPIC, // topic 'TagA', // tag 'Hello MetaQ !!! ' // body ); const sendResult = await producer.send(msg); assert(sendResult && sendResult.msgId); await sleep(3000); const message = await consumer.viewMessage(sendResult.msgId); assert(message.body.toString() === 'Hello MetaQ !!! '); }); }); }); }); // 集群消费 describe('cluster', () => { let consumer; let producer; before(async () => { consumer = new Consumer(Object.assign({ httpclient, isBroadcast: false, }, config)); await consumer.ready(); producer = new Producer(Object.assign({ httpclient, }, config)); await producer.ready(); }); after(async () => { await consumer.close(); await producer.close(); }); afterEach(mm.restore); it('should subscribe message ok', async () => { await sleep(3000); const msg = new Message(TOPIC, // topic 'TagA', // tag 'Hello MetaQ !!! ' // body ); const sendResult = await producer.send(msg); assert(sendResult && sendResult.msgId); const msgId = sendResult.msgId; console.log(sendResult); await new Promise(r => { consumer.subscribe(TOPIC, '*', async msg => { if (msg.msgId === msgId) { assert(msg.body.toString() === 'Hello MetaQ !!! '); r(); } }); }); }); }); describe('process exception', () => { let consumer; let producer; beforeEach(async () => { consumer = new Consumer(Object.assign({ httpclient, rebalanceInterval: 2000, }, config)); producer = new Producer(Object.assign({ httpclient, }, config)); await consumer.ready(); await producer.ready(); }); afterEach(async () => { await producer.close(); await consumer.close(); }); it('should not correctTags if process queue not empty', () => { let done = false; mm(consumer._offsetStore, 'updateOffset', () => { done = true; }); consumer.correctTagsOffset({ processQueue: { msgCount: 0, }, }); assert(done); done = true; mm.restore(); mm(consumer._offsetStore, 'updateOffset', () => { done = false; }); consumer.correctTagsOffset({ processQueue: { msgCount: 1, }, }); assert(done); mm.restore(); }); it('should retry(consume later) if process failed', async () => { let msgId; consumer.subscribe(TOPIC, '*', async msg => { console.warn('message receive ------------> ', msg.body.toString()); if (msg.msgId === msgId || msg.originMessageId === msgId) { assert(msg.body.toString() === 'Hello MetaQ !!! '); if (msg.reconsumeTimes === 0) { throw new Error('mock error'); } consumer.emit('*'); } }); await sleep(5000); const msg = new Message(TOPIC, // topic 'TagA', // tag 'Hello MetaQ !!! ' // body ); const sendResult = await producer.send(msg); assert(sendResult && sendResult.msgId); msgId = sendResult.msgId; await consumer.await('*'); }); it('should retry(retry message) if process failed', async () => { let msgId; mm(consumer._mqClient, 'consumerSendMessageBack', async () => { throw new Error('mock error'); }); consumer.subscribe(TOPIC, '*', async msg => { console.warn('message receive ------------> ', msg.body.toString()); if (msg.msgId === msgId || msg.originMessageId === msgId) { assert(msg.body.toString() === 'Hello MetaQ !!! '); if (msg.reconsumeTimes === 0) { throw new Error('mock error'); } consumer.emit('*'); } }); await sleep(5000); const msg = new Message(TOPIC, // topic 'TagA', // tag 'Hello MetaQ !!! ' // body ); const sendResult = await producer.send(msg); assert(sendResult && sendResult.msgId); msgId = sendResult.msgId; await consumer.await('*'); }); it('should retry(fall to local) if process failed', async () => { let msgId; mm(consumer, 'sendMessageBack', async () => { throw new Error('mock error'); }); consumer.subscribe(TOPIC, '*', async msg => { console.warn('message receive ------------> ', msg.body.toString()); if (msg.msgId === msgId) { assert(msg.body.toString() === 'Hello MetaQ !!! '); if (msg.reconsumeTimes === 0) { throw new Error('mock error'); } consumer.emit('*'); } }); await sleep(5000); const msg = new Message(TOPIC, // topic 'TagA', // tag 'Hello MetaQ !!! ' // body ); const sendResult = await producer.send(msg); assert(sendResult && sendResult.msgId); msgId = sendResult.msgId; await consumer.await('*'); }); }); describe('flow control', () => { let consumer; let producer; before(async () => { await rimraf(localOffsetStoreDir); consumer = new Consumer(Object.assign({ httpclient, isBroadcast: true, rebalanceInterval: 2000, pullThresholdForQueue: 1, consumeFromWhere: 'CONSUME_FROM_FIRST_OFFSET', pullTimeDelayMillsWhenFlowControl: 5000, }, config)); producer = new Producer(Object.assign({ httpclient, }, config)); await consumer.ready(); await producer.ready(); }); after(async () => { await producer.close(); await consumer.close(); }); it('should retry if process failed', async () => { let count = 20; while (count--) { const msg = new Message(TOPIC, // topic 'TagA', // tag 'Hello MetaQ !!! ' // body ); const sendResult = await producer.send(msg); assert(sendResult && sendResult.msgId); } consumer.subscribe(TOPIC, '*', async (msg, mq, pq) => { console.log('message receive ------------> ', msg.body.toString()); const msgCount = pq.msgCount; await sleep(10000); // 不再拉取 try { console.info('----------------> origin msgCount %d, current msgCount %d', msgCount, pq.msgCount); assert(msgCount === 1 || pq.msgCount === msgCount); consumer.emit('over'); } catch (err) { consumer.emit('error', err); } }); await Promise.race([ consumer.await('over'), consumer.await('error'), ]); }); }); }); ================================================ FILE: test/message/message_decoder.test.js ================================================ 'use strict'; const assert = require('assert'); const ByteBuffer = require('byte'); const utils = require('../utils'); const Message = require('../../lib/message/message'); const MessageDecoder = require('../../lib/message/message_decoder'); const NAME_VALUE_SEPARATOR = String.fromCharCode(1); const PROPERTY_SEPARATOR = String.fromCharCode(2); const SYSTEM_PROP_KEY_STARTDELIVERTIME = '__STARTDELIVERTIME'; describe('test/message/message_decoder.test.js', function() { it('should decode message ok', function() { const buf = ByteBuffer.wrap(utils.bytes('message.bin')); const message = MessageDecoder.decode(buf); assert(message); assert(message.msgId === '0ADA91A6000029CC0000006500E59F3E'); assert.deepEqual(message.body, Buffer.from('{"room":"1","msg":"1"}')); assert(message.tags === 'TagA'); assert(!message.keys); assert(message.delayTimeLevel === 0); assert(!message.waitStoreMsgOK); assert(!message.buyerId); assert(message.topic === 'TopicTest'); }); it('should decode compress message ok', function() { const buf = ByteBuffer.wrap(utils.bytes('message_compress.bin')); const message = MessageDecoder.decode(buf); assert(message); assert(message.msgId === '0ADA91A6000029CC000000661A586D98'); assert.deepEqual(message.body, Buffer.from('Hello MetaQ !!!')); assert(message.tags, 'TagA'); assert(!message.keys); assert(message.delayTimeLevel === 0); assert(!message.waitStoreMsgOK); assert(!message.buyerId); assert(message.topic === 'TopicTest'); }); it('should decode message and not read body ok', function() { const buf = ByteBuffer.wrap(utils.bytes('message.bin')); const message = MessageDecoder.decode(buf, false); assert(message); assert(message.msgId === '0ADA91A6000029CC0000006500E59F3E'); assert(!message.body); assert(message.tags === 'TagA'); assert(!message.keys); assert(message.delayTimeLevel === 0); assert(!message.waitStoreMsgOK); assert(!message.buyerId); assert(message.topic === 'TopicTest'); }); it('should batch decode message ok', function() { const buf = ByteBuffer.wrap(utils.bytes('batch_message.bin')); const messages = MessageDecoder.decodes(buf); assert(messages.length === 3); assert(messages[0].msgId === '0ADA91A6000029CC000000659560A2AA'); assert.deepEqual(messages[0].body, Buffer.from('Hello MetaQ !!!')); assert(messages[1].msgId === '0ADA91A6000029CC000000659560D349'); assert(messages[2].msgId === '0ADA91A6000029CC000000659560D578'); }); it('should decodeMessageId ok', function() { const data = MessageDecoder.decodeMessageId('0ADA91A6000029CC0000006504009B6F'); assert(data.address === '10.218.145.166:10700'); assert(data.offset.toString() === '433858845551'); }); it('should messageProperties2String ok', function() { assert(MessageDecoder.messageProperties2String({ foo: 'bar', xxx: 'yyy', a: '中文', }) === `foo${NAME_VALUE_SEPARATOR}bar${PROPERTY_SEPARATOR}xxx${NAME_VALUE_SEPARATOR}yyy${PROPERTY_SEPARATOR}a${NAME_VALUE_SEPARATOR}中文${PROPERTY_SEPARATOR}`); }); it('should create message ok', function() { const message = new Message('fake_topic', 'fake body'); message.tags = 'TagB'; assert(message.properties.TAGS === 'TagB'); message.keys = [ 'xxx', 'yyy' ]; assert(message.properties.KEYS === 'xxx yyy'); message.delayTimeLevel = 1000; assert(message.delayTimeLevel === 1000); assert(message.properties.DELAY === '1000'); message.waitStoreMsgOK = true; assert(message.properties.WAIT === 'true'); message.buyerId = '123'; assert(message.properties.BUYER_ID === '123'); }); it('should create deliver time message ok', function() { const deliverTime = Date.now() + 3000; const message = new Message('fake_topic', 'fake body'); message.setStartDeliverTime(deliverTime); assert(message.getStartDeliverTime() === deliverTime); assert(message.properties[SYSTEM_PROP_KEY_STARTDELIVERTIME] === deliverTime); }); }); ================================================ FILE: test/mq_client.test.js ================================================ 'use strict'; const assert = require('assert'); const httpclient = require('urllib'); const ClientConfig = require('../lib/client_config'); const MQClient = require('../lib/mq_client'); const { consumerGroup, producerGroup, topic } = require('../example/config'); describe('test/mq_client.test.js', () => { const config = new ClientConfig(Object.assign({ httpclient }, require('../example/config'))); let client; before(async () => { client = MQClient.getAndCreateMQClient(config); client.unregisterProducer('CLIENT_INNER_PRODUCER'); return client.ready(); }); after(() => client.close()); it('should mqclient is singleton', () => { assert(MQClient.getAndCreateMQClient(config) === client); }); it('should registerConsumer ok', async () => { client.registerConsumer(consumerGroup, { subscriptions: new Map(), updateTopicSubscribeInfo() {}, isSubscribeTopicNeedUpdate() {}, doRebalance() {}, }); assert(client._consumerTable.size === 1); client.registerConsumer(consumerGroup, {}); assert(client._consumerTable.size === 1); await client.unregisterConsumer(consumerGroup); assert(client._consumerTable.size === 0); }); it('should registerProducer ok', async () => { client.registerProducer(producerGroup, { publishTopicList: new Map(), updateTopicPublishInfo() {}, isPublishTopicNeedUpdate() {}, }); assert(client._producerTable.size === 1); client.registerProducer(producerGroup, {}); assert(client._producerTable.size === 1); await client.unregisterProducer(producerGroup); assert(client._producerTable.size === 0); }); it('should not close if there are producer or consumer', async () => { client.registerConsumer(consumerGroup, { subscriptions: new Map(), updateTopicSubscribeInfo() {}, isSubscribeTopicNeedUpdate() {}, doRebalance() {}, }); await client.close(); await client.unregisterConsumer(consumerGroup); client.registerProducer(producerGroup, { publishTopicList: new Map(), updateTopicPublishInfo() {}, isPublishTopicNeedUpdate() {}, }); await client.close(); await client.unregisterProducer(producerGroup); }); it('should updateAllTopicRouterInfo ok', async () => { const subscriptions = new Map(); subscriptions.set(topic, { topic, subString: '*', classFilterMode: false, tagsSet: [], codeSet: [], subVersion: Date.now(), }); client.registerConsumer(consumerGroup, { subscriptions, updateTopicSubscribeInfo() {}, isSubscribeTopicNeedUpdate() {}, doRebalance() {}, }); client.registerConsumer('xxx', null); client.registerProducer(producerGroup, { publishTopicList: [ topic ], updateTopicPublishInfo() {}, isPublishTopicNeedUpdate() {}, }); client.registerProducer('xxx', null); await client.updateAllTopicRouterInfo(); await client.unregisterConsumer(consumerGroup); await client.unregisterProducer(producerGroup); try { await client.unregisterConsumer('xxx'); } catch (err) { assert(err.message.includes('xxx not created')); } await client.unregisterProducer('xxx'); }); it('should updateTopicRouteInfoFromNameServer ok', async () => { client.registerConsumer(config.consumerGroup, { subscriptions: new Map(), updateTopicSubscribeInfo() {}, isSubscribeTopicNeedUpdate() {}, doRebalance() {}, }); client.registerProducer(config.producerGroup, { publishTopicList: [ topic ], updateTopicPublishInfo() {}, isPublishTopicNeedUpdate() {}, }); await client.updateTopicRouteInfoFromNameServer(topic); let topicRouteData = client._topicRouteTable.get(topic); assert(topicRouteData); assert(topicRouteData.brokerDatas.length > 0); assert(client._brokerAddrTable.size > 0); await client.unregisterConsumer(config.consumerGroup); await client.unregisterProducer(config.producerGroup); await client.updateTopicRouteInfoFromNameServer(topic, true, { createTopicKey: topic, defaultTopicQueueNums: 8, }); topicRouteData = client._topicRouteTable.get(topic); assert(topicRouteData); assert(topicRouteData.brokerDatas.length > 0); assert(client._brokerAddrTable.size > 0); }); it('should topicRouteData2TopicPublishInfo ok', function() { let info = client._topicRouteData2TopicPublishInfo(topic, { orderTopicConf: 'xxx:8;yyy:8', }); assert(info.orderTopic); assert(info.messageQueueList.length === 16); info = client._topicRouteData2TopicPublishInfo(topic, { brokerDatas: [{ brokerAddrs: { 0: '10.10.10.10:1000', }, brokerName: 'xxx', }], filterServerTable: {}, queueDatas: [{ brokerName: 'xxx', perm: 6, readQueueNums: 8, topicSynFlag: 0, writeQueueNums: 8, }], }); assert(!info.orderTopic); assert(info.messageQueueList.length === 8); info = client._topicRouteData2TopicPublishInfo(topic, { brokerDatas: [{ brokerAddrs: { 1: '10.10.10.10:1000', }, brokerName: 'yyy', }], filterServerTable: {}, queueDatas: [{ brokerName: 'yyy', perm: 6, readQueueNums: 8, topicSynFlag: 0, writeQueueNums: 8, }], }); assert(!info.orderTopic); assert(info.messageQueueList.length === 0); }); it('should cleanOfflineBroker ok', function() { client._brokerAddrTable.set('fake-broker', { 0: '127.0.0.1:10910', }); client._cleanOfflineBroker(); assert(!client._brokerAddrTable.has('fake-broker')); }); it('should sendHeartbeatToAllBroker ok', async () => { await client.sendHeartbeatToAllBroker(); const subscriptions = new Map(); subscriptions.set(config.consumerGroup, { topic, subString: '*', classFilterMode: false, tagsSet: [], codeSet: [], subVersion: Date.now(), }); client.registerConsumer(config.consumerGroup, { subscriptions, updateTopicSubscribeInfo() {}, isSubscribeTopicNeedUpdate() {}, doRebalance() {}, }); client.registerProducer(config.producerGroup, { publishTopicList: [ topic ], updateTopicPublishInfo() {}, isPublishTopicNeedUpdate() {}, }); await client.updateTopicRouteInfoFromNameServer(topic); const topicRouteData = client._topicRouteTable.get(topic); assert(topicRouteData); assert(topicRouteData.brokerDatas.length > 0); assert(client._brokerAddrTable.size > 0); await client.sendHeartbeatToAllBroker(); await client.unregisterConsumer(config.consumerGroup); await client.unregisterProducer(config.producerGroup); }); it('should persistAllConsumerOffset & doRebalance ok', async () => { client.registerConsumer(config.consumerGroup, { // async persistConsumerOffset() { console.log('persistConsumerOffset'); }, async doRebalance() { console.log('doRebalance'); }, }); await client.persistAllConsumerOffset(); await client.doRebalance(); // await this.client.notifyConsumerIdsChanged(); await client.unregisterConsumer(config.consumerGroup); }); // it('should pull message ok', function(done) { // done = pedding(done, 2); // const request = { // consumerGroup: 'please_rename_unique_group_name_1', // }; // let i = 0; // this.client.registerConsumer('please_rename_unique_group_name_1', { // pullMessage: function(pullRequest) { // pullRequest.should.eql(request); // if (++i === 2) { // this.client.unregisterConsumer('please_rename_unique_group_name_1'); // } // done(); // }.bind(this), // }); // this.client.executePullRequestImmediately(request); // this.client.executePullRequestLater(request, 100); // }); }); ================================================ FILE: test/mq_client_api.test.js ================================================ 'use strict'; const assert = require('assert'); const address = require('address'); const httpclient = require('urllib'); const config = require('../example/config'); const MQClientAPI = require('../lib/mq_client_api'); const TOPIC = config.topic; describe('test/mq_client_api.test.js', function() { let client; before(() => { client = new MQClientAPI(Object.assign({ httpclient }, Object.assign({}, config, { nameSrv: [ config.nameSrv ], }))); return client.ready(); }); after(() => client.close()); it('should getProjectGroupByIp ok', () => { return client.getProjectGroupByIp(address.ip(), 3000); }); it('should getDefaultTopicRouteInfoFromNameServer ok', async () => { const res = await client.getDefaultTopicRouteInfoFromNameServer(TOPIC, 3000); assert(res); }); it('should getDefaultTopicRouteInfoFromNameServer for exception', async () => { let isError = false; try { await client.getDefaultTopicRouteInfoFromNameServer('NOT_EXIST_TOPIC', 3000); } catch (err) { isError = true; assert(err.name === 'MQClientException'); } assert(isError); }); it('should getTopicRouteInfoFromNameServer ok', async () => { const res = await client.getTopicRouteInfoFromNameServer(TOPIC, 3000); assert(res); }); it('should getTopicRouteInfoFromNameServer ok if old one is closed', async () => { client._namesrvAddrList.unshift('1.2.3.4:80', '2.3.4.5:80'); client._namesrvAddrList.push('6.7.8.9:80'); const res = await client.getTopicRouteInfoFromNameServer(TOPIC, 3000); assert(res); }); it('should getTopicRouteInfoFromNameServer ok if old one is closed empty list', async () => { client._namesrvAddrList = []; let isError = false; try { await client.getTopicRouteInfoFromNameServer(TOPIC, 3000); } catch (err) { isError = true; assert(err.name === 'MQClientException'); } assert(isError); }); }); ================================================ FILE: test/protocol/command/opaque_generator.test.js ================================================ 'use strict'; const assert = require('assert'); const OpaqueGenerator = require('../../../lib/protocol/command/opaque_generator'); describe('test/protocol/command/opaque_generator.test.js', function() { it('should get current & next opaque', function() { OpaqueGenerator.resetOpaque(); assert(OpaqueGenerator.getNextOpaque() === 1); assert(OpaqueGenerator.getCurrentOpaque() === 1); }); }); ================================================ FILE: test/protocol/command/remoting_command.test.js ================================================ 'use strict'; const assert = require('assert'); const RequestCode = require('../../../lib/protocol/request_code'); const ResponseCode = require('../../../lib/protocol/response_code'); const RemotingCommand = require('../../../lib/protocol/command/remoting_command'); describe('test/protocol/command/remoting_command.test.js', function() { it('should create request command ok', function() { const command = RemotingCommand.createRequestCommand(RequestCode.GET_KV_CONFIG_BY_VALUE, { namespace: 'PROJECT_CONFIG', key: '192.168.1.103', }); assert(command.type === 'REQUEST_COMMAND'); assert(!command.isResponseType); assert(!command.isOnewayRPC); command.markOnewayRPC(); assert(command.isOnewayRPC); command.makeCustomHeaderToNet(); assert.deepEqual(command.decodeCommandCustomHeader(), { namespace: 'PROJECT_CONFIG', key: '192.168.1.103', }); }); it('should create response command ok', function() { const command = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, 1); assert(command.type === 'RESPONSE_COMMAND'); assert(command.isResponseType); assert(!command.isOnewayRPC); assert(command.remark === 'not set any response code'); }); it('should encode command ok', function() { const request = RemotingCommand.createRequestCommand(RequestCode.GET_KV_CONFIG_BY_VALUE, { namespace: 'PROJECT_CONFIG', key: '192.168.1.103', }); request.opaque = 1; assert.deepEqual(request.encode(), Buffer.from('00000085000000817b22636f6465223a3231372c226c616e6775616765223a224a415641222c226f7061717565223a312c22666c6167223a302c2276657273696f6e223a3132312c226578744669656c6473223a7b226e616d657370616365223a2250524f4a4543545f434f4e464947222c226b6579223a223139322e3136382e312e313033227d7d', 'hex')); const response = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, 1); assert.deepEqual(response.encode(), Buffer.from('00000076000000727b22636f6465223a302c226c616e6775616765223a224a415641222c226f7061717565223a312c22666c6167223a312c2276657273696f6e223a3132312c2272656d61726b223a226e6f742073657420616e7920726573706f6e736520636f6465222c226578744669656c6473223a7b7d7d', 'hex')); }); it('should decode command ok', function() { const request = RemotingCommand.decode(Buffer.from('00000084000000807b22636f6465223a3231372c226c616e6775616765223a224a415641222c226f7061717565223a312c22666c6167223a302c2276657273696f6e223a37382c226578744669656c6473223a7b226e616d657370616365223a2250524f4a4543545f434f4e464947222c226b6579223a223139322e3136382e312e313033227d7d', 'hex')); assert(request.opaque === 1); assert(request.code === RequestCode.GET_KV_CONFIG_BY_VALUE); assert.deepEqual(request.extFields, { namespace: 'PROJECT_CONFIG', key: '192.168.1.103', }); const response = RemotingCommand.decode(Buffer.from('00000066000000627b22636f6465223a302c226c616e6775616765223a224a415641222c226f7061717565223a312c22666c6167223a312c2276657273696f6e223a37382c2272656d61726b223a226e6f742073657420616e7920726573706f6e736520636f6465227d', 'hex')); assert(response.opaque === 1); assert(response.code === ResponseCode.SUCCESS); assert(response.remark === 'not set any response code'); }); }); ================================================ FILE: test/remoting_client.test.js ================================================ 'use strict'; const mm = require('mm'); const assert = require('assert'); const httpclient = require('urllib'); const config = require('../example/config'); const RemotingClient = require('../lib/remoting_client'); const RequestCode = require('../lib/protocol/request_code'); const ResponseCode = require('../lib/protocol/response_code'); const RemotingCommand = require('../lib/protocol/command/remoting_command'); describe('test/remoting_client.test.js', function() { let client; before(() => { client = new RemotingClient(Object.assign({ httpclient }, config)); return client.ready(); }); afterEach(mm.restore); after(() => client.close()); it('should create & ready ok', async () => { const client = new RemotingClient(Object.assign({ httpclient }, config)); await client.ready(); assert(client._namesrvAddrList.length > 0); await client.close(); }); it('should invoke ok', async () => { await client.ready(); const res = await client.invoke(null, new RemotingCommand({ code: RequestCode.GET_ROUTEINTO_BY_TOPIC, customHeader: { topic: 'NOT_EXISTS', }, }), 5000); assert(res); assert(res.code === ResponseCode.TOPIC_NOT_EXIST); }); it('should invoke for namesrv with retry', async () => { let retryCount = 0; mm(client, '_namesrvAddrList', [ 0, 0 ]); mm(client, 'invoke', async () => { retryCount++; mm.restore(); throw new Error('invoke fail'); }); const res = await client.invokeForNameSrvAtLeastOnce(new RemotingCommand({ code: RequestCode.GET_ROUTEINTO_BY_TOPIC, customHeader: { topic: 'NOT_EXISTS', }, }), 5000); assert(retryCount === 1); assert(res); assert(res.code === ResponseCode.TOPIC_NOT_EXIST); }); it('should invoke oneway ok', async () => { await client.invokeOneway(null, new RemotingCommand({ code: RequestCode.GET_KV_CONFIG_BY_VALUE, customHeader: { namespace: 'PROJECT_CONFIG', key: '127.0.0.1', }, })); }); it('should close ok', async () => { const client = new RemotingClient(Object.assign({ httpclient }, config)); await client.ready(); const res = await client.invoke(null, new RemotingCommand({ code: RequestCode.GET_ROUTEINTO_BY_TOPIC, customHeader: { topic: 'NOT_EXISTS', }, }), 5000); assert(res); assert(res.code === ResponseCode.TOPIC_NOT_EXIST); await client.close(); await client.close(); }); it('should invoke ok after close', async () => { const client = new RemotingClient(Object.assign({ httpclient }, config)); await client.ready(); const res = await client.invoke(null, new RemotingCommand({ code: RequestCode.GET_ROUTEINTO_BY_TOPIC, customHeader: { topic: 'NOT_EXISTS', }, }), 5000); assert(res); assert(res.code === ResponseCode.TOPIC_NOT_EXIST); await client.close(); await client.invokeOneway(null, new RemotingCommand({ code: RequestCode.GET_ROUTEINTO_BY_TOPIC, customHeader: { topic: 'NOT_EXISTS', }, })); await client.invoke(null, new RemotingCommand({ code: RequestCode.GET_ROUTEINTO_BY_TOPIC, customHeader: { topic: 'NOT_EXISTS', }, }), 5000); }); it('should call multi times ok', async () => { await client.invoke(null, new RemotingCommand({ code: RequestCode.GET_ROUTEINTO_BY_TOPIC, customHeader: { topic: 'NOT_EXISTS', }, }), 5000); await client.invoke(null, new RemotingCommand({ code: RequestCode.GET_ROUTEINTO_BY_TOPIC, customHeader: { topic: 'NOT_EXISTS', }, }), 5000); await client.invoke(null, new RemotingCommand({ code: RequestCode.GET_ROUTEINTO_BY_TOPIC, customHeader: { topic: 'NOT_EXISTS', }, }), 5000); await client.invokeOneway(null, new RemotingCommand({ code: RequestCode.GET_ROUTEINTO_BY_TOPIC, customHeader: { topic: 'NOT_EXISTS', }, })); await client.invokeOneway(null, new RemotingCommand({ code: RequestCode.GET_ROUTEINTO_BY_TOPIC, customHeader: { topic: 'NOT_EXISTS', }, })); }); // it('should not create two channel with same address', function() { // const address = client._namesrvAddrList[0]; // assert(client.getAndCreateChannel(address).clientId === client.getAndCreateChannel(address).clientId); // }); it('should create a new channel if old one is closed', function() { const address = client._namesrvAddrList[0]; const channel = client.getAndCreateChannel(address); channel.close(); assert(client.getAndCreateChannel(address).clientId !== channel.clientId); }); it('should support unitName', async () => { mm(httpclient, 'request', async url => { assert(url.includes('-xxx?nofix=1')); return { status: 200, data: '127.0.0.1:9876;127.0.0.2:9876', }; }); const client = new RemotingClient(Object.assign({ httpclient, unitName: 'xxx' }, config)); await client.ready(); assert(client._namesrvAddrList, [ '127.0.0.1:9876', '127.0.0.2:9876', ]); }); }); ================================================ FILE: test/store/local_file.test.js ================================================ 'use strict'; const assert = require('assert'); const httpclient = require('urllib'); const MQClient = require('../../lib/mq_client'); const ClientConfig = require('../../lib/client_config'); const MessageQueue = require('../../lib/message_queue'); const LocalFileOffsetStore = require('../../lib/store/local_file'); describe('test/store/local_file.test.js', function() { before(async () => { const client = new MQClient(new ClientConfig({ instanceName: Date.now() + '', httpclient, })); await client.ready(); this.store = new LocalFileOffsetStore(client, 'please_rename_unique_group_name_1'); }); it('should load ok', () => { return this.store.load(); }); it('should updateOffset ok', () => { const mq = new MessageQueue('TopicTest_1', 'taobaodaily-04', 1); this.store.updateOffset(mq, 1000, true); assert(this.store.offsetTable.get('[topic="TopicTest_1", brokerName="taobaodaily-04", queueId="1"]') === 1000); this.store.updateOffset(null, 1000); }); it('should readOffset ok', async () => { const mq = new MessageQueue('TopicTest_1', 'taobaodaily-04', 1); this.store.updateOffset(mq, 1000, true); let offset = await this.store.readOffset(mq, 'READ_FROM_MEMORY'); assert(offset === 1000); offset = await this.store.readOffset(mq, 'READ_FROM_STORE'); assert(typeof offset === 'number'); offset = await this.store.readOffset(null, 'READ_FROM_STORE'); assert(offset === -1); const mq1 = new MessageQueue('TopicTest_1', 'xxx', 1); offset = await this.store.readOffset(mq1, 'READ_FROM_STORE'); assert(typeof offset === 'number'); assert(offset === -1); const mq2 = new MessageQueue('TopicTest_1', 'yyy', 1); offset = await this.store.readOffset(mq2, 'MEMORY_FIRST_THEN_STORE'); assert(typeof offset === 'number'); assert(offset === -1); }); it('should persistAll ok', async () => { const mq = new MessageQueue('TopicTest_1', 'taobaodaily-04', 1); this.store.updateOffset(mq, 1000); await this.store.persistAll([ mq ]); await this.store.persistAll([]); await this.store.persistAll(null); }); it('should persist ok', async () => { const mq = new MessageQueue('TopicTest_1', 'taobaodaily-04', 1); this.store.updateOffset(mq, 1000); await this.store.persist(mq); const mq1 = new MessageQueue('TopicTest_1', 'zzz', 1); await this.store.persist(mq1); }); it('should removeOffset ok', () => { const mq = new MessageQueue('TopicTest_1', 'taobaodaily-04', 1); this.store.updateOffset(mq, 1000); this.store.removeOffset(mq); assert(!this.store.offsetTable.has(mq.key)); }); }); ================================================ FILE: test/store/local_memory.test.js ================================================ 'use strict'; const mm = require('mm'); const assert = require('assert'); const MessageQueue = require('../../lib/message_queue'); const ReadOffsetType = require('../../lib/store/read_offset_type'); const LocalMemoryOffsetStore = require('../../lib/store/local_memory'); describe('test/store/local_file.test.js', () => { let offsetStore; const groupName = 'S_appname_service'; before(() => { offsetStore = new LocalMemoryOffsetStore({ logger: console, }, groupName); }); beforeEach(mm.restore); it('should updateOffset & readOffset ok', async () => { const mq = new MessageQueue('TopicTest_1', 'taobaodaily-04', 1); offsetStore.updateOffset(mq, 1000, false); let offset = await offsetStore.readOffset(mq, ReadOffsetType.MEMORY_FIRST_THEN_STORE); assert(offset === 1000); offset = await offsetStore.readOffset(mq, ReadOffsetType.READ_FROM_MEMORY); assert(offset === 1000); offset = await offsetStore.readOffset(mq, ReadOffsetType.READ_FROM_STORE); assert(offset === 1000); await offsetStore.persist(mq); offset = await offsetStore.readOffset(mq, ReadOffsetType.READ_FROM_STORE); assert(offset === 1000); const mq2 = new MessageQueue('TopicTest_1', 'taobaodaily-04', 2); await offsetStore.persistAll([ mq, mq2 ]); offset = await offsetStore.readOffset(mq, ReadOffsetType.READ_FROM_STORE); assert(offset === 1000); }); it('should updateOffset increaseOnly', async () => { const mq = new MessageQueue('TopicTest_1', 'taobaodaily-04', 1); offsetStore.updateOffset(mq, 1000, false); let offset = await offsetStore.readOffset(mq, ReadOffsetType.READ_FROM_MEMORY); assert(offset === 1000); offsetStore.updateOffset(); offsetStore.updateOffset(mq, 999, true); offset = await offsetStore.readOffset(mq, ReadOffsetType.READ_FROM_MEMORY); assert(offset === 1000); offsetStore.updateOffset(mq, 999, false); offset = await offsetStore.readOffset(mq, ReadOffsetType.READ_FROM_MEMORY); assert(offset === 999); offset = await offsetStore.readOffset(null, ReadOffsetType.READ_FROM_MEMORY); assert(offset === -1); offset = await offsetStore.readOffset(null, ReadOffsetType.MEMORY_FIRST_THEN_STORE); assert(offset === -1); offset = await offsetStore.readOffset(null, ReadOffsetType.READ_FROM_STORE); assert(offset === -1); offset = await offsetStore.readOffset(mq); assert(offset === -1); }); it('should load ok', async () => { const mq = new MessageQueue('TopicTest_1', 'taobaodaily-04', 1); offsetStore.updateOffset(mq, 1000, false); const offsetStore2 = new LocalMemoryOffsetStore({ logger: console, }, groupName); let offset = await offsetStore2.readOffset(mq, ReadOffsetType.READ_FROM_MEMORY); assert(offset === -1); await offsetStore2.load(); offset = await offsetStore2.readOffset(mq, ReadOffsetType.READ_FROM_MEMORY); assert(offset === -1); offsetStore.removeOffset(mq); offset = await offsetStore.readOffset(mq, ReadOffsetType.READ_FROM_MEMORY); assert(offset === -1); offset = await offsetStore.readOffset(mq, ReadOffsetType.READ_FROM_STORE); assert(offset === -1); }); }); ================================================ FILE: test/store/remote_broker.test.js ================================================ 'use strict'; const assert = require('assert'); const httpclient = require('urllib'); const MQClient = require('../../lib/mq_client'); const config = require('../../example/config'); const ClientConfig = require('../../lib/client_config'); const MessageQueue = require('../../lib/message_queue'); const RemoteBrokerOffsetStore = require('../../lib/store/remote_broker'); const TOPIC = config.topic; const consumerGroup = config.consumerGroup; describe('test/store/remote_broker.test.js', function() { let client; let brokerName; before(async () => { client = new MQClient(new ClientConfig(Object.assign({ httpclient }, config))); await client.ready(); const routerInfoData = await client.getDefaultTopicRouteInfoFromNameServer(TOPIC, 3000); brokerName = routerInfoData.brokerDatas[0].brokerName; this.store = new RemoteBrokerOffsetStore(client, consumerGroup); }); after(() => client.close()); it('should load ok', () => this.store.load()); it('should updateOffset ok', () => { const mq = new MessageQueue(TOPIC, brokerName, 1); this.store.updateOffset(mq, 1000); assert(this.store.offsetTable.get(`[topic="${TOPIC}", brokerName="${brokerName}", queueId="1"]`) === 1000); this.store.updateOffset(null, 1000); }); it('should readOffset ok', async () => { const mq = new MessageQueue(TOPIC, brokerName, 1); this.store.updateOffset(mq, 1000); let offset = await this.store.readOffset(mq, 'READ_FROM_MEMORY'); assert(offset === 1000); offset = await this.store.readOffset(mq, 'READ_FROM_STORE'); assert(typeof offset === 'number'); offset = await this.store.readOffset(null, 'READ_FROM_STORE'); assert(offset === -1); const mq1 = new MessageQueue(TOPIC, 'xxx', 1); offset = await this.store.readOffset(mq1, 'READ_FROM_STORE'); assert(typeof offset === 'number'); assert(offset === -1); const mq2 = new MessageQueue(TOPIC, 'yyy', 1); offset = await this.store.readOffset(mq2, 'MEMORY_FIRST_THEN_STORE'); assert(typeof offset === 'number'); assert(offset === -1); }); it('should persistAll ok', async () => { const mq = new MessageQueue(TOPIC, brokerName, 1); this.store.updateOffset(mq, 1000); await this.store.persistAll([ mq ]); await this.store.persistAll([]); await this.store.persistAll(null); }); it('should persist ok', async () => { const mq = new MessageQueue(TOPIC, brokerName, 1); this.store.updateOffset(mq, 1000); await this.store.persist(mq); const mq1 = new MessageQueue(TOPIC, 'zzz', 1); await this.store.persist(mq1); }); it('should removeOffset ok', () => { const mq = new MessageQueue(TOPIC, brokerName, 1); this.store.updateOffset(mq, 1000); this.store.removeOffset(mq); assert(!this.store.offsetTable.has(mq.key)); }); }); ================================================ FILE: test/utils/index.test.js ================================================ 'use strict'; const assert = require('assert'); const util = require('../../lib/utils/index'); describe('test/utils/index.test.js', () => { it('should parseDate ok', () => { const d = util.parseDate('2018112000000'); assert.equal(d.getTime() - d.getTimezoneOffset() * 60 * 1000, 1542672000000); }); it('should getRetryTopic ok', () => { const msg = { retryTopic: 'xxx', }; util.resetRetryTopic(msg, 'yyy'); assert.equal(msg.retryTopic, 'xxx'); }); }); ================================================ FILE: test/utils/message_sys_flag.test.js ================================================ 'use strict'; const assert = require('assert'); const MessageSysFlag = require('../../lib/utils/message_sys_flag'); describe('test/utils/message_sys_flag.test.js', () => { it('should getTransactionValue ok', () => { assert(MessageSysFlag.getTransactionValue(7) === 4); assert(MessageSysFlag.getTransactionValue(6) === 4); assert(MessageSysFlag.getTransactionValue(15) === 12); assert(MessageSysFlag.getTransactionValue(100) === 4); }); it('should resetTransactionValue ok', () => { assert(MessageSysFlag.getTransactionValue(MessageSysFlag.resetTransactionValue(7, 0)) === 0); assert(MessageSysFlag.getTransactionValue(MessageSysFlag.resetTransactionValue(7, 4)) === 4); assert(MessageSysFlag.getTransactionValue(MessageSysFlag.resetTransactionValue(7, 8)) === 8); assert(MessageSysFlag.getTransactionValue(MessageSysFlag.resetTransactionValue(7, 12)) === 12); }); it('should clearCompressedFlag ok', () => { assert(MessageSysFlag.clearCompressedFlag(7) === 6); assert(MessageSysFlag.clearCompressedFlag(6) === 6); }); }); ================================================ FILE: test/utils/pull_sys_flag.test.js ================================================ 'use strict'; const assert = require('assert'); const PullSysFlag = require('../../lib/utils/pull_sys_flag'); describe('test/utils/pull_sys_flag.test.js', () => { it('should buildSysFlag ok', () => { assert(PullSysFlag.buildSysFlag(true, true, true, true) === 15); assert(PullSysFlag.buildSysFlag(false, true, true, true) === 14); assert(PullSysFlag.buildSysFlag(false, false, true, true) === 12); assert(PullSysFlag.buildSysFlag(false, false, false, true) === 8); assert(PullSysFlag.buildSysFlag(false, false, false, false) === 0); assert(PullSysFlag.buildSysFlag(false, true, true, true) === PullSysFlag.clearCommitOffsetFlag(PullSysFlag.buildSysFlag(true, true, true, true))); assert(PullSysFlag.hasCommitOffsetFlag(PullSysFlag.buildSysFlag(true, true, true, true))); assert(!PullSysFlag.hasCommitOffsetFlag(PullSysFlag.buildSysFlag(false, true, true, true))); assert(PullSysFlag.hasSuspendFlag(PullSysFlag.buildSysFlag(true, true, true, true))); assert(!PullSysFlag.hasSuspendFlag(PullSysFlag.buildSysFlag(false, false, true, true))); assert(PullSysFlag.hasSubscriptionFlag(PullSysFlag.buildSysFlag(true, true, true, true))); assert(!PullSysFlag.hasSubscriptionFlag(PullSysFlag.buildSysFlag(false, true, false, true))); assert(PullSysFlag.hasClassFilterFlag(PullSysFlag.buildSysFlag(true, true, true, true))); assert(!PullSysFlag.hasClassFilterFlag(PullSysFlag.buildSysFlag(false, true, true, false))); }); }); ================================================ FILE: test/utils.js ================================================ 'use strict'; const fs = require('fs'); const path = require('path'); const fixtures = path.join(__dirname, 'fixtures'); exports.bytes = name => { return fs.readFileSync(path.join(fixtures, name)); }; exports.write = (name, buf) => { fs.writeFileSync(path.join(fixtures, name), buf); };