Showing preview only (269K chars total). Download the full file or copy to clipboard to get everything.
Repository: ali-sdk/ali-ons
Branch: master
Commit: 85b7d51f8f24
Files: 69
Total size: 244.6 KB
Directory structure:
gitextract_g5wtfrkh/
├── .eslintignore
├── .eslintrc
├── .github/
│ └── workflows/
│ ├── ci-actions.yml
│ └── release.yml
├── .gitignore
├── AUTHORS
├── CHANGELOG.md
├── History.md
├── LICENSE
├── README.md
├── example/
│ ├── config.js
│ ├── consumer.js
│ ├── logger.js
│ └── producer.js
├── lib/
│ ├── channel.js
│ ├── client_config.js
│ ├── consumer/
│ │ ├── consume_from_where.js
│ │ ├── mq_push_consumer.js
│ │ ├── pull_status.js
│ │ └── rebalance/
│ │ ├── allocate_message_queue_averagely.js
│ │ └── allocate_message_queue_strategy.js
│ ├── index.js
│ ├── logger.js
│ ├── message/
│ │ ├── message.js
│ │ ├── message_const.js
│ │ └── message_decoder.js
│ ├── message_queue.js
│ ├── mix_all.js
│ ├── mq_client.js
│ ├── mq_client_api.js
│ ├── process_queue.js
│ ├── producer/
│ │ ├── mq_producer.js
│ │ ├── send_status.js
│ │ └── topic_publish_info.js
│ ├── protocol/
│ │ ├── command/
│ │ │ ├── opaque_generator.js
│ │ │ └── remoting_command.js
│ │ ├── consume_type.js
│ │ ├── message_model.js
│ │ ├── perm_name.js
│ │ ├── request_code.js
│ │ └── response_code.js
│ ├── remoting_client.js
│ ├── store/
│ │ ├── index.js
│ │ ├── local_file.js
│ │ ├── local_memory.js
│ │ ├── read_offset_type.js
│ │ └── remote_broker.js
│ └── utils/
│ ├── index.js
│ ├── message_sys_flag.js
│ └── pull_sys_flag.js
├── package.json
└── test/
├── allocate_message_queue_averagely.test.js
├── channel.test.js
├── consumer/
│ └── rebalance/
│ └── allocate_message_queue_averagely.test.js
├── index.test.js
├── index_namesrv.test.js
├── message/
│ └── message_decoder.test.js
├── mq_client.test.js
├── mq_client_api.test.js
├── protocol/
│ └── command/
│ ├── opaque_generator.test.js
│ └── remoting_command.test.js
├── remoting_client.test.js
├── store/
│ ├── local_file.test.js
│ ├── local_memory.test.js
│ └── remote_broker.test.js
├── utils/
│ ├── index.test.js
│ ├── message_sys_flag.test.js
│ └── pull_sys_flag.test.js
└── utils.js
================================================
FILE CONTENTS
================================================
================================================
FILE: .eslintignore
================================================
test/fixtures
test/benchmark
node_modules
coverage
benchmark
_lib
example
================================================
FILE: .eslintrc
================================================
{
"extends": "eslint-config-egg"
}
================================================
FILE: .github/workflows/ci-actions.yml
================================================
name: ci-actions
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
merge_group:
jobs:
test:
runs-on: ubuntu-latest
environment: 'ci'
strategy:
matrix:
node-version: [16, 18, 20, 22, 24]
env:
ALI_SDK_ONS_ID: ${{ secrets.ALI_SDK_ONS_ID }}
ALI_SDK_ONS_SECRET: ${{ secrets.ALI_SDK_ONS_SECRET }}
steps:
- name: Start RocketMQ Services
run: |
docker network create rocketmq
docker run -d --name rocketmq-namesrv --network rocketmq -p 9876:9876 apache/rocketmq:5.3.2 sh mqnamesrv
echo "Waiting for RocketMQ services to be ready..."
sleep 20
docker logs rocketmq-namesrv
echo "brokerIP1=127.0.0.1" > broker.conf
broker_ports=(10911 10912)
for port in ${broker_ports[@]}; do
docker run -d --name rocketmq-broker-${port} --network rocketmq -p ${port}:${port} -e "NAMESRV_ADDR=rocketmq-namesrv:9876" -v ./broker.conf:/home/rocketmq/conf/broker.conf apache/rocketmq:5.3.2 sh mqbroker --enable-proxy -c /home/rocketmq/conf/broker.conf
echo "Waiting for RocketMQ broker to be ready..."
sleep 20
docker logs rocketmq-broker-${port}
docker exec rocketmq-broker-${port} sh mqadmin updatetopic -t TP_alions_test_topic -c DefaultCluster
done
- uses: actions/checkout@v5
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v6
with:
node-version: ${{ matrix.node-version }}
- name: npm install, build, and test
run: |
npm install
npm run ci
- name: Codecov
uses: codecov/codecov-action@v2
with:
token: ${{ secrets.CODECOV_TOKEN }}
================================================
FILE: .github/workflows/release.yml
================================================
name: Release
on:
push:
branches: [ master ]
permissions:
contents: write
deployments: write
issues: write
pull-requests: write
id-token: write
jobs:
release:
name: NPM
uses: ali-sdk/github-actions/.github/workflows/npm-release.yml@master
secrets:
GIT_TOKEN: ${{ secrets.GIT_TOKEN }}
================================================
FILE: .gitignore
================================================
.DS_Store
# Logs
logs
*.log
# Runtime data
pids
*.pid
*.seed
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage
# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# node-waf configuration
.lock-wscript
# Compiled binary addons (http://nodejs.org/api/addons.html)
build/Release
# Dependency directory
# https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git
node_modules
example/.config.js
.nyc_output/
.idea
================================================
FILE: AUTHORS
================================================
# Ordered by date of first contribution.
# Auto-generated by 'contributors' on Fri, 29 Sep 2017 01:42:55 GMT.
# https://github.com/xingrz/node-contributors
GaoXiaochen <gxcsoccer@users.noreply.github.com>
宗羽 <xiaochen.gaoxc@alibaba-inc.com>
================================================
FILE: CHANGELOG.md
================================================
# Changelog
## <small>3.12.2 (2025-12-08)</small>
* fix: make auto release work again ([5b43dda](https://github.com/ali-sdk/ali-ons/commit/5b43dda))
## <small>3.12.1 (2025-12-08)</small>
* fix: retry fetching nameservers 3 times until yielding error (#118) ([bdc44eb](https://github.com/ali-sdk/ali-ons/commit/bdc44eb)), closes [#118](https://github.com/ali-sdk/ali-ons/issues/118)
* chore: enable auto release (#119) ([669abdc](https://github.com/ali-sdk/ali-ons/commit/669abdc)), closes [#119](https://github.com/ali-sdk/ali-ons/issues/119)
================================================
FILE: History.md
================================================
3.12.0 / 2023-07-31
==================
**features**
* [[`32d4d09`](http://github.com/ali-sdk/ali-ons/commit/32d4d0969fb3f10f691e38a1202f06359516593f)] - feat: 支持shardingKey 客户端实现 (#109) (Orange Mi <<orangemiwj@gmail.com>>)
3.11.0 / 2023-01-12
==================
**fixes**
* [[`15846d4`](http://github.com/ali-sdk/ali-ons/commit/15846d4176bb2e4ef7c4d7399b02affdbcb66d58)] - fix: update name server by throttle (#108) (Gao Yang <<82163514@qq.com>>)
3.10.0 / 2022-06-20
==================
**features**
* [[`6bc5a62`](http://github.com/ali-sdk/ali-ons/commit/6bc5a6271afe5c274db36c961cccc36643b1ad3f)] - feat: support consumer filter message by sql92 (#105) (#106) (Hongcai Deng <<admin@dhchouse.com>>)
3.9.0 / 2021-05-08
==================
**features**
* [[`483b4fc`](http://github.com/ali-sdk/ali-ons/commit/483b4fc64457554331f5c0654a4aa642ab8a0ccf)] - feat: support set unitName (#101) (zōng yǔ <<gxcsoccer@users.noreply.github.com>>)
3.8.2 / 2021-01-28
==================
**fixes**
* [[`b6ab961`](http://github.com/ali-sdk/ali-ons/commit/b6ab96159ab584ca27ab3390eb91eb79dddb95e8)] - fix: 🐛 repeat pull on same queue (#100) (Hongcai Deng <<admin@dhchouse.com>>)
3.8.1 / 2020-04-21
==================
**others**
* [[`836d9e5`](http://github.com/ali-sdk/ali-ons/commit/836d9e5fae2b9cb0a79f0cc7874b26a7939af791)] - chore: add more debug log (#87) (Hongcai Deng <<admin@dhchouse.com>>)
* [[`6851d8e`](http://github.com/ali-sdk/ali-ons/commit/6851d8efe0a2f91cf598440b350060abafdcbff7)] - chore: reduce error (#64) (Hongcai Deng <<admin@dhchouse.com>>)
3.8.0 / 2020-04-14
==================
**features**
* [[`207280f`](http://github.com/ali-sdk/ali-ons/commit/207280f8f00f82d98a2fb5bd32a528c3fdc1c68f)] - feat: support reconsume message later (#97) (Gaara <<hubiao1003hf@163.com>>)
3.7.2 / 2020-03-17
==================
**fixes**
* [[`637aee4`](http://github.com/ali-sdk/ali-ons/commit/637aee4424e26ffce2bb1398e2a587b0ce1d6d94)] - fix: random-publish-queue-select-start-point (#36) (Hongcai Deng <<admin@dhchouse.com>>)
3.7.1 / 2020-03-12
==================
**fixes**
* [[`cb244b7`](http://github.com/ali-sdk/ali-ons/commit/cb244b7c089f1d30fa1388b4334f27742ada196a)] - fix: use co-gather instead of p-gather (#95) (Yiyu He <<dead_horse@qq.com>>)
3.7.0 / 2019-10-16
==================
**features**
* [[`3889a0e`](http://github.com/ali-sdk/ali-ons/commit/3889a0e03637fbb4ce8501abee3bd45d70000e1b)] - feat: support nameAddr as an array (#91) (zōng yǔ <<gxcsoccer@users.noreply.github.com>>)
3.6.5 / 2019-05-10
==================
**others**
* [[`92892df`](http://github.com/ali-sdk/ali-ons/commit/92892df3ca1a4f33bec36ebd5fcfc8b3a3251a45)] - chore: adjust pulling log format (#82) (zōng yǔ <<gxcsoccer@users.noreply.github.com>>)
3.6.4 / 2019-05-10
==================
**fixes**
* [[`456cd1f`](http://github.com/ali-sdk/ali-ons/commit/456cd1f70d22161f51749f48737aa038e0b69f6a)] - fix: producer should format topic first (#81) (Hongcai Deng <<admin@dhchouse.com>>)
3.6.3 / 2019-05-10
==================
**fixes**
* [[`d477806`](http://github.com/ali-sdk/ali-ons/commit/d47780628ef18be8fda5922c63c71b0bacc4c75c)] - fix: close socket while pulling message timeout (#80) (zōng yǔ <<gxcsoccer@users.noreply.github.com>>)
**others**
* [[`d1ca295`](http://github.com/ali-sdk/ali-ons/commit/d1ca295a9a600da2cb3bee99c206aa03c92025cf)] - update RocketMQ link to https://rocketmq.apache.org (#79) (小雷 <<863837949@qq.com>>)
3.6.2 / 2019-04-16
==================
**fixes**
* [[`cb385e3`](http://github.com/ali-sdk/ali-ons/commit/cb385e31c486877b625eb6b27a69245c1d43e65d)] - fix: namespace logic (#78) (zōng yǔ <<gxcsoccer@users.noreply.github.com>>)
3.6.1 / 2019-04-15
==================
**fixes**
* [[`7e5c6a4`](http://github.com/ali-sdk/ali-ons/commit/7e5c6a4ccf3f488df0f0e41a9a7bcfb0aadefb14)] - fix: not set nameSrv by default (#77) (zōng yǔ <<gxcsoccer@users.noreply.github.com>>)
3.6.0 / 2019-04-15
==================
**features**
* [[`2cd16be`](http://github.com/ali-sdk/ali-ons/commit/2cd16bed4608ca4235ad9eb5c211b9aad090a411)] - feat: support namespace (#75) (zōng yǔ <<gxcsoccer@users.noreply.github.com>>)
**fixes**
* [[`0176223`](http://github.com/ali-sdk/ali-ons/commit/0176223b3fdd3a2e5a219fb02b03552764951588)] - fix: retry should process by broker (#73) (Hongcai Deng <<admin@dhchouse.com>>)
**others**
* [[`35f9341`](http://github.com/ali-sdk/ali-ons/commit/35f934174077e6835284ca5cbebb3e171122fd11)] - chore: update test config (#74) (zōng yǔ <<gxcsoccer@users.noreply.github.com>>)
3.5.1 / 2019-04-12
==================
3.5.0 / 2019-04-12
==================
**features**
* [[`073058b`](http://github.com/ali-sdk/ali-ons/commit/073058b9d82415f7063758e815f0d31b2b9d5a2b)] - feat: support delay deliver message (#65) (MarvinWilliam <<la805779602@hotmail.com>>)
**fixes**
* [[`8ef9502`](http://github.com/ali-sdk/ali-ons/commit/8ef95027eef89456b17531b89761fa375168bfd6)] - fix: drop message if reconsumeTimes > maxReconsumeTimes (#72) (zōng yǔ <<gxcsoccer@users.noreply.github.com>>)
**others**
* [[`a44d4d0`](http://github.com/ali-sdk/ali-ons/commit/a44d4d05919d68200e012442e7831e5b2217f808)] - chore: upgrade deps (#71) (zōng yǔ <<gxcsoccer@users.noreply.github.com>>)
3.4.0 / 2019-03-13
==================
**features**
* [[`7dfe610`](http://github.com/ali-sdk/ali-ons/commit/7dfe6105f41d7478d587732e3ed9c14e2b96e1bb)] - feat: support set namesrv (#61) (Hongcai Deng <<admin@dhchouse.com>>)
3.3.0 / 2018-12-14
==================
**features**
* [[`9acff9b`](http://github.com/ali-sdk/ali-ons/commit/9acff9b91c0325ed88ad634ae15161f62dceb574)] - feat: support consume back (#56) (Hongcai Deng <<admin@dhchouse.com>>)
3.2.2 / 2018-11-20
==================
**fixes**
* [[`39e3782`](http://github.com/ali-sdk/ali-ons/commit/39e37827c8eac0c16e1fb5ecd64792da331673f3)] - fix: parse date format (#53) (Hongcai Deng <<admin@dhchouse.com>>)
**others**
* [[`e2dc377`](http://github.com/ali-sdk/ali-ons/commit/e2dc377babb3a420e2dfc99a33bf6118979825ff)] - test: fix ci (#54) (zōng yǔ <<gxcsoccer@users.noreply.github.com>>)
3.2.1 / 2018-10-18
==================
**fixes**
* [[`d6b6fc6`](http://github.com/ali-sdk/ali-ons/commit/d6b6fc61d049c5925c65913c299d0b423573d4b6)] - fix: accessKeyID => accessKeyId (#50) (fengmk2 <<fengmk2@gmail.com>>)
3.2.0 / 2018-10-17
==================
**features**
* [[`8ec58c8`](http://github.com/ali-sdk/ali-ons/commit/8ec58c8f773633a4f1bb0341306d89afda1972e5)] - feat: unify aliyun keys to accessKeyID and accessKeySecret (fengmk2 <<fengmk2@gmail.com>>)
**others**
* [[`9ecd381`](http://github.com/ali-sdk/ali-ons/commit/9ecd381f7079d7be6f2551a6baabfc51cdefde46)] - f (fengmk2 <<fengmk2@gmail.com>>)
* [[`742f269`](http://github.com/ali-sdk/ali-ons/commit/742f269534c2e30f0772663ca3690ad000d9c75e)] - f (fengmk2 <<fengmk2@gmail.com>>)
3.1.0 / 2018-09-14
==================
**others**
* [[`d1de082`](http://github.com/ali-sdk/ali-ons/commit/d1de0823fcc860ae12133a0026ddcfe2105db82d)] - name-server-fault-tolerance (wujia <<geoff.j.wu@gmail.com>>)
3.0.0 / 2018-07-03
==================
**features**
* [[`5624319`](http://github.com/ali-sdk/ali-ons/commit/56243195ad838b88932be8b5e5c1f2f6a2eb3d4f)] - feat: suppory async (ngot <<zhuanghengfei@gmail.com>>)
**others**
* [[`4b84644`](http://github.com/ali-sdk/ali-ons/commit/4b846446ceec7f175a363ef3dd011f67dc2dcaea)] - test: update ci command (Hongcai Deng <<admin@dhchouse.com>>)
* [[`ebb7942`](http://github.com/ali-sdk/ali-ons/commit/ebb7942269662843357ce4070a19628c6d51fe88)] - test: trigger ci (Hongcai Deng <<admin@dhchouse.com>>)
* [[`79b56c8`](http://github.com/ali-sdk/ali-ons/commit/79b56c86f03a957cad4442e52ddae8e49389ab57)] - breaking: async (Hongcai Deng <<admin@dhchouse.com>>)
* [[`1ee4785`](http://github.com/ali-sdk/ali-ons/commit/1ee47857f4846a4d70b33b60234972651d1c37d3)] - f (ngot <<zhuanghengfei@gmail.com>>)
* [[`e0a7a77`](http://github.com/ali-sdk/ali-ons/commit/e0a7a77e0830a04c020c7a2762f32a2f210ef426)] - f (ngot <<zhuanghengfei@gmail.com>>)
* [[`519e164`](http://github.com/ali-sdk/ali-ons/commit/519e1641e224deec08504ccf60ca4fa05e01788b)] - f (ngot <<zhuanghengfei@gmail.com>>)
* [[`1de2b1a`](http://github.com/ali-sdk/ali-ons/commit/1de2b1a6cdc46fd19faeceda50d89c2f2928144d)] - doc: updaet readme (ngot <<zhuanghengfei@gmail.com>>)
2.0.4 / 2018-01-10
==================
**fixes**
* [[`6f78013`](http://github.com/ali-sdk/ali-ons/commit/6f780131b713465827588ac3ee866b9e2b2bd2ae)] - fix: fix invokeOneWay issue (gxcsoccer <<gxcsoccer@126.com>>)
**others**
* [[`46d1bfe`](http://github.com/ali-sdk/ali-ons/commit/46d1bfe4cd5af83aa814d2a6dfd490efde5172bc)] - chore: release 2.0.3 (gxcsoccer <<gxcsoccer@126.com>>),
2.0.3 / 2017-11-11
==================
* fix: memory leak may occurred cause by Promise.race
* doc: fix consumer and producer initialize issue
2.0.2 / 2017-10-09
==================
* fix: persist consumer offset issue
2.0.1 / 2017-09-29
==================
* fix: support subscribe before ready
2.0.0 / 2017-09-29
==================
* feat: add local memory store
* refactor: new consumer api
* fix: consumer offset update issue
1.0.0 / 2017-03-14
==================
* feat: implement ali-ons base on rocketmq #1
================================================
FILE: LICENSE
================================================
The MIT License (MIT)
Copyright (c) ali-sdk
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
================================================
FILE: README.md
================================================
# ali-ons
[![NPM version][npm-image]][npm-url]
[![ci-actions][ci-image]][ci-url]
[![npm download][download-image]][download-url]
[](https://nodejs.org/en/download/)
[](https://makeapullrequest.com)
[npm-image]: https://img.shields.io/npm/v/ali-ons.svg?style=flat-square
[npm-url]: https://npmjs.org/package/ali-ons
[ci-image]: https://github.com/ali-sdk/ali-ons/actions/workflows/ci-actions.yml/badge.svg
[ci-url]: https://github.com/ali-sdk/ali-ons/actions/workflows/ci-actions.yml
[download-image]: https://img.shields.io/npm/dm/ali-ons.svg?style=flat-square
[download-url]: https://npmjs.org/package/ali-ons
Aliyun Open Notification Service Client (base on opensource project [RocketMQ](https://rocketmq.apache.org/))
Sub module of [ali-sdk](https://github.com/ali-sdk/ali-sdk).
## Install
```bash
npm install ali-ons --save
```
## Usage
consumer
```js
'use strict';
const httpclient = require('urllib');
const Consumer = require('ali-ons').Consumer;
const consumer = new Consumer({
httpclient,
accessKeyId: 'your-accessKeyId',
accessKeySecret: 'your-AccessKeySecret',
consumerGroup: 'your-consumer-group',
// namespace: '', // aliyun namespace support
// isBroadcast: true,
});
consumer.subscribe(config.topic, '*', async msg => {
console.log(`receive message, msgId: ${msg.msgId}, body: ${msg.body.toString()}`)
// return Consumer.ACTION_RETRY; // you can return ACTION_RETRY, then this message will be directly retried
});
consumer.on('error', err => console.log(err));
```
If you want to use sql filter, you can subscribe a topic with a sql expression:
```js
consumer.subscribe(
config.topic,
{
expressionType: 'SQL92',
subString: 'a is not null'
},
async msg => {
console.log(`receive message, msgId: ${msg.msgId}, body: ${msg.body.toString()}`)
}
);
```
For more information about sql filter, see: [Filter Messages By SQL92](https://rocketmq.apache.org/rocketmq/filter-messages-by-sql92-in-rocketmq/)
producer
```js
'use strict';
const httpclient = require('urllib');
const Producer = require('ali-ons').Producer;
const Message = require('ali-ons').Message;
const producer = new Producer({
httpclient,
accessKeyId: 'your-accessKeyId',
accessKeySecret: 'your-AccessKeySecret',
producerGroup: 'your-producer-group',
// namespace: '', // aliyun namespace support
});
(async () => {
const msg = new Message('your-topic', // topic
'TagA', // tag
'Hello ONS !!! ' // body
);
// set Message#keys
msg.keys = ['key1'];
// delay consume
// msg.setStartDeliverTime(Date.now() + 5000);
const sendResult = await producer.send(msg);
console.log(sendResult);
})().catch(err => console.error(err))
```
## Secure Keys
Please contact to @gxcsoccer to give you accessKey
- [ons secure data](https://sharelock.io/1/UM02CJiYyhXiZDOn1nhX0iqPqMIQtdwI_T5BY3F-tHs.d8-ycA/01veKH9kgAuFuKCqlVPzGsyPWJ8mQLaKPJjjcB9tpdbvi9L6XQ/IgqDvAdVDMzV9lK2gQzyAj7q-CNk8-1tWrLmdqMV0oJ5qgky40/HgpZyKKDfOGAcyqQ20RUdRgCLRWqF8LUUko0uDl_L-ATNOsi5z/W2bsvBc8tAoqSwNR7u2Sqe6XkNmD98s3UQOK-6T8--VwTbHzcG/dwkHwie3EkGB-TbiMnbRh7_5A-DaOTCtALP3xvl4G0XKxuOriC/2yfuPp7WRucTAoqx2STO5Hv3MZEhh3IXf7YiOQ8pWDDqjLuQSY/_irqzYyeseY9m106ksMUq3-yS_qkBRIuoyL-hHk9ZRhGppsdA5/Dw4Pjg.fmNP3aFkLnvPuhlPRwNcng)
## License
[MIT](LICENSE)
## Contributors
[](https://github.com/ali-sdk/ali-ons/graphs/contributors)
Made with [contributors-img](https://contrib.rocks).
================================================
FILE: example/config.js
================================================
'use strict';
const env = process.env;
// export ALI_SDK_ONS_ID=your-accesskey
// export ALI_SDK_ONS_SECRET=your-secretkey
module.exports = {
accessKeyId: env.ALI_SDK_ONS_ID,
accessKeySecret: env.ALI_SDK_ONS_SECRET,
producerGroup: 'PID_GXCSOCCER',
consumerGroup: 'GID_alions',
topic: 'TP_alions_test_topic',
// https://help.aliyun.com/document_detail/102895.html 阿里云产品更新，支持实例化
nameSrv: 'localhost:9876',
// onsAddr: 'http://onsaddr-internet.aliyun.com/rocketmq/nsaddr4client-internet',
};
================================================
FILE: example/consumer.js
================================================
'use strict';
const httpclient = require('urllib');
const logger = require('./logger');
const Consumer = require('../').Consumer;
const config = require('./config');
const consumer = new Consumer(Object.assign(config, {
httpclient,
logger,
maxReconsumeTimes: 1,
// isBroadcast: true,
// consumeFromWhere: 'CONSUME_FROM_LAST_OFFSET_AND_FROM_MIN_WHEN_BOOT_FIRST',
}));
consumer.subscribe(config.topic, '*', async function(msg) {
console.log(`receive message, msgId: ${msg.msgId}, body: ${msg.body.toString()}`)
// throw new Error(123);
});
consumer.on('error', err => console.log(err));
================================================
FILE: example/logger.js
================================================
'use strict';
module.exports = {
info() {},
warn() {},
error(...args) {
console.error(...args);
},
debug() {},
};
================================================
FILE: example/producer.js
================================================
'use strict';
const logger = require('./logger');
const config = require('./config');
const httpclient = require('urllib');
const Producer = require('../').Producer;
const Message = require('../').Message;
const producer = new Producer(Object.assign({ httpclient, logger }, config));
(async () => {
try {
await producer.ready();
const msg = new Message(config.topic, // topic
'TagA', // tag
'Hello ONS !!! ' // body
);
const sendResult = await producer.send(msg);
console.log(sendResult);
} catch (err) {
console.error(err)
}
})();
================================================
FILE: lib/channel.js
================================================
'use strict';
const is = require('is-type-of');
const crypto = require('crypto');
const Base = require('tcp-base');
const promisify = require('util').promisify;
const RemotingCommand = require('./protocol/command/remoting_command');
class Channel extends Base {
/**
* rocketmq tcp channel object
* @param {String} address - server address
* @param {Object} options
* - {String} accessKey
* - {String} secretKey
* - {String} onsChannel
* @class
*/
constructor(address, options = {}) {
// 10.18.214.201:8080
const arr = address.split(':');
// support alias: accessKeyId and accessKeySecret
options.accessKey = options.accessKey || options.accessKeyId;
options.secretKey = options.secretKey || options.accessKeySecret;
super(Object.assign({
host: arr[0],
port: arr[1],
headerLength: 4,
needHeartbeat: false,
}, options));
this.sendPromise = promisify(this.send);
}
get accessKey() {
return this.options.accessKey;
}
get secretKey() {
return this.options.secretKey;
}
get onsChannel() {
return 'ALIYUN';
}
/**
* Get packet length from header
* @param {Buffer} header - packet header
* @return {Number} bodyLength
*/
getBodyLength(header) {
return header.readInt32BE(0);
}
decode(body, header) {
const command = RemotingCommand.decode(Buffer.concat([ header, body ]));
return {
id: command.opaque,
isResponse: command.isResponseType,
data: command,
};
}
beforeRequest(command) {
if (!this.accessKey || !this.secretKey) {
return;
}
const header = command.customHeader;
const map = new Map();
map.set('AccessKey', this.accessKey);
map.set('OnsChannel', this.onsChannel);
if (header) {
for (const field in header) {
if (!is.nullOrUndefined(header[field])) {
map.set(field, header[field].toString());
}
}
}
let val = '';
const fields = Array.from(map.keys()).sort();
for (const key of fields) {
if (key !== 'Signature') {
val += map.get(key);
}
}
let total = Buffer.from(val, 'utf8');
const bodyLength = command.body ? command.body.length : 0;
if (bodyLength) {
total = Buffer.concat([ total, command.body ], total.length + bodyLength);
}
const hmac = crypto.createHmac('sha1', this.secretKey);
const signature = hmac.update(total).digest('base64');
command.extFields.Signature = signature;
command.extFields.AccessKey = this.accessKey;
command.extFields.OnsChannel = this.onsChannel;
}
/**
* invoke rocketmq api
* @param {RemotingCommand} command - remoting command
* @param {Number} timeout - response timeout
* @return {Promise<Object>} response
*/
invoke(command, timeout) {
this.beforeRequest(command);
return this.sendPromise({
id: command.opaque,
data: command.encode(),
timeout,
}).catch(err => {
// TODO: not sure whether is work ?
if (err.name === 'ResponseTimeoutError') {
this.close();
}
throw err;
});
}
/**
* invoke rocketmq api without need response
* @param {RemotingCommand} command - remoting command
* @return {Promise} Promise
*/
invokeOneway(command) {
this.beforeRequest(command);
return this.sendPromise({
id: command.opaque,
data: command.encode(),
oneway: true,
});
}
}
module.exports = Channel;
================================================
FILE: lib/client_config.js
================================================
'use strict';
const Base = require('sdk-base');
const address = require('address');
const MixAll = require('./mix_all');
const defaultOptions = {
instanceName: 'DEFAULT',
pollNameServerInteval: 30 * 1000,
heartbeatBrokerInterval: 30 * 1000,
persistConsumerOffsetInterval: 5 * 1000,
rebalanceInterval: 10 * 1000,
clientIP: address.ip(),
unitMode: false,
unitName: null,
// 阿里云自创建实例，需要 ns 前缀
namespace: '',
// 公有云生产环境：http://onsaddr-internal.aliyun.com:8080/rocketmq/nsaddr4client-internal
// 公有云公测环境：http://onsaddr-internet.aliyun.com/rocketmq/nsaddr4client-internet
// 杭州金融云环境：http://jbponsaddr-internal.aliyun.com:8080/rocketmq/nsaddr4client-internal
// 杭州深圳云环境：http://mq4finance-sz.addr.aliyun.com:8080/rocketmq/nsaddr4client-internal
onsAddr: 'http://onsaddr-internet.aliyun.com/rocketmq/nsaddr4client-internet',
// https://help.aliyun.com/document_detail/102895.html 阿里云产品更新，支持实例化
// nameSrv: 'onsaddr.mq-internet-access.mq-internet.aliyuncs.com:80',
onsChannel: 'ALIYUN', // CLOUD, ALIYUN, ALL
};
class ClientConfig extends Base {
/**
* Producer 与 Consumer的公共配置
* @param {Object} options
* - {String} instanceName 示例名称
* - {Number} pollNameServerInteval name server 同步的间隔
* - {Number} heartbeatBrokerInterval 心跳的间隔
* - {Number} persistConsumerOffsetInterval 持久化消费进度的间隔
* - {Boolean} unitMode 是否为单元化的订阅组
*/
constructor(options) {
super(Object.assign({}, defaultOptions, options));
this.instanceName = this.options.instanceName;
}
get clientId() {
return this.unitName ?
`${this.options.clientIP}@${this.instanceName}@${this.unitName}` :
`${this.options.clientIP}@${this.instanceName}`;
}
get pollNameServerInteval() {
return this.options.pollNameServerInteval;
}
get heartbeatBrokerInterval() {
return this.options.heartbeatBrokerInterval;
}
get persistConsumerOffsetInterval() {
return this.options.persistConsumerOffsetInterval;
}
get rebalanceInterval() {
return this.options.rebalanceInterval;
}
get unitMode() {
return this.options.unitMode;
}
get unitName() {
return this.options.unitName;
}
get namespace() {
return this.options.namespace;
}
formatTopic(topic) {
if (this.namespace && (!topic.startsWith(this.namespace) &&
!topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX))) {
topic = `${this.namespace}%${topic}`;
}
return topic;
}
/**
* 将实例名修改为进程号
*/
changeInstanceNameToPID() {
if (this.instanceName === 'DEFAULT') {
this.instanceName = process.pid + '';
}
}
}
module.exports = ClientConfig;
================================================
FILE: lib/consumer/consume_from_where.js
================================================
'use strict';
module.exports = {
/**
* 一个新的订阅组第一次启动从队列的最后位置开始消费<br>
* 后续再启动接着上次消费的进度开始消费
*/
CONSUME_FROM_LAST_OFFSET: 'CONSUME_FROM_LAST_OFFSET',
CONSUME_FROM_LAST_OFFSET_AND_FROM_MIN_WHEN_BOOT_FIRST: 'CONSUME_FROM_LAST_OFFSET_AND_FROM_MIN_WHEN_BOOT_FIRST',
CONSUME_FROM_MIN_OFFSET: 'CONSUME_FROM_MIN_OFFSET',
CONSUME_FROM_MAX_OFFSET: 'CONSUME_FROM_MAX_OFFSET',
/**
* 一个新的订阅组第一次启动从队列的最前位置开始消费<br>
* 后续再启动接着上次消费的进度开始消费
*/
CONSUME_FROM_FIRST_OFFSET: 'CONSUME_FROM_FIRST_OFFSET',
/**
* 一个新的订阅组第一次启动从指定时间点开始消费<br>
* 后续再启动接着上次消费的进度开始消费<br>
* 时间点设置参见DefaultMQPushConsumer.consumeTimestamp参数
*/
CONSUME_FROM_TIMESTAMP: 'CONSUME_FROM_TIMESTAMP',
};
================================================
FILE: lib/consumer/mq_push_consumer.js
================================================
'use strict';
const assert = require('assert');
const is = require('is-type-of');
const utils = require('../utils');
const logger = require('../logger');
const MixAll = require('../mix_all');
const MQClient = require('../mq_client');
const MQProducer = require('../producer/mq_producer');
const sleep = require('mz-modules/sleep');
const ClientConfig = require('../client_config');
const ProcessQueue = require('../process_queue');
const PullStatus = require('../consumer/pull_status');
const PullSysFlag = require('../utils/pull_sys_flag');
const ConsumeFromWhere = require('./consume_from_where');
const MessageModel = require('../protocol/message_model');
const ConsumeType = require('../protocol/consume_type');
const ReadOffsetType = require('../store/read_offset_type');
const LocalFileOffsetStore = require('../store/local_file');
const LocalMemoryOffsetStore = require('../store/local_memory');
const RemoteBrokerOffsetStore = require('../store/remote_broker');
const AllocateMessageQueueAveragely = require('./rebalance/allocate_message_queue_averagely');
const Message = require('../message/message');
const MessageConst = require('../message/message_const');
const defaultOptions = {
logger,
persistent: false, // 是否持久化消费进度
isBroadcast: false, // 是否是广播模式（默认集群消费模式）
brokerSuspendMaxTimeMillis: 1000 * 15, // 长轮询模式，Consumer连接在Broker挂起最长时间
pullTimeDelayMillsWhenException: 3000, // 拉消息异常时，延迟一段时间再拉
pullTimeDelayMillsWhenFlowControl: 5000, // 进入流控逻辑，延迟一段时间再拉
consumerTimeoutMillisWhenSuspend: 1000 * 30, // 长轮询模式，Consumer超时时间（必须要大于brokerSuspendMaxTimeMillis）
consumerGroup: MixAll.DEFAULT_CONSUMER_GROUP,
consumeFromWhere: ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET, // Consumer第一次启动时，从哪里开始消费
/**
* Consumer第一次启动时，如果回溯消费，默认回溯到哪个时间点，数据格式如下，时间精度秒：
* 20131223171201
* 表示2013年12月23日17点12分01秒
* 默认回溯到相对启动时间的半小时前
*/
consumeTimestamp: utils.timeMillisToHumanString(Date.now() - 1000 * 60 * 30),
pullThresholdForQueue: 500, // 本地队列消息数超过此阀值，开始流控
pullInterval: 0, // 拉取消息的频率, 如果为了降低拉取速度，可以设置大于0的值
consumeMessageBatchMaxSize: 1, // 消费一批消息，最大数
pullBatchSize: 32, // 拉消息，一次拉多少条
parallelConsumeLimit: 1, // 并发消费消息限制
postSubscriptionWhenPull: true, // 是否每次拉消息时，都上传订阅关系
allocateMessageQueueStrategy: new AllocateMessageQueueAveragely(), // 队列分配算法，应用可重写
maxReconsumeTimes: 16, // 最大重试次数
};
class MQPushConsumer extends ClientConfig {
constructor(options) {
assert(options && options.consumerGroup, '[MQPushConsumer] options.consumerGroup is required');
const mergedOptions = Object.assign({ initMethod: 'init' }, defaultOptions, options);
assert(mergedOptions.parallelConsumeLimit <= mergedOptions.pullBatchSize,
'[MQPushConsumer] options.parallelConsumeLimit must lte pullBatchSize');
super(mergedOptions);
// @example:
// pullFromWhichNodeTable => {
// '[topic="TEST_TOPIC", brokerName="qdinternet-03", queueId="1"]': 0
// }
this._pullFromWhichNodeTable = new Map();
this._subscriptions = new Map();
this._handles = new Map();
this._topicSubscribeInfoTable = new Map();
this._processQueueTable = new Map();
this._inited = false;
this._isClosed = false;
if (this.messageModel === MessageModel.CLUSTERING) {
this.changeInstanceNameToPID();
}
this._mqClient = MQClient.getAndCreateMQClient(this);
this._offsetStore = this.newOffsetStoreInstance();
this._mqClient.on('error', err => this._handleError(err));
this._offsetStore.on('error', err => this._handleError(err));
}
get logger() {
return this.options.logger;
}
get subscriptions() {
return this._subscriptions;
}
get processQueueTable() {
return this._processQueueTable;
}
get parallelConsumeLimit() {
return this.options.parallelConsumeLimit;
}
get consumerGroup() {
if (this.namespace) {
return `${this.namespace}%${this.options.consumerGroup}`;
}
return this.options.consumerGroup;
}
get messageModel() {
return this.options.isBroadcast ? MessageModel.BROADCASTING : MessageModel.CLUSTERING;
}
get consumeType() {
return ConsumeType.CONSUME_PASSIVELY;
}
get consumeFromWhere() {
return this.options.consumeFromWhere;
}
get allocateMessageQueueStrategy() {
return this.options.allocateMessageQueueStrategy;
}
async init() {
this._mqClient.registerConsumer(this.consumerGroup, this);
await MQProducer.getDefaultProducer(this.options);
await this._mqClient.ready();
await this._offsetStore.load();
this.logger.info('[mq:consumer] consumer started');
this._inited = true;
// 订阅重试 TOPIC
if (this.messageModel === MessageModel.CLUSTERING) {
const retryTopic = MixAll.getRetryTopic(this.consumerGroup);
this.subscribe(retryTopic, '*', async msg => {
const originTopic = msg.retryTopic;
const originMsgId = msg.originMessageId;
const subscription = this._subscriptions.get(originTopic) || {};
const handler = subscription.handler;
if (!MixAll.isRetryTopic(originTopic) && handler) {
await handler(msg);
} else {
this.logger.warn('[MQPushConsumer] retry message no handler, originTopic: %s, originMsgId: %s, msgId: %s',
originTopic,
originMsgId,
msg.msgId);
}
});
}
}
/**
* close the consumer
*/
async close() {
this._isClosed = true;
await this.persistConsumerOffset();
this._pullFromWhichNodeTable.clear();
this._subscriptions.clear();
this._topicSubscribeInfoTable.clear();
this._processQueueTable.clear();
await this._mqClient.unregisterConsumer(this.consumerGroup);
await this._mqClient.close();
this.logger.info('[mq:consumer] consumer closed');
this.emit('close');
}
newOffsetStoreInstance() {
if (this.messageModel === MessageModel.BROADCASTING) {
if (this.options.persistent) {
return new LocalFileOffsetStore(this._mqClient, this.consumerGroup);
}
return new LocalMemoryOffsetStore(this._mqClient, this.consumerGroup);
}
return new RemoteBrokerOffsetStore(this._mqClient, this.consumerGroup);
}
/**
* subscribe
* @param {String} topic - topic
* @param {String} subExpression - tag
* @param {Function} handler - message handler
* @return {void}
*/
subscribe(topic, subExpression, handler) {
// 添加 namespace 前缀
topic = this.formatTopic(topic);
if (arguments.length === 2) {
handler = subExpression;
subExpression = null;
}
assert(is.asyncFunction(handler), '[MQPushConsumer] handler should be a asyncFunction');
assert(!this.subscriptions.has(topic), `[MQPushConsumer] ONLY one handler allowed for topic=${topic}`);
const subscriptionData = this.buildSubscriptionData(this.consumerGroup, topic, subExpression);
const tagsSet = subscriptionData.tagsSet;
const needFilter = !!tagsSet.length;
this.subscriptions.set(topic, {
handler,
subscriptionData,
});
(async () => {
try {
await this.ready();
// 如果 topic 没有路由信息，先更新一下
if (!this._topicSubscribeInfoTable.has(topic)) {
await this._mqClient.updateAllTopicRouterInfo();
await this._mqClient.sendHeartbeatToAllBroker();
await this._mqClient.doRebalance();
}
// 消息消费循环
while (!this._isClosed && this.subscriptions.has(topic)) {
await this._consumeMessageLoop(topic, needFilter, tagsSet, subExpression);
}
} catch (err) {
this._handleError(err);
}
})();
this.logger.info('[MQPushConsumer] cancel subscribe for topic=%s, subExpression=%s', topic, subExpression);
}
async _consumeMessageLoop(topic, needFilter, tagsSet, subExpression) {
const mqList = this._topicSubscribeInfoTable.get(topic);
let hasMsg = false;
if (mqList && mqList.length) {
for (const mq of mqList) {
const item = this._processQueueTable.get(mq.key);
if (item) {
const pq = item.processQueue;
this.logger.debug('[MQPushConsumer] process msg for processQueue=%s, msgCount=%', mq.key, pq.msgCount);
while (pq.msgCount) {
hasMsg = true;
let msgs;
if (this.parallelConsumeLimit > pq.msgCount) {
msgs = pq.msgList.slice(0, pq.msgCount);
} else {
msgs = pq.msgList.slice(0, this.parallelConsumeLimit);
}
// 并发消费任务
const consumeTasks = [];
for (const msg of msgs) {
const handler = this._subscriptions.get(msg.topic).handler;
if (!msg.tags || !needFilter || tagsSet.includes(msg.tags)) {
consumeTasks.push(this.consumeSingleMsg(handler, msg, mq, pq));
} else {
this.logger.debug('[MQPushConsumer] message filter by tags=, msg.tags=%s', subExpression, msg.tags);
}
}
// 必须全部成功
try {
await Promise.all(consumeTasks);
} catch (err) {
continue;
}
// 注意这里必须是批量确认
const offset = pq.remove(msgs.length);
if (offset >= 0) {
this._offsetStore.updateOffset(mq, offset, true);
}
}
}
}
}
if (!hasMsg) {
const changedEvent = `topic_${topic}_changed`;
this.logger.debug(`[MQPushConsumer] waiting event for ${changedEvent}`);
await this.await(changedEvent);
}
}
async consumeSingleMsg(handler, msg, mq, pq) {
// 集群消费模式下，如果消费失败，反复重试
while (!this._isClosed) {
if (msg.reconsumeTimes > this.options.maxReconsumeTimes) {
this.logger.warn('[MQPushConsumer] consume message failed, drop it for reconsumeTimes=%d and maxReconsumeTimes=%d, msgId: %s, originMsgId: %s',
msg.reconsumeTimes, this.options.maxReconsumeTimes, msg.msgId, msg.originMessageId);
return;
}
utils.resetRetryTopic(msg, this.consumerGroup);
try {
const value = await handler(msg, mq, pq);
if (value !== MQPushConsumer.ACTION_RETRY) {
return;
}
} catch (err) {
err.message = `process mq message failed, topic: ${msg.topic}, msgId: ${msg.msgId}, ${err.message}`;
this.emit('error', err);
}
if (this.messageModel === MessageModel.CLUSTERING) {
// 发送重试消息
try {
// delayLevel 为 0 代表由服务端控制重试间隔
await this.sendMessageBack(msg, 0, mq.brokerName, this.consumerGroup);
return;
} catch (err) {
this.emit('error', err);
this.logger.error(
'[MQPushConsumer] send reconsume message failed, fallback to local retry, msgId: %s',
msg.msgId
);
// 重试消息发送失败，本地重试
await this._sleep(5000);
}
// 本地重试情况下需要给 reconsumeTimes +1
msg.reconsumeTimes++;
} else {
this.logger.warn('[MQPushConsumer] BROADCASTING consume message failed, drop it, msgId: %s', msg.msgId);
return;
}
}
this.logger.info('[MQPushConsumer] consumer is closed, skip consume msg, msgId=%s', msg.msgId);
}
/**
* construct subscription data
* @param {String} consumerGroup - consumer group name
* @param {String} topic - topic
* @param {String} subString - tag
* @return {Object} subscription
*/
buildSubscriptionData(consumerGroup, topic, subString) {
const subscriptionData = {
topic,
subString,
classFilterMode: false,
tagsSet: [],
codeSet: [],
subVersion: Date.now(),
};
if (is.nullOrUndefined(subString) || subString === '*' || subString === '') {
subscriptionData.subString = '*';
} else if (typeof subString === 'object') {
subscriptionData.expressionType = subString.expressionType;
subscriptionData.subString = subString.subString;
} else {
const tags = subString.split('||');
for (let tag of tags) {
tag = tag.trim();
if (tag) {
subscriptionData.tagsSet.push(tag);
subscriptionData.codeSet.push(utils.hashCode(tag));
}
}
}
return subscriptionData;
}
async persistConsumerOffset() {
const mqs = [];
for (const key of this._processQueueTable.keys()) {
if (this._processQueueTable.get(key)) {
mqs.push(this._processQueueTable.get(key).messageQueue);
}
}
await this._offsetStore.persistAll(mqs);
}
/**
* pull message from queue
* @param {MessageQueue} messageQueue - message queue
* @param {ProcessQueue} processQueue - process queue
* @return {void}
*/
pullMessageQueue(messageQueue, processQueue) {
const _self = this;
(async () => {
while (!_self._isClosed) {
const pullRequest = _self._processQueueTable.get(messageQueue.key);
if (!pullRequest) {
break;
}
if (pullRequest.processQueue !== processQueue) {
// 不是同一个引用，退出循环，避免重复监听
break;
}
try {
await _self.executePullRequestImmediately(messageQueue);
await _self._sleep(_self.options.pullInterval);
} catch (err) {
if (!_self._isClosed) {
err.name = 'MQConsumerPullMessageError';
err.message = `[mq:consumer] pull message for queue: ${messageQueue.key}, occurred error: ${err.message}`;
_self._handleError(err);
await _self._sleep(_self.options.pullTimeDelayMillsWhenException);
}
}
}
_self.logger.info('[MQPushConsumer] stop pulling message from queue: %s', messageQueue.key);
})();
}
/**
* execute pull message immediately
* @param {MessageQueue} messageQueue - messageQueue
* @return {Promise}
*/
async executePullRequestImmediately(messageQueue) {
// close or queue removed
if (!this._processQueueTable.has(messageQueue.key)) {
return;
}
const pullRequest = this._processQueueTable.get(messageQueue.key);
const processQueue = pullRequest.processQueue;
// queue droped
if (processQueue.droped) {
return;
}
// flow control
const size = processQueue.msgCount;
if (size > this.options.pullThresholdForQueue) {
await this._sleep(this.options.pullTimeDelayMillsWhenFlowControl);
return;
}
processQueue.lastPullTimestamp = Date.now();
const data = this.subscriptions.get(messageQueue.topic);
const subscriptionData = data && data.subscriptionData;
if (!subscriptionData) {
this.logger.warn('[mq:consumer] execute pull request, but subscriptionData not found, topic: %s, queueId: %s', messageQueue.topic, messageQueue.queueId);
await this._sleep(this.options.pullTimeDelayMillsWhenException);
return;
}
let commitOffset = 0;
const subExpression = this.options.postSubscriptionWhenPull ? subscriptionData.subString : null;
const expressionType = subscriptionData.expressionType;
const subVersion = subscriptionData.subVersion;
// cluster model
if (MessageModel.CLUSTERING === this.messageModel) {
const offset = await this._offsetStore.readOffset(pullRequest.messageQueue, ReadOffsetType.READ_FROM_MEMORY);
if (offset) {
commitOffset = offset;
}
}
this.logger.info('[MQPushConsumer] start to pull message from queue: %s, nextOffset: %s, commitOffset: %s, subExpression: %s, expressionType: %s, subVersion: %s',
messageQueue.key, pullRequest.nextOffset, commitOffset, subExpression, expressionType, subVersion);
const pullResult = await this.pullKernelImpl(messageQueue, subExpression, expressionType, subVersion, pullRequest.nextOffset, commitOffset);
this.updatePullFromWhichNode(messageQueue, pullResult.suggestWhichBrokerId);
const originOffset = pullRequest.nextOffset;
// update next pull offset
pullRequest.nextOffset = pullResult.nextBeginOffset;
this.logger.info('[MQPushConsumer] pull message result: %s from queue: %s, requestOffset: %s, nextBeginOffset: %s, minOffset: %s, maxOffset: %s, suggestWhichBrokerId: %s',
pullResult.pullStatus, messageQueue.key, originOffset, pullResult.nextBeginOffset, pullResult.minOffset, pullResult.maxOffset, pullResult.suggestWhichBrokerId);
switch (pullResult.pullStatus) {
case PullStatus.FOUND:
{
const pullRT = Date.now() - processQueue.lastPullTimestamp;
const msgIds = pullResult.msgFoundList.map(v => v.msgId);
this.logger.info('[MQPushConsumer] pull message success, found new message size: %d, topic: %s, msgId: [%s], consumerGroup: %s, messageQueue: %s, cost: %dms.',
pullResult.msgFoundList.length, messageQueue.topic, msgIds.join(','), this.consumerGroup, messageQueue.key, pullRT);
// submit to consumer
processQueue.putMessage(pullResult.msgFoundList);
this.emit(`topic_${messageQueue.topic}_changed`);
break;
}
case PullStatus.NO_NEW_MSG:
case PullStatus.NO_MATCHED_MSG:
this.logger.debug('[mq:consumer] no new message for topic: %s at message queue => %s', subscriptionData.topic, messageQueue.key);
this.correctTagsOffset(pullRequest);
break;
case PullStatus.OFFSET_ILLEGAL:
this.logger.warn('[mq:consumer] the pull request offset illegal, message queue => %s, the originOffset => %d, pullResult => %j', messageQueue.key, originOffset, pullResult);
this._offsetStore.updateOffset(messageQueue, pullRequest.nextOffset);
break;
default:
break;
}
}
async pullKernelImpl(messageQueue, subExpression, expressionType, subVersion, offset, commitOffset) {
let sysFlag = PullSysFlag.buildSysFlag( //
commitOffset > 0, // commitOffset
true, // suspend
!!subExpression, // subscription
false // class filter
);
const result = await this.findBrokerAddress(messageQueue);
if (!result) {
throw new Error(`The broker[${messageQueue.brokerName}] not exist`);
}
// Slave不允许实时提交消费进度，可以定时提交
if (result.slave) {
sysFlag = PullSysFlag.clearCommitOffsetFlag(sysFlag);
}
const requestHeader = {
consumerGroup: this.consumerGroup,
topic: messageQueue.topic,
queueId: messageQueue.queueId,
queueOffset: offset,
maxMsgNums: this.options.pullBatchSize,
sysFlag,
commitOffset,
suspendTimeoutMillis: this.options.brokerSuspendMaxTimeMillis,
subscription: subExpression,
subVersion,
};
// 拉取消息时需要设置表达式类型，未设置默认为 tag
if (expressionType) {
requestHeader.expressionType = expressionType;
}
return await this._mqClient.pullMessage(result.brokerAddr, requestHeader, this.options.consumerTimeoutMillisWhenSuspend);
}
async findBrokerAddress(messageQueue) {
let findBrokerResult = this._mqClient.findBrokerAddressInSubscribe(
messageQueue.brokerName, this.recalculatePullFromWhichNode(messageQueue), false);
if (!findBrokerResult) {
await this._mqClient.updateTopicRouteInfoFromNameServer(messageQueue.topic);
findBrokerResult = this._mqClient.findBrokerAddressInSubscribe(
messageQueue.brokerName, this.recalculatePullFromWhichNode(messageQueue), false);
}
return findBrokerResult;
}
recalculatePullFromWhichNode(messageQueue) {
// @example:
// pullFromWhichNodeTable => {
// '[topic="TEST_TOPIC", brokerName="qdinternet-03", queueId="1"]': 0
// }
return this._pullFromWhichNodeTable.get(messageQueue.key) || MixAll.MASTER_ID;
}
correctTagsOffset(pullRequest) {
// 仅当已拉下的消息消费完的情况下才更新 offset
if (pullRequest.processQueue.msgCount === 0) {
this._offsetStore.updateOffset(pullRequest.messageQueue, pullRequest.nextOffset, true);
}
}
updatePullFromWhichNode(messageQueue, brokerId) {
this._pullFromWhichNodeTable.set(messageQueue.key, brokerId);
}
/**
* update subscription data
* @param {String} topic - topic
* @param {Array} info - info
* @return {void}
*/
updateTopicSubscribeInfo(topic, info) {
if (this._subscriptions.has(topic)) {
this._topicSubscribeInfoTable.set(topic, info);
}
}
/**
* whether need update
* @param {String} topic - topic
* @return {Boolean} need update?
*/
isSubscribeTopicNeedUpdate(topic) {
if (this._subscriptions && this._subscriptions.has(topic)) {
return !this._topicSubscribeInfoTable.has(topic);
}
return false;
}
/**
* rebalance
* @return {void}
*/
async doRebalance() {
for (const topic of this.subscriptions.keys()) {
await this.rebalanceByTopic(topic);
}
}
async rebalanceByTopic(topic) {
this.logger.info('[mq:consumer] rebalanceByTopic: %s, messageModel: %s', topic, this.messageModel);
const mqSet = this._topicSubscribeInfoTable.get(topic); // messageQueue list
if (!mqSet || !mqSet.length) {
this.logger.warn('[mq:consumer] doRebalance, %s, but the topic[%s] not exist.', this.consumerGroup, topic);
return;
}
let changed;
let allocateResult = mqSet;
if (this.options.isBroadcast) {
changed = await this.updateProcessQueueTableInRebalance(topic, mqSet);
} else {
const cidAll = await this._mqClient.findConsumerIdList(topic, this.consumerGroup);
this.logger.info('[mq:consumer] rebalance topic: %s, with consumer ids: %j', topic, cidAll);
if (cidAll && cidAll.length) {
// 排序
mqSet.sort(compare);
cidAll.sort();
allocateResult = this.allocateMessageQueueStrategy.allocate(this.consumerGroup, this._mqClient.clientId, mqSet, cidAll);
this.logger.info('[mq:consumer] allocate queue for group: %s, clientId: %s, result: %j', this.consumerGroup, this._mqClient.clientId, allocateResult);
changed = await this.updateProcessQueueTableInRebalance(topic, allocateResult);
}
}
if (changed) {
this.logger.info('[mq:consumer] do rebalance and message queue changed, topic: %s, mqSet: %j', topic, allocateResult);
this.emit(`topic_${topic}_queue_changed`);
}
}
/**
* update process queue
* @param {String} topic - topic
* @param {Array} mqSet - message queue set
* @return {Promise<void>}
*/
async updateProcessQueueTableInRebalance(topic, mqSet) {
let changed = false;
// delete unnecessary queue
for (const key of this._processQueueTable.keys()) {
const obj = this._processQueueTable.get(key);
const messageQueue = obj.messageQueue;
const processQueue = obj.processQueue;
if (topic === messageQueue.topic) {
// not found in mqSet, that means the process queue is unnecessary.
if (!mqSet.some(mq => mq.key === messageQueue.key)) {
processQueue.droped = true;
await this.removeProcessQueue(messageQueue);
changed = true;
} else if (processQueue.isPullExpired && this.consumeType === ConsumeType.CONSUME_PASSIVELY) {
processQueue.droped = true;
await this.removeProcessQueue(messageQueue);
changed = true;
this.logger.warn('[MQPushConsumer] BUG doRebalance, %s, remove unnecessary mq=%s, because pull is pause, so try to fixed it',
this.consumerGroup, messageQueue.key);
}
}
}
for (const messageQueue of mqSet) {
if (this._processQueueTable.has(messageQueue.key)) {
continue;
}
const nextOffset = await this.computePullFromWhere(messageQueue);
// double check
if (this._processQueueTable.has(messageQueue.key)) {
continue;
}
if (nextOffset >= 0) {
const processQueue = new ProcessQueue();
changed = true;
this._processQueueTable.set(messageQueue.key, {
messageQueue,
processQueue,
nextOffset,
});
// start to pull this queue;
this.pullMessageQueue(messageQueue, processQueue);
this.logger.info('[mq:consumer] doRebalance, %s, add a new messageQueue, %j, its nextOffset: %s', this.consumerGroup, messageQueue, nextOffset);
} else {
this.logger.warn('[mq:consumer] doRebalance, %s, new messageQueue, %j, has invalid nextOffset: %s', this.consumerGroup, messageQueue, nextOffset);
}
}
return changed;
}
/**
* compute consume offset
* @param {MessageQueue} messageQueue - message queue
* @return {Promise<Number>} offset
*/
async computePullFromWhere(messageQueue) {
try {
const lastOffset = await this._offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_STORE);
this.logger.info('[mq:consumer] read lastOffset => %s from store, topic="%s", brokerName="%s", queueId="%s"', lastOffset, messageQueue.topic, messageQueue.brokerName, messageQueue.queueId);
let result = -1;
switch (this.consumeFromWhere) {
case ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET_AND_FROM_MIN_WHEN_BOOT_FIRST:
case ConsumeFromWhere.CONSUME_FROM_MIN_OFFSET:
case ConsumeFromWhere.CONSUME_FROM_MAX_OFFSET:
case ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET:
// 第二次启动，根据上次的消费位点开始消费
if (lastOffset >= 0) {
result = lastOffset;
} else if (lastOffset === -1) { // 第一次启动，没有记录消费位点
// 重试队列则从队列头部开始
if (messageQueue.topic.indexOf(MixAll.RETRY_GROUP_TOPIC_PREFIX) === 0) {
result = 0;
} else { // 正常队列则从队列尾部开始
return await this._mqClient.maxOffset(messageQueue);
}
}
break;
case ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET:
// 第二次启动，根据上次的消费位点开始消费
if (lastOffset >= 0) {
result = lastOffset;
} else {
result = 0;
}
break;
case ConsumeFromWhere.CONSUME_FROM_TIMESTAMP:
// 第二次启动，根据上次的消费位点开始消费
if (lastOffset >= 0) {
result = lastOffset;
} else if (lastOffset === -1) { // 第一次启动，没有记录消费为点
// 重试队列则从队列尾部开始
if (messageQueue.topic.indexOf(MixAll.RETRY_GROUP_TOPIC_PREFIX) === 0) {
return await this._mqClient.maxOffset(messageQueue);
}
// 正常队列则从指定时间点开始
// 时间点需要参数配置
const timestamp = utils.parseDate(this.options.consumeTimestamp).getTime();
return await this._mqClient.searchOffset(messageQueue, timestamp);
}
break;
default:
break;
}
this.logger.info('[mq:consumer] computePullFromWhere() messageQueue => %s should read from offset: %s and lastOffset: %s', messageQueue.key, result, lastOffset);
return result;
} catch (err) {
err.mesasge = 'computePullFromWhere() occurred an exception, ' + err.mesasge;
this._handleError(err);
return -1;
}
}
/**
* 移除消费队列
* @param {MessageQueue} messageQueue - message queue
* @return {Promise<void>}
*/
async removeProcessQueue(messageQueue) {
const processQueue = this._processQueueTable.get(messageQueue.key);
this._processQueueTable.delete(messageQueue.key);
if (processQueue) {
processQueue.droped = true;
await this.removeUnnecessaryMessageQueue(messageQueue, processQueue);
this.logger.info('[mq:consumer] remove unnecessary messageQueue, %s, Droped: %s', messageQueue.key, processQueue.droped);
}
}
/**
* remove unnecessary queue
* @param {MessageQueue} messageQueue - message queue
* @return {Promise<void>}
*/
async removeUnnecessaryMessageQueue(messageQueue) {
await this._offsetStore.persist(messageQueue);
this._offsetStore.removeOffset(messageQueue);
// todo: consume later ？
}
async sendMessageBack(msg, delayLevel, brokerName, consumerGroup) {
const brokerAddr = brokerName ? this._mqClient.findBrokerAddressInPublish(brokerName) :
msg.storeHost;
const thatConsumerGroup = consumerGroup ? consumerGroup : this.consumerGroup;
try {
await this._mqClient.consumerSendMessageBack(
brokerAddr,
msg,
thatConsumerGroup,
delayLevel,
3000,
this.options.maxReconsumeTimes);
this.logger.info('[MQPushConsumer] consumerSendMessageBack success, topic=%s, consumerGroup=%s, msgId=%s', msg.topic, thatConsumerGroup, msg.msgId);
} catch (err) {
err.mesasge = 'sendMessageBack() occurred an exception, ' + thatConsumerGroup + ', ' + err.mesasge;
this._handleError(err);
let newMsg;
if (MixAll.isRetryTopic(msg.topic)) {
newMsg = msg;
} else {
newMsg = new Message(MixAll.getRetryTopic(thatConsumerGroup), '', msg.body);
newMsg.flag = msg.flag;
newMsg.properties = msg.properties;
newMsg.originMessageId = msg.originMessageId || msg.msgId;
newMsg.retryTopic = msg.topic;
// 这里需要加 1，因为如果 maxReconsumeTimes 为 1，那么这条 retry 消息发出去始终不会被重新投递了
newMsg.properties[MessageConst.PROPERTY_MAX_RECONSUME_TIMES] = String(this.options.maxReconsumeTimes + 1);
}
newMsg.properties[MessageConst.PROPERTY_RECONSUME_TIME] = String(msg.reconsumeTimes + 1);
newMsg.delayTimeLevel = 0;
await (await MQProducer.getDefaultProducer()).send(newMsg);
}
}
// * viewMessage(msgId) {
// const info = MessageDecoder.decodeMessageId(msgId);
// return yield this._mqClient.viewMessage(info.address, Number(info.offset.toString()), 3000);
// }
_handleError(err) {
err.message = 'MQPushConsumer occurred an error ' + err.message;
this.emit('error', err);
}
_sleep(timeout) {
return sleep(timeout);
}
}
// if subscriber return ACTION_RETRY, message will be directly retried
MQPushConsumer.ACTION_RETRY = Symbol('ACTION_RETRY');
module.exports = MQPushConsumer;
// Helper
// ------------------
function compare(mqA, mqB) {
if (mqA.topic === mqB.topic) {
if (mqA.brokerName === mqB.brokerName) {
return mqA.queueId - mqB.queueId;
}
return mqA.brokerName > mqB.brokerName ? 1 : -1;
}
return mqA.topic > mqB.topic ? 1 : -1;
}
================================================
FILE: lib/consumer/pull_status.js
================================================
'use strict';
module.exports = {
/**
* 找到消息
*/
FOUND: 'FOUND',
/**
* 没有新的消息可以被拉取
*/
NO_NEW_MSG: 'NO_NEW_MSG',
/**
* 经过过滤后，没有匹配的消息
*/
NO_MATCHED_MSG: 'NO_MATCHED_MSG',
/**
* Offset不合法，可能过大或者过小
*/
OFFSET_ILLEGAL: 'OFFSET_ILLEGAL',
};
================================================
FILE: lib/consumer/rebalance/allocate_message_queue_averagely.js
================================================
'use strict';
const debug = require('debug')('mq:allocate');
const AllocateMessageQueueStrategy = require('./allocate_message_queue_strategy');
class AllocateMessageQueueAveragely extends AllocateMessageQueueStrategy {
get name() {
return 'AVG';
}
allocate(consumerGroup, currentCID, mqAll, cidAll) {
if (!currentCID || currentCID.length < 1) {
throw new Error('currentCID is empty');
}
if (!mqAll || !mqAll.length) {
throw new Error('mqAll is null or mqAll empty');
}
if (!cidAll || !cidAll.length) {
throw new Error('cidAll is null or cidAll empty');
}
const result = [];
const index = cidAll.indexOf(currentCID);
if (index === -1) { // 不存在此ConsumerId ,直接返回
debug('[BUG] ConsumerGroup: %s The consumerId: %s not in cidAll: %j', //
consumerGroup, //
currentCID, //
cidAll);
return result;
}
const mqLen = mqAll.length;
const cidLen = cidAll.length;
const mod = mqLen % cidLen;
let averageSize = mqLen <= cidLen ?
1 :
mod > 0 && index < mod ?
mqLen / cidLen + 1 :
mqLen / cidLen;
averageSize = Math.floor(averageSize); // 取整
const startIndex = mod > 0 && index < mod ? index * averageSize : index * averageSize + mod;
const range = Math.min(averageSize, mqLen - startIndex);
for (let i = 0; i < range; i++) {
result.push(mqAll[(startIndex + i) % mqLen]);
}
return result;
}
}
module.exports = AllocateMessageQueueAveragely;
================================================
FILE: lib/consumer/rebalance/allocate_message_queue_strategy.js
================================================
/* istanbul ignore next */
/* eslint valid-jsdoc:0 */
'use strict';
class AllocateMessageQueueStrategy {
/**
* 给当前的 ConsumerId 分配队列
* @param {String} consumerGroup -
* @param {String} currentCID - 当前 ConsumerId
* @param {Array} mqAll - 当前 Topic 的所有队列集合，无重复数据，且有序
* @param {Array} cidAll - 当前订阅组的所有 Consumer 集合，无重复数据，且有序
*/
allocate(consumerGroup, currentCID, mqAll, cidAll) { /* eslint no-unused-vars: 0 */
throw new Error('no implementation');
}
/**
* rebalance 算法的名字
*/
get name() {
throw new Error('no implementation');
}
}
module.exports = AllocateMessageQueueStrategy;
================================================
FILE: lib/index.js
================================================
'use strict';
exports.Message = require('./message/message');
exports.Producer = require('./producer/mq_producer');
exports.Consumer = require('./consumer/mq_push_consumer');
================================================
FILE: lib/logger.js
================================================
'use strict';
module.exports = {
info() {},
warn(...args) {
console.warn(...args);
},
error(...args) {
console.error(...args);
},
debug() {},
};
================================================
FILE: lib/message/message.js
================================================
'use strict';
const is = require('is-type-of');
const MessageConst = require('./message_const');
class Message {
/**
* 创建消息对象
* @param {String} topic -
* @param {String} tags -
* @param {String|Buffer} body -
* @class
*/
constructor(topic, tags, body) {
if (arguments.length === 2) {
body = tags;
tags = null;
}
this.storeSize = null;
this.bodyCRC = null;
this.queueId = null;
this.flag = 0;
this.queueOffset = null; // long
this.commitLogOffset = null; // long
this.bornHost = null;
this.bornTimestamp = null; // long
this.storeTimestamp = null; // long
this.storeHost = null;
this.reconsumeTimes = 0;
this.preparedTransactionOffset = null; // long
this.topic = topic;
this.properties = {};
this.msgId = null;
this.tags = tags;
if (body && is.string(body)) {
this.body = Buffer.from(body);
} else {
this.body = body;
}
}
/**
* 消息标签，用于过滤
* @property {String} Message#tags
*/
get tags() {
return this.properties && this.properties[MessageConst.PROPERTY_TAGS];
}
set tags(val) {
this.properties[MessageConst.PROPERTY_TAGS] = val;
}
/**
* 消息关键词
* @property {String} Message#keys
*/
get keys() {
return this.properties && this.properties[MessageConst.PROPERTY_KEYS];
}
set keys(val) {
this.properties[MessageConst.PROPERTY_KEYS] = val.join(MessageConst.KEY_SEPARATOR).trim();
}
/**
* 原始消息 Id
* @property {String} Message#originMessageId
*/
get originMessageId() {
return this.properties && this.properties[MessageConst.PROPERTY_ORIGIN_MESSAGE_ID];
}
set originMessageId(val) {
this.properties[MessageConst.PROPERTY_ORIGIN_MESSAGE_ID] = val;
}
/**
* 重试 Topic
* @property {String} Message#retryTopic
*/
get retryTopic() {
return this.properties && this.properties[MessageConst.PROPERTY_RETRY_TOPIC];
}
set retryTopic(val) {
this.properties[MessageConst.PROPERTY_RETRY_TOPIC] = val;
}
/**
* 消息延时投递时间级别，0表示不延时，大于0表示特定延时级别（具体级别在服务器端定义）
* @property {Number} Message#delayTimeLevel
*/
get delayTimeLevel() {
const t = this.properties && this.properties[MessageConst.PROPERTY_DELAY_TIME_LEVEL];
if (t) {
return parseInt(t, 10);
}
return 0;
}
set delayTimeLevel(val) {
this.properties[MessageConst.PROPERTY_DELAY_TIME_LEVEL] = val + '';
}
/**
* 是否等待服务器将消息存储完毕再返回（可能是等待刷盘完成或者等待同步复制到其他服务器）
* @property {Boolean} Message#waitStoreMsgOK
*/
get waitStoreMsgOK() {
const result = this.properties && this.properties[MessageConst.PROPERTY_WAIT_STORE_MSG_OK];
return !!result;
}
set waitStoreMsgOK(val) {
this.properties[MessageConst.PROPERTY_WAIT_STORE_MSG_OK] = String(!!val);
}
/**
* userId，用于单元化
* @property {String} Message#buyerId
*/
get buyerId() {
return this.properties && this.properties[MessageConst.PROPERTY_BUYER_ID];
}
set buyerId(val) {
this.properties[MessageConst.PROPERTY_BUYER_ID] = val;
}
/**
* 延迟执行的时间点，毫秒
* @param {number} delayTime 延迟执行目标时间戳
*/
setStartDeliverTime(delayTime) {
this.properties[MessageConst.SYSTEM_PROP_KEY_STARTDELIVERTIME] = delayTime;
}
/**
* 获取消息延迟时间
* @return {number} 延迟执行目标时间戳
*/
getStartDeliverTime() {
return this.properties[MessageConst.SYSTEM_PROP_KEY_STARTDELIVERTIME] || 0;
}
get shardingKey() {
return this.properties && this.properties[MessageConst.PROPERTY_SHARDING_KEY];
}
set shardingKey(val) {
this.properties[MessageConst.PROPERTY_SHARDING_KEY] = val;
}
}
module.exports = Message;
================================================
FILE: lib/message/message_const.js
================================================
'use strict';
// 消息关键词，多个Key用KEY_SEPARATOR隔开（查询消息使用）
exports.PROPERTY_KEYS = 'KEYS';
// 消息标签，只支持设置一个Tag（服务端消息过滤使用）
exports.PROPERTY_TAGS = 'TAGS';
// 是否等待服务器将消息存储完毕再返回（可能是等待刷盘完成或者等待同步复制到其他服务器）
exports.PROPERTY_WAIT_STORE_MSG_OK = 'WAIT';
// 消息延时投递时间级别，0表示不延时，大于0表示特定延时级别（具体级别在服务器端定义）
exports.PROPERTY_DELAY_TIME_LEVEL = 'DELAY';
// 内部使用
exports.PROPERTY_RETRY_TOPIC = 'RETRY_TOPIC';
exports.PROPERTY_REAL_TOPIC = 'REAL_TOPIC';
exports.PROPERTY_REAL_QUEUE_ID = 'REAL_QID';
exports.PROPERTY_TRANSACTION_PREPARED = 'TRAN_MSG';
exports.PROPERTY_PRODUCER_GROUP = 'PGROUP';
exports.PROPERTY_MIN_OFFSET = 'MIN_OFFSET';
exports.PROPERTY_MAX_OFFSET = 'MAX_OFFSET';
exports.PROPERTY_BUYER_ID = 'BUYER_ID';
exports.PROPERTY_SHARDING_KEY = '__SHARDING_KEY';
exports.PROPERTY_ORIGIN_MESSAGE_ID = 'ORIGIN_MESSAGE_ID';
exports.PROPERTY_TRANSFER_FLAG = 'TRANSFER_FLAG';
exports.PROPERTY_CORRECTION_FLAG = 'CORRECTION_FLAG';
exports.PROPERTY_MQ2_FLAG = 'MQ2_FLAG';
exports.PROPERTY_RECONSUME_TIME = 'RECONSUME_TIME';
exports.PROPERTY_MAX_RECONSUME_TIMES = 'MAX_RECONSUME_TIMES';
exports.KEY_SEPARATOR = ' ';
// 延迟消息配置key
exports.SYSTEM_PROP_KEY_STARTDELIVERTIME = '__STARTDELIVERTIME';
================================================
FILE: lib/message/message_decoder.js
================================================
'use strict';
/* eslint no-bitwise: 0 */
const debug = require('debug')('mq:decoder');
const Long = require('long');
const is = require('is-type-of');
const utils = require('../utils');
const ByteBuffer = require('byte');
const Message = require('./message');
const MessageSysFlag = require('../utils/message_sys_flag');
const MSG_ID_LENGTH = 8 + 8;
const NAME_VALUE_SEPARATOR = String.fromCharCode(1);
const PROPERTY_SEPARATOR = String.fromCharCode(2);
const byteBufferMsgId = ByteBuffer.allocate(MSG_ID_LENGTH);
// 将属性转换为字符串
exports.messageProperties2String = function(properties) {
let str = '';
if (properties) {
for (const key in properties) {
str += key + NAME_VALUE_SEPARATOR;
str += properties[key] + PROPERTY_SEPARATOR;
}
}
return str;
};
// 解析 messageId，从中提取出 address 和 offset 信息
exports.decodeMessageId = function(msgId) {
// 地址
const host = string2bytes(msgId.slice(0, 8));
const port = string2bytes(msgId.slice(8, 16)).readInt32BE(0);
// offset
const data = string2bytes(msgId.slice(16, 32));
const offset = new Long(
data.readInt32BE(4), // low, high
data.readInt32BE(0)
);
return {
address: host[0] + '.' + host[1] + '.' + host[2] + '.' + host[3] + ':' + port,
offset,
};
};
/**
* 解析一条消息
* @param {ByteBuffer} byteBuffer -
* @param {Boolean} readBody - 是否读取 body 部分
* @param {Boolean} deCompressBody - 是否解压 body
* @return {RemotingCommand} command
*/
const decode = exports.decode = function(byteBuffer, readBody, deCompressBody) {
if (is.nullOrUndefined(readBody)) {
readBody = true;
}
if (is.nullOrUndefined(deCompressBody)) {
deCompressBody = true;
}
try {
return innerDecode(byteBuffer, readBody, deCompressBody);
} catch (err) {
debug('Error occurred during innerDecode, err: %s', err.stack);
// 指向末尾
byteBuffer.position(byteBuffer.limit());
}
return null;
};
/**
* 执行解码
* @param {ByteBuffer} byteBuffer -
* @param {Boolean} readBody - 是否读取 body 部分
* @param {Boolean} deCompressBody - 是否解压 body
* @return {RemotingCommand} command
*/
function innerDecode(byteBuffer, readBody, deCompressBody) {
const msgExt = new Message();
// 消息ID
byteBufferMsgId.reset();
// 1 TOTALSIZE
msgExt.storeSize = byteBuffer.getInt();
// 2 MAGICCODE
byteBuffer.getInt();
// 3 BODYCRC
msgExt.bodyCRC = byteBuffer.getInt();
// 4 QUEUEID
msgExt.queueId = byteBuffer.getInt();
// 5 FLAG
msgExt.flag = byteBuffer.getInt();
// 6 QUEUEOFFSET
msgExt.queueOffset = byteBuffer.getLong().toNumber();
// 7 PHYSICALOFFSET
msgExt.commitLogOffset = byteBuffer.getLong().toNumber();
// 8 SYSFLAG
const sysFlag = byteBuffer.getInt();
msgExt.sysFlag = sysFlag;
// 9 BORNTIMESTAMP
msgExt.bornTimestamp = byteBuffer.getLong().toNumber();
// 10 BORNHOST
const host = Buffer.alloc(4);
byteBuffer.get(host);
let port = byteBuffer.getInt();
msgExt.bornHost = host[0] + '.' + host[1] + '.' + host[2] + '.' + host[3] + ':' + port;
// 11 STORETIMESTAMP
msgExt.storeTimestamp = byteBuffer.getLong().toNumber();
// 12 STOREHOST
host.fill(0);
byteBuffer.get(host);
port = byteBuffer.getInt();
msgExt.storeHost = host[0] + '.' + host[1] + '.' + host[2] + '.' + host[3] + ':' + port;
byteBufferMsgId.put(host);
byteBufferMsgId.putInt(port);
byteBufferMsgId.putLong(msgExt.commitLogOffset);
// 13 RECONSUMETIMES
msgExt.reconsumeTimes = byteBuffer.getInt();
// 14 Prepared Transaction Offset
msgExt.preparedTransactionOffset = byteBuffer.getLong().toNumber();
// 15 BODY
const bodyLen = byteBuffer.getInt();
if (bodyLen > 0) {
if (readBody) {
let body = Buffer.alloc(bodyLen);
byteBuffer.get(body);
// uncompress body
if (deCompressBody && (sysFlag & MessageSysFlag.CompressedFlag) === MessageSysFlag.CompressedFlag) {
body = utils.uncompress(body);
}
msgExt.body = body;
} else {
byteBuffer.position(byteBuffer.position() + bodyLen);
}
}
// 16 TOPIC
const topicLen = byteBuffer.get();
const topic = Buffer.alloc(topicLen);
byteBuffer.get(topic);
msgExt.topic = topic.toString();
// 17 properties
const propertiesLength = byteBuffer.getShort();
if (propertiesLength > 0) {
const properties = Buffer.alloc(propertiesLength);
byteBuffer.get(properties);
const propertiesString = properties.toString();
msgExt.properties = string2messageProperties(propertiesString);
}
msgExt.msgId = bytes2string(byteBufferMsgId.copy());
return msgExt;
}
/**
* 解析一批消息
* @param {ByteBuffer} byteBuffer -
* @param {Boolean} readBody - 是否读取 body 部分
* @return {RemotingCommand} command
*/
exports.decodes = function(byteBuffer, readBody) {
readBody = readBody || true;
const msgExts = [];
let msgExt;
while (byteBuffer.hasRemaining()) {
msgExt = decode(byteBuffer, readBody);
if (msgExt) {
msgExts.push(msgExt);
} else {
break;
}
}
return msgExts;
};
// Helper
// -----------
function string2bytes(hexString) {
if (!hexString) {
return null;
}
return Buffer.from(hexString, 'hex');
}
function bytes2string(src) {
if (!src) {
return null;
}
return src.toString('hex').toUpperCase();
}
function string2messageProperties(properties) {
const map = {
TAGS: null,
};
if (properties) {
const items = properties.split(PROPERTY_SEPARATOR);
for (const item of items) {
const index = item.indexOf(NAME_VALUE_SEPARATOR);
map[item.slice(0, index)] = item.slice(index + 1);
}
}
return map;
}
================================================
FILE: lib/message_queue.js
================================================
'use strict';
const fmt = require('util').format;
class MessageQueue {
constructor(topic, brokerName, queueId) {
this.topic = topic;
this.brokerName = brokerName;
this.queueId = queueId;
this.key = fmt('[topic="%s", brokerName="%s", queueId="%s"]', this.topic, this.brokerName, this.queueId);
}
}
module.exports = MessageQueue;
================================================
FILE: lib/mix_all.js
================================================
'use strict';
exports.DEFAULT_TOPIC = 'TBW102';
exports.DEFAULT_PRODUCER_GROUP = 'DEFAULT_PRODUCER';
exports.DEFAULT_CONSUMER_GROUP = 'DEFAULT_CONSUMER';
exports.CLIENT_INNER_PRODUCER_GROUP = 'CLIENT_INNER_PRODUCER';
exports.DEFAULT_CHARSET = 'UTF-8';
exports.MASTER_ID = 0;
// 为每个Consumer Group建立一个默认的Topic，前缀 + GroupName，用来保存处理失败需要重试的消息
exports.RETRY_GROUP_TOPIC_PREFIX = '%RETRY%';
// 为每个Consumer Group建立一个默认的Topic，前缀 + GroupName，用来保存重试多次都失败，接下来不再重试的消息
exports.DLQ_GROUP_TOPIC_PREFIX = '%DLQ%';
/**
* 获取 RETRY_TOPIC
* @param {string} consumerGroup consumerGroup
* @return {string} %RETRY%+consumerGroup
*/
exports.getRetryTopic = consumerGroup => {
return exports.RETRY_GROUP_TOPIC_PREFIX + consumerGroup;
};
/**
* 判断是否为 RETRY_TOPIC
* @param {String} topic topic
* @return {boolean} ret
*/
exports.isRetryTopic = topic => {
return topic && topic.startsWith(exports.RETRY_GROUP_TOPIC_PREFIX);
};
================================================
FILE: lib/mq_client.js
================================================
'use strict';
const is = require('is-type-of');
const gather = require('co-gather');
const sleep = require('mz-modules/sleep');
const utility = require('utility');
const MixAll = require('./mix_all');
const MQClientAPI = require('./mq_client_api');
const MessageQueue = require('./message_queue');
const PermName = require('./protocol/perm_name');
const TopicPublishInfo = require('./producer/topic_publish_info');
const instanceTable = new Map();
class MQClient extends MQClientAPI {
/**
* metaq client
* @param {Object} clientConfig -
* @class
*/
constructor(clientConfig) {
super(clientConfig.options);
this._clientConfig = clientConfig;
this._brokerAddrTable = new Map();
this._consumerTable = new Map();
this._producerTable = new Map();
this._topicRouteTable = new Map();
// this.API.on('command', this.handleServerRequest.bind(this));
}
/**
* @property {String} MQClient#clientId
*/
get clientId() {
return this._clientConfig.clientId;
}
/**
* @property {Number} MQClient#pollNameServerInteval
*/
get pollNameServerInteval() {
return this._clientConfig.pollNameServerInteval;
}
/**
* @property {Number} MQClient#heartbeatBrokerInterval
*/
get heartbeatBrokerInterval() {
return this._clientConfig.heartbeatBrokerInterval;
}
/**
* @property {Number} MQClient#persistConsumerOffsetInterval
*/
get persistConsumerOffsetInterval() {
return this._clientConfig.persistConsumerOffsetInterval;
}
/**
* @property {Number} MQClient#rebalanceInterval
*/
get rebalanceInterval() {
return this._clientConfig.rebalanceInterval;
}
/**
* start the client
*/
async init() {
await super.init();
await this.updateAllTopicRouterInfo();
await this.sendHeartbeatToAllBroker();
await this.doRebalance();
this.startScheduledTask('updateAllTopicRouterInfo', this.pollNameServerInteval);
this.startScheduledTask('sendHeartbeatToAllBroker', this.heartbeatBrokerInterval);
this.startScheduledTask('doRebalance', this.rebalanceInterval);
this.startScheduledTask('persistAllConsumerOffset', this.persistConsumerOffsetInterval);
}
async close() {
if (this._consumerTable.size || this._producerTable.size) {
return;
}
await super.close();
// todo:
}
/**
* start a schedule task
* @param {String} name - method name
* @param {Number} interval - schedule interval
* @param {Number} [delay] - delay time interval
* @return {void}
*/
startScheduledTask(name, interval, delay) {
(async () => {
await sleep(delay || interval);
while (this._inited) {
try {
this.logger.info('[mq:client] execute `%s` at %s', name, utility.YYYYMMDDHHmmss());
await this[name]();
} catch (err) {
this.logger.error(err);
}
await sleep(interval);
}
})();
}
/**
* regitser consumer
* @param {String} group - consumer group name
* @param {Consumer} consumer - consumer instance
* @return {void}
*/
registerConsumer(group, consumer) {
if (this._consumerTable.has(group)) {
this.logger.warn('[mq:client] the consumer group [%s] exist already.', group);
return;
}
this._consumerTable.set(group, consumer);
this.logger.info('[mq:client] new consumer has regitsered, group: %s', group);
}
/**
* unregister consumer
* @param {String} group - consumer group name
* @return {void}
*/
async unregisterConsumer(group) {
this._consumerTable.delete(group);
await this.unregister(null, group);
this.logger.info('[mq:client] unregister consumer, group: %s', group);
}
/**
* register producer
* @param {String} group - producer group name
* @param {Producer} producer - producer
* @return {void}
*/
registerProducer(group, producer) {
if (this._producerTable.has(group)) {
this.logger.warn('[mq:client] the producer group [%s] exist already.', group);
return;
}
this._producerTable.set(group, producer);
this.logger.info('[mq:client] new producer has regitsered, group: %s', group);
}
/**
* unregister producer
* @param {String} group - producer group name
* @return {void}
*/
async unregisterProducer(group) {
this._producerTable.delete(group);
await this.unregister(group, null);
this.logger.info('[mq:client] unregister producer, group: %s', group);
}
/**
* notify all broker that producer or consumer is offline
* @param {String} producerGroup - producer group name
* @param {String} consumerGroup - consumer group name
* @return {void}
*/
async unregister(producerGroup, consumerGroup) {
const brokerAddrTable = this._brokerAddrTable;
for (const brokerName of brokerAddrTable.keys()) {
const oneTable = brokerAddrTable.get(brokerName);
if (!oneTable) {
continue;
}
for (const id in oneTable) {
const addr = oneTable[id];
if (addr) {
await this.unregisterClient(addr, this.clientId, producerGroup, consumerGroup, 3000);
}
}
}
}
/**
* update all router info
* @return {void}
*/
async updateAllTopicRouterInfo() {
const topics = [];
// consumer
for (const groupName of this._consumerTable.keys()) {
const consumer = this._consumerTable.get(groupName);
if (!consumer) {
continue;
}
for (const topic of consumer.subscriptions.keys()) {
topics.push(topic);
}
}
// producer
for (const groupName of this._producerTable.keys()) {
const producer = this._producerTable.get(groupName);
if (!producer) {
continue;
}
for (const topic of producer.publishTopicList) {
topics.push(topic);
}
}
this.logger.info('[mq:client] try to update all topic route info. topic: %j', topics);
const ret = await gather(topics.map(topic => () => this.updateTopicRouteInfoFromNameServer(topic)));
ret.forEach(data => {
if (data.isError) {
data.error.message = `[mq:client] updateAllTopicRouterInfo occurred error, ${data.error.message}`;
this.emit('error', data.error);
}
});
}
/**
* update topic route info
* @param {String} topic - topic
* @param {Boolean} [isDefault] - is default or not
* @param {Producer} [defaultMQProducer] - producer
*/
async updateTopicRouteInfoFromNameServer(topic, isDefault, defaultMQProducer) {
this.logger.info('[mq:client] updateTopicRouteInfoFromNameServer() topic: %s, isDefault: %s', topic, !!isDefault);
if (isDefault && defaultMQProducer) {
const topicRouteData = await this.getDefaultTopicRouteInfoFromNameServer(defaultMQProducer.createTopicKey, 3000);
if (topicRouteData) {
for (const data of topicRouteData.queueDatas) {
const queueNums =
defaultMQProducer.defaultTopicQueueNums < data.readQueueNums ?
defaultMQProducer.defaultTopicQueueNums : data.readQueueNums;
data.readQueueNums = queueNums;
data.writeQueueNums = queueNums;
}
this._refreshTopicRouteInfo(topic, topicRouteData);
}
} else {
const topicRouteData = await this.getTopicRouteInfoFromNameServer(topic, 3000);
if (topicRouteData) {
this._refreshTopicRouteInfo(topic, topicRouteData);
}
}
}
_refreshTopicRouteInfo(topic, topicRouteData) {
if (!topicRouteData) {
return;
}
// @example
// topicRouteData => {
// "brokerDatas": [{
// "brokerAddrs": {
// "0": "10.218.145.166:10911"
// },
// "brokerName": "taobaodaily-02"
// }],
// "filterServerTable": {},
// "queueDatas": [{
// "brokerName": "taobaodaily-02",
// "perm": 6,
// "readQueueNums": 8,
// "topicSynFlag": 0,
// "writeQueueNums": 8
// }]
// }
const prev = this._topicRouteTable.get(topic);
const needUpdate = this._isRouteDataChanged(prev, topicRouteData) || this._isNeedUpdateTopicRouteInfo(topic);
this.logger.info('[mq:client] refresh route data for topic: %s, route data: %j, needUpdate: %s', topic, topicRouteData, needUpdate);
if (!needUpdate) {
return;
}
// @example:
// this.brokerAddrTable => {
// "taobaodaily-02": {
// "0": "10.218.145.166:10911"
// }
// }
for (const brokerData of topicRouteData.brokerDatas) {
this._brokerAddrTable.set(brokerData.brokerName, brokerData.brokerAddrs);
}
// update producer route data
const publishInfo = this._topicRouteData2TopicPublishInfo(topic, topicRouteData);
publishInfo.haveTopicRouterInfo = true;
for (const groupName of this._producerTable.keys()) {
const producer = this._producerTable.get(groupName);
if (producer) {
producer.updateTopicPublishInfo(topic, publishInfo);
}
}
// 更新订阅队列信息
const subscribeInfo = this._topicRouteData2TopicSubscribeInfo(topic, topicRouteData);
for (const groupName of this._consumerTable.keys()) {
const consumer = this._consumerTable.get(groupName);
if (consumer) {
consumer.updateTopicSubscribeInfo(topic, subscribeInfo);
}
}
this.logger.info('[mq:client] update topicRouteTable for topic: %s, with topicRouteData: %j', topic, topicRouteData);
this._topicRouteTable.set(topic, topicRouteData);
}
_isRouteDataChanged(prev, current) {
if (is.nullOrUndefined(prev) || is.nullOrUndefined(current)) {
return true;
}
// todo: performance enhance ?
return JSON.stringify(prev) !== JSON.stringify(current);
}
_isNeedUpdateTopicRouteInfo(topic) {
// producer
for (const groupName of this._producerTable.keys()) {
const producer = this._producerTable.get(groupName);
if (producer && producer.isPublishTopicNeedUpdate(topic)) {
return true;
}
}
// consumer
for (const groupName of this._consumerTable.keys()) {
const consumer = this._consumerTable.get(groupName);
if (consumer && consumer.isSubscribeTopicNeedUpdate(topic)) {
return true;
}
}
return false;
}
_topicRouteData2TopicPublishInfo(topic, topicRouteData) {
const info = new TopicPublishInfo();
// 顺序消息
if (topicRouteData.orderTopicConf && topicRouteData.orderTopicConf.length) {
const brokers = topicRouteData.orderTopicConf.split(';');
for (const broker of brokers) {
const item = broker.split(':');
const nums = parseInt(item[1], 10);
for (let i = 0; i < nums; i++) {
info.messageQueueList.push(new MessageQueue(topic, item[0], i));
}
}
info.orderTopic = true;
} else { // 非顺序消息
for (const queueData of topicRouteData.queueDatas) {
if (PermName.isWriteable(queueData.perm)) {
const brokerData = topicRouteData.brokerDatas.find(data => {
return data.brokerName === queueData.brokerName;
});
if (!brokerData || !brokerData.brokerAddrs[MixAll.MASTER_ID]) {
continue;
}
for (let i = 0, nums = queueData.writeQueueNums; i < nums; i++) {
info.messageQueueList.push(new MessageQueue(topic, queueData.brokerName, i));
}
}
}
info.orderTopic = false;
}
return info;
}
_topicRouteData2TopicSubscribeInfo(topic, topicRouteData) {
const messageQueueList = [];
for (const queueData of topicRouteData.queueDatas) {
if (PermName.isReadable(queueData.perm)) {
for (let i = 0, nums = queueData.readQueueNums; i < nums; i++) {
messageQueueList.push(new MessageQueue(topic, queueData.brokerName, i));
}
}
}
return messageQueueList;
}
/**
* send heartbeat to all brokers
* @return {void}
*/
async sendHeartbeatToAllBroker() {
this._cleanOfflineBroker();
const heartbeatData = this._prepareHeartbeatData();
const consumerEmpty = heartbeatData.consumerDataSet.length === 0;
const producerEmpty = heartbeatData.producerDataSet.length === 0;
if (consumerEmpty && producerEmpty) {
this.logger.info('[mq:client] sending hearbeat, but no consumer and no producer');
return;
}
const brokers = [];
for (const brokerName of this._brokerAddrTable.keys()) {
const oneTable = this._brokerAddrTable.get(brokerName);
if (!oneTable) {
continue;
}
for (const id in oneTable) {
const addr = oneTable[id];
if (!addr) {
continue;
}
// 说明只有Producer，则不向Slave发心跳
if (consumerEmpty && Number(id) !== MixAll.MASTER_ID) {
continue;
}
brokers.push(addr);
}
}
const ret = await gather(brokers.map(addr => () => this.sendHearbeat(addr, heartbeatData, 3000)));
this.logger.info('[mq:client] send heartbeat: %j to : %j, and result: %j', heartbeatData, brokers, ret);
}
_prepareHeartbeatData() {
const heartbeatData = {
clientID: this.clientId,
consumerDataSet: [],
producerDataSet: [],
};
// Consumer
for (const groupName of this._consumerTable.keys()) {
const consumer = this._consumerTable.get(groupName);
if (consumer) {
const subscriptionDataSet = [];
for (const topic of consumer.subscriptions.keys()) {
const data = consumer.subscriptions.get(topic);
if (data && data.subscriptionData) {
subscriptionDataSet.push(data.subscriptionData);
}
}
heartbeatData.consumerDataSet.push({
groupName: consumer.consumerGroup,
consumeType: consumer.consumeType,
messageModel: consumer.messageModel,
consumeFromWhere: consumer.consumeFromWhere,
subscriptionDataSet,
unitMode: consumer.unitMode,
});
}
}
// Producer
for (const groupName of this._producerTable.keys()) {
const producer = this._producerTable.get(groupName);
if (producer) {
heartbeatData.producerDataSet.push({
groupName,
});
}
}
return heartbeatData;
}
_cleanOfflineBroker() {
for (const brokerName of this._brokerAddrTable.keys()) {
const oneTable = this._brokerAddrTable.get(brokerName);
let exists = false;
for (const brokerId in oneTable) {
const addr = oneTable[brokerId];
if (this._isBrokerAddrExistInTopicRouteTable(addr)) {
exists = true;
} else {
delete oneTable[brokerId];
this.logger.info('[mq:client] the broker addr[%s %s] is offline, remove it', brokerName, addr);
}
}
if (!exists) {
this._brokerAddrTable.delete(brokerName);
this.logger.info('[mq:client] the broker[%s] name\'s host is offline, remove it', brokerName);
}
}
}
_isBrokerAddrExistInTopicRouteTable(addr) {
for (const topic of this._topicRouteTable.keys()) {
const topicRouteData = this._topicRouteTable.get(topic);
for (const brokerData of topicRouteData.brokerDatas) {
for (const brokerId in brokerData.brokerAddrs) {
if (brokerData.brokerAddrs[brokerId] === addr) {
return true;
}
}
}
}
return false;
}
/**
* rebalance
* @return {void}
*/
async doRebalance() {
for (const groupName of this._consumerTable.keys()) {
const consumer = this._consumerTable.get(groupName);
if (consumer) {
await consumer.doRebalance();
}
}
}
/**
* get all consumer list of topic
* @param {String} topic - topic
* @param {String} group - consumer group
* @return {Array} consumer list
*/
async findConsumerIdList(topic, group) {
let brokerAddr = this.findBrokerAddrByTopic(topic);
if (!brokerAddr) {
await this.updateTopicRouteInfoFromNameServer(topic);
brokerAddr = this.findBrokerAddrByTopic(topic);
if (!brokerAddr) {
throw new Error(`The broker of topic[${topic}] not exist`);
} else {
return await this.getConsumerIdListByGroup(brokerAddr, group, 3000);
}
} else {
return await this.getConsumerIdListByGroup(brokerAddr, group, 3000);
}
}
// get broker address by topic
findBrokerAddrByTopic(topic) {
const topicRouteData = this._topicRouteTable.get(topic);
if (topicRouteData) {
const brokerDatas = topicRouteData.brokerDatas || [];
const broker = brokerDatas[0];
if (broker) {
// master first, not found try slave
const addr = broker.brokerAddrs[MixAll.MASTER_ID];
if (!addr) {
for (const id in broker.brokerAddrs) {
return broker.brokerAddrs[id];
}
}
return addr;
}
}
return null;
}
/**
* find broker address
* @param {String} brokerName - broker name
* @return {Object} broker info
*/
findBrokerAddressInAdmin(brokerName) {
const map = this._brokerAddrTable.get(brokerName);
if (!map) {
return null;
}
if (map[MixAll.MASTER_ID]) {
return {
brokerAddr: map[MixAll.MASTER_ID],
slave: false,
};
}
for (const id in map) {
if (map[id]) {
return {
brokerAddr: map[id],
slave: true,
};
}
}
return null;
}
findBrokerAddressInSubscribe(brokerName, brokerId, onlyThisBroker) {
let brokerAddr = null;
let slave = false;
let found = false;
const map = this._brokerAddrTable.get(brokerName);
if (map) {
brokerAddr = map[brokerId];
slave = Number(brokerId) !== MixAll.MASTER_ID;
found = !is.nullOrUndefined(brokerAddr);
// 尝试寻找其他Broker
if (!found && !onlyThisBroker) {
for (const id in map) {
brokerAddr = map[id];
slave = Number(id) !== MixAll.MASTER_ID;
found = true;
break;
}
}
}
if (found) {
return {
brokerAddr,
slave,
};
}
return null;
}
/**
* get current offset of message queue
* @param {MessageQueue} messageQueue - message queue
* @return {Number} offset
*/
async maxOffset(messageQueue) {
let brokerAddr = this.findBrokerAddressInPublish(messageQueue.brokerName);
if (!brokerAddr) {
await this.updateTopicRouteInfoFromNameServer(messageQueue.topic);
brokerAddr = this.findBrokerAddressInPublish(messageQueue.brokerName);
if (!brokerAddr) {
throw new Error(`The broker[${messageQueue.brokerName}] not exist`);
} else {
return await this.getMaxOffset(brokerAddr, messageQueue.topic, messageQueue.queueId, 3000);
}
} else {
return await this.getMaxOffset(brokerAddr, messageQueue.topic, messageQueue.queueId, 3000);
}
}
// should be master
findBrokerAddressInPublish(brokerName) {
const map = this._brokerAddrTable.get(brokerName);
return map && map[MixAll.MASTER_ID];
}
async persistAllConsumerOffset() {
for (const groupName of this._consumerTable.keys()) {
const consumer = this._consumerTable.get(groupName);
if (consumer) {
await consumer.persistConsumerOffset();
}
}
}
async searchOffset(messageQueue, timestamp) {
let brokerAddr = this.findBrokerAddressInPublish(messageQueue.brokerName);
if (!brokerAddr) {
await this.updateTopicRouteInfoFromNameServer(messageQueue.topic);
brokerAddr = this.findBrokerAddressInPublish(messageQueue.brokerName);
if (!brokerAddr) {
throw new Error('The broker[' + messageQueue.brokerName + '] not exist');
} else {
return await super.searchOffset(brokerAddr, messageQueue.topic, messageQueue.queueId, timestamp, 3000);
}
} else {
return await super.searchOffset(brokerAddr, messageQueue.topic, messageQueue.queueId, timestamp, 3000);
}
}
static getAndCreateMQClient(clientConfig) {
const clientId = clientConfig.clientId;
let instance = instanceTable.get(clientId);
if (!instance) {
instance = new MQClient(clientConfig);
instanceTable.set(clientId, instance);
instance.once('close', () => {
instanceTable.delete(clientId);
});
}
return instance;
}
}
module.exports = MQClient;
================================================
FILE: lib/mq_client_api.js
================================================
'use strict';
const JSON2 = require('JSON2');
const bytes = require('bytes');
const fmt = require('util').format;
const ByteBuffer = require('byte');
const MessageQueue = require('./message_queue');
const RemotingClient = require('./remoting_client');
const PullStatus = require('./consumer/pull_status');
const SendStatus = require('./producer/send_status');
const RequestCode = require('./protocol/request_code');
const ResponseCode = require('./protocol/response_code');
const MessageConst = require('./message/message_const');
const MessageDecoder = require('./message/message_decoder');
const RemotingCommand = require('./protocol/command/remoting_command');
// const localIp = require('address').ip();
// const NAMESPACE_ORDER_TOPIC_CONFIG = 'ORDER_TOPIC_CONFIG';
const NAMESPACE_PROJECT_CONFIG = 'PROJECT_CONFIG';
const VIRTUAL_APPGROUP_PREFIX = '%%PROJECT_%s%%';
const byteBuffer = ByteBuffer.allocate(bytes('1m'));
/* eslint no-fallthrough:0 */
class MQClientAPI extends RemotingClient {
/**
* Metaq api wrapper
* @param {Object} options
* - {HttpClient} httpclient - http request client
* - {Object} [logger] - log module
* - {Number} [responseTimeout] - tcp response timeout
* @class
*/
constructor(options) {
super(options);
// virtual env project group
this.projectGroupPrefix = null;
}
/**
* start the client
* @return {void}
*/
async init() {
await super.init();
// this.projectGroupPrefix = await this.getProjectGroupByIp(localIp, 3000);
}
/**
* get project group prefix by ip address
* @param {String} ip - ip address
* @param {Number} [timeoutMillis] - timeout in milliseconds
* @return {String} prefix
*/
async getProjectGroupByIp(ip, timeoutMillis) {
try {
return await this.getKVConfigByValue(NAMESPACE_PROJECT_CONFIG, ip, timeoutMillis);
} catch (err) {
err.message = `[mq:api] Can not get project config from server, ${err.message}`;
this.logger.error(err);
return null;
}
}
/**
* get key-value config
* @param {String} namespace - config namespace
* @param {String} value - config key
* @param {Number} [timeoutMillis] - timeout in milliseconds
* @return {String} config value
*/
async getKVConfigByValue(namespace, value, timeoutMillis) {
const requestHeader = {
namespace,
key: value,
};
const request = RemotingCommand.createRequestCommand(RequestCode.GET_KV_CONFIG_BY_VALUE, requestHeader);
const response = await this.invokeForNameSrvAtLeastOnce(request, timeoutMillis);
switch (response.code) {
case ResponseCode.SUCCESS:
{
const responseHeader = response.decodeCommandCustomHeader();
return responseHeader && responseHeader.value;
}
default:
this._defaultHandler(request, response);
break;
}
}
/**
* get route info of topic
* @param {String} topic - topic
* @param {Number} [timeoutMillis] - timeout in milliseconds
* @return {Object} router info
*/
async getDefaultTopicRouteInfoFromNameServer(topic, timeoutMillis) {
const requestHeader = {
topic,
};
const request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINTO_BY_TOPIC, requestHeader);
const response = await this.invokeForNameSrvAtLeastOnce(request, timeoutMillis);
switch (response.code) {
case ResponseCode.SUCCESS:
{
const body = response.body;
if (body) {
this.logger.info('[mq:client_api] get Topic [%s] RouteInfoFromNameServer: %s', topic, body.toString());
// JSON.parse dose not work here
const routerInfoData = JSON2.parse(body.toString());
// sort
routerInfoData.queueDatas.sort(compare);
routerInfoData.brokerDatas.sort(compare);
return routerInfoData;
}
break;
}
case ResponseCode.TOPIC_NOT_EXIST:
this.logger.info('[mq:client_api] get Topic [%s] RouteInfoFromNameServer is not exist value', topic);
default:
this._defaultHandler(request, response);
break;
}
}
/**
* notify broker that client is offline
* @param {String} addr - brokder address
* @param {String} clientId - clientId
* @param {String} producerGroup - producer group name
* @param {String} consumerGroup - consumer group name
* @param {Number} [timeoutMillis] - timeout in milliseconds
* @return {void}
*/
async unregisterClient(addr, clientId, producerGroup, consumerGroup, timeoutMillis) {
producerGroup = this._buildWithProjectGroup(producerGroup);
consumerGroup = this._buildWithProjectGroup(consumerGroup);
const requestHeader = {
clientID: clientId,
producerGroup,
consumerGroup,
};
const request = RemotingCommand.createRequestCommand(RequestCode.UNREGISTER_CLIENT, requestHeader);
const response = await this.invoke(addr, request, timeoutMillis);
switch (response.code) {
case ResponseCode.SUCCESS:
break;
default:
this._defaultHandler(request, response);
break;
}
}
/**
* get route info from name server
* @param {String} topic - topic
* @param {Number} [timeoutMillis] - timeout in milliseconds
* @return {Object} route info
*/
async getTopicRouteInfoFromNameServer(topic, timeoutMillis) {
topic = this._buildWithProjectGroup(topic);
const request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINTO_BY_TOPIC, {
topic,
});
const response = await this.invokeForNameSrvAtLeastOnce(request, timeoutMillis);
switch (response.code) {
case ResponseCode.SUCCESS:
{
const body = response.body;
if (body) {
const routerInfoData = JSON2.parse(body.toString());
// sort
routerInfoData.queueDatas.sort(compare);
routerInfoData.brokerDatas.sort(compare);
return routerInfoData;
}
break;
}
case ResponseCode.TOPIC_NOT_EXIST:
this.logger.warn('[mq:client_api] get Topic [%s] RouteInfoFromNameServer is not exist value', topic);
default:
this._defaultHandler(request, response);
break;
}
}
/**
* send heartbeat
* @param {String} addr - broker address
* @param {Object} heartbeatData - heartbeat data
* @param {Number} [timeout] - timeout in milliseconds
* @return {void}
*/
async sendHearbeat(addr, heartbeatData, timeout) {
if (this.projectGroupPrefix) {
for (const consumerData of heartbeatData.consumerDataSet) {
consumerData.groupName = this._buildWithProjectGroup(consumerData.groupName);
for (const subscriptionData of consumerData.subscriptionDataSet) {
subscriptionData.topic = this._buildWithProjectGroup(subscriptionData.topic);
}
}
for (const producerData of heartbeatData.producerDataSet) {
producerData.groupName = this._buildWithProjectGroup(producerData.groupName);
}
}
const request = RemotingCommand.createRequestCommand(RequestCode.HEART_BEAT, null);
request.body = Buffer.from(JSON.stringify(heartbeatData));
const response = await this.invoke(addr, request, timeout);
if (response.code !== ResponseCode.SUCCESS) {
this._defaultHandler(request, response);
}
}
/**
* update consumer offset
* @param {String} brokerAddr - broker address
* @param {Object} requestHeader - request header
* @return {void}
*/
async updateConsumerOffsetOneway(brokerAddr, requestHeader) {
requestHeader.consumerGroup = this._buildWithProjectGroup(requestHeader.consumerGroup);
requestHeader.topic = this._buildWithProjectGroup(requestHeader.topic);
const request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_CONSUMER_OFFSET, requestHeader);
await this.invokeOneway(brokerAddr, request);
}
/**
* query consume offset
* @param {String} brokerAddr - broker address
* @param {Object} requestHeader - request header
* @param {Number} [timeoutMillis] - timeout in milliseconds
* @return {Number} offset
*/
async queryConsumerOffset(brokerAddr, requestHeader, timeoutMillis) {
requestHeader.consumerGroup = this._buildWithProjectGroup(requestHeader.consumerGroup);
requestHeader.topic = this._buildWithProjectGroup(requestHeader.topic);
const request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUMER_OFFSET, requestHeader);
const response = await this.invoke(brokerAddr, request, timeoutMillis);
switch (response.code) {
case ResponseCode.SUCCESS:
{
const responseHeader = response.decodeCommandCustomHeader();
return Number(responseHeader.offset.toString());
}
default:
this._defaultHandler(request, response);
break;
}
}
/**
* get current max offset of queue
* @param {String} addr - broker address
* @param {String} topic - topic
* @param {Number} queueId - queue id
* @param {Number} [timeoutMillis] - timeout in milliseconds
* @return {Number} offset
*/
async getMaxOffset(addr, topic, queueId, timeoutMillis) {
topic = this._buildWithProjectGroup(topic);
const requestHeader = {
topic,
queueId,
};
const request = RemotingCommand.createRequestCommand(RequestCode.GET_MAX_OFFSET, requestHeader);
const response = await this.invoke(addr, request, timeoutMillis);
switch (response.code) {
case ResponseCode.SUCCESS:
{
const responseHeader = response.decodeCommandCustomHeader();
// todo:
return responseHeader && Number(responseHeader.offset);
}
default:
this._defaultHandler(request, response);
break;
}
}
/**
* search consume offset by timestamp
* @param {String} addr - broker address
* @param {String} topic - topic
* @param {Number} queueId - queue id
* @param {String} timestamp - timestamp used to query
* @param {Number} [timeoutMillis] - timeout in milliseconds
* @return {Number} offset
*/
async searchOffset(addr, topic, queueId, timestamp, timeoutMillis) {
topic = this._buildWithProjectGroup(topic);
const requestHeader = {
topic,
queueId,
timestamp,
};
const request = RemotingCommand.createRequestCommand(RequestCode.SEARCH_OFFSET_BY_TIMESTAMP, requestHeader);
const response = await this.invoke(addr, request, timeoutMillis);
switch (response.code) {
case ResponseCode.SUCCESS:
{
const responseHeader = response.decodeCommandCustomHeader();
// todo:
return responseHeader && Number(responseHeader.offset);
}
default:
this._defaultHandler(request, response);
break;
}
}
/**
* get all consumer's id in same group
* @param {String} addr - broker address
* @param {String} consumerGroup - consumer group
* @param {Number} [timeoutMillis] - timeout in milliseconds
* @return {Array} consumer list
*/
async getConsumerIdListByGroup(addr, consumerGroup, timeoutMillis) {
consumerGroup = this._buildWithProjectGroup(consumerGroup);
const requestHeader = {
consumerGroup,
};
const request = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_LIST_BY_GROUP, requestHeader);
const response = await this.invoke(addr, request, timeoutMillis);
switch (response.code) {
case ResponseCode.SUCCESS:
if (response.body) {
const body = JSON2.parse(response.body.toString());
return body.consumerIdList;
}
break;
default:
this._defaultHandler(request, response);
break;
}
}
/**
* pull message from broker
* @param {String} brokerAddr - broker address
* @param {Object} requestHeader - request header
* @param {Number} [timeoutMillis] - timeout in milliseconds
* @return {Object} pull result
*/
async pullMessage(brokerAddr, requestHeader, timeoutMillis) {
requestHeader.consumerGroup = this._buildWithProjectGroup(requestHeader.consumerGroup);
requestHeader.topic = this._buildWithProjectGroup(requestHeader.topic);
const request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, requestHeader);
const response = await this.invoke(brokerAddr, request, timeoutMillis);
let pullStatus = PullStatus.NO_NEW_MSG;
switch (response.code) {
case ResponseCode.SUCCESS:
pullStatus = PullStatus.FOUND;
break;
case ResponseCode.PULL_NOT_FOUND:
pullStatus = PullStatus.NO_NEW_MSG;
break;
case ResponseCode.PULL_RETRY_IMMEDIATELY:
pullStatus = PullStatus.NO_MATCHED_MSG;
break;
case ResponseCode.PULL_OFFSET_MOVED:
pullStatus = PullStatus.OFFSET_ILLEGAL;
break;
default:
this._defaultHandler(request, response);
break;
}
const responseHeader = response.decodeCommandCustomHeader();
let msgList = [];
if (pullStatus === PullStatus.FOUND) {
byteBuffer.reset();
byteBuffer.put(response.body).flip();
msgList = MessageDecoder.decodes(byteBuffer);
for (const msg of msgList) {
msg.topic = this._clearProjectGroup(msg.topic);
msg.properties[MessageConst.PROPERTY_MIN_OFFSET] = responseHeader.minOffset.toString();
msg.properties[MessageConst.PROPERTY_MAX_OFFSET] = responseHeader.maxOffset.toString();
}
}
return {
pullStatus,
nextBeginOffset: Number(responseHeader.nextBeginOffset),
minOffset: Number(responseHeader.minOffset),
maxOffset: Number(responseHeader.maxOffset),
msgFoundList: msgList,
suggestWhichBrokerId: responseHeader.suggestWhichBrokerId,
};
}
/**
* create topic
* @param {String} addr - broker address
* @param {String} defaultTopic - default topic: TBW102
* @param {Object} topicConfig - new topic config
* @param {Number} [timeoutMillis] - timeout in milliseconds
* @return {void}
*/
async createTopic(addr, defaultTopic, topicConfig, timeoutMillis) {
const topicWithProjectGroup = this._buildWithProjectGroup(topicConfig.topicName);
const requestHeader = {
topic: topicWithProjectGroup,
defaultTopic,
readQueueNums: topicConfig.readQueueNums,
writeQueueNums: topicConfig.writeQueueNums,
perm: topicConfig.perm,
topicFilterType: topicConfig.topicFilterType,
topicSysFlag: topicConfig.topicSysFlag,
order: topicConfig.order,
};
const request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_TOPIC, requestHeader);
const response = await this.invoke(addr, request, timeoutMillis);
switch (response.code) {
case ResponseCode.SUCCESS:
return;
default:
this._defaultHandler(request, response);
break;
}
}
/**
* send message
* @param {String} brokerAddr - broker address
* @param {String} brokerName - broker name
* @param {Message} msg - msg object
* @param {Object} requestHeader - request header
* @param {Number} [timeoutMillis] - timeout in milliseconds
* @return {Object} sendResult
*/
async sendMessage(brokerAddr, brokerName, msg, requestHeader, timeoutMillis) {
msg.topic = this._buildWithProjectGroup(msg.topic);
requestHeader.producerGroup = this._buildWithProjectGroup(requestHeader.producerGroup);
requestHeader.topic = this._buildWithProjectGroup(requestHeader.topic);
const requestHeaderV2 = {
a: requestHeader.producerGroup,
b: requestHeader.topic,
c: requestHeader.defaultTopic,
d: requestHeader.defaultTopicQueueNums,
e: requestHeader.queueId,
f: requestHeader.sysFlag,
g: requestHeader.bornTimestamp,
h: requestHeader.flag,
i: requestHeader.properties,
j: requestHeader.reconsumeTimes,
k: requestHeader.unitMode,
l: requestHeader.maxReconsumeTimes,
};
const request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, requestHeaderV2);
request.body = msg.body;
const response = await this.invoke(brokerAddr, request, timeoutMillis);
let sendStatus = SendStatus.SEND_OK;
switch (response.code) {
case ResponseCode.FLUSH_DISK_TIMEOUT:
sendStatus = SendStatus.FLUSH_DISK_TIMEOUT;
break;
case ResponseCode.FLUSH_SLAVE_TIMEOUT:
sendStatus = SendStatus.FLUSH_SLAVE_TIMEOUT;
break;
case ResponseCode.SLAVE_NOT_AVAILABLE:
sendStatus = SendStatus.SLAVE_NOT_AVAILABLE;
break;
case ResponseCode.SUCCESS:
sendStatus = SendStatus.SEND_OK;
break;
default:
this._defaultHandler(request, response);
break;
}
const responseHeader = response.decodeCommandCustomHeader();
const messageQueue = new MessageQueue(msg.topic, brokerName, responseHeader.queueId);
messageQueue.topic = this._clearProjectGroup(messageQueue.topic);
return {
sendStatus,
msgId: responseHeader.msgId,
messageQueue,
queueOffset: Number(responseHeader.queueOffset),
};
}
/**
* consumer send message back
* @param {String} brokerAddr - broker address
* @param {Message} msg - message object
* @param {String} consumerGroup - consumer group
* @param {Number} delayLevel - delay level
* @param {Number} timeoutMillis - timeout in millis
* @param {Number} maxConsumeRetryTimes - max retry times
*/
async consumerSendMessageBack(brokerAddr, msg, consumerGroup, delayLevel, timeoutMillis, maxConsumeRetryTimes) {
const requestHeader = {
offset: msg.commitLogOffset,
group: consumerGroup,
delayLevel,
originMsgId: msg.msgId,
originTopic: msg.topic,
unitMode: false,
maxReconsumeTimes: maxConsumeRetryTimes,
};
const request = RemotingCommand.createRequestCommand(RequestCode.CONSUMER_SEND_MSG_BACK, requestHeader);
const response = await this.invoke(brokerAddr, request, timeoutMillis);
switch (response.code) {
case ResponseCode.SUCCESS:
return;
default:
this._defaultHandler(request, response);
break;
}
}
// * viewMessage(brokerAddr, phyoffset, timeoutMillis) {
// const requestHeader = {
// offset: phyoffset,
// };
// const request = RemotingCommand.createRequestCommand(RequestCode.VIEW_MESSAGE_BY_ID, requestHeader);
// const response = await this.invoke(brokerAddr, request, timeoutMillis);
// switch (response.code) {
// case ResponseCode.SUCCESS:
// {
// const byteBuffer = ByteBuffer.wrap(response.body);
// const messageExt = MessageDecoder.decode(byteBuffer);
// // 清除虚拟运行环境相关的projectGroupPrefix
// if (this.projectGroupPrefix) {
// messageExt.topic = this._clearProjectGroup(messageExt.topic, this.projectGroupPrefix);
// }
// return messageExt;
// }
// default:
// this._defaultHandler(request, response);
// break;
// }
// }
// default handler
_defaultHandler(request, response) {
const err = new Error(response.remark);
err.name = 'MQClientException';
err.code = response.code;
throw err;
}
_buildWithProjectGroup(origin) {
if (this.projectGroupPrefix) {
const prefix = fmt(VIRTUAL_APPGROUP_PREFIX, this.projectGroupPrefix);
if (!origin.endsWith(prefix)) {
return origin + prefix;
}
return origin;
}
return origin;
}
_clearProjectGroup(origin) {
const prefix = fmt(VIRTUAL_APPGROUP_PREFIX, this.projectGroupPrefix);
if (prefix && origin.endsWith(prefix)) {
return origin.slice(0, origin.lastIndexOf(prefix));
}
return origin;
}
}
module.exports = MQClientAPI;
// Helper
// ---------------
function compare(routerA, routerB) {
if (routerA.brokerName > routerB.brokerName) {
return 1;
} else if (routerA.brokerName < routerB.brokerName) {
return -1;
}
return 0;
}
================================================
FILE: lib/process_queue.js
================================================
'use strict';
const Base = require('sdk-base');
const bsInsert = require('binary-search-insert');
const comparator = (a, b) => a.queueOffset - b.queueOffset;
const pullMaxIdleTime = 120000;
class ProcessQueue extends Base {
constructor(options = {}) {
super(options);
this.msgList = [];
this.droped = false;
this.lastPullTimestamp = Date.now();
this.lastConsumeTimestamp = Date.now();
this.locked = false;
this.lastLockTimestamp = Date.now();
}
get maxSpan() {
const msgCount = this.msgCount;
if (msgCount) {
return this.msgList[msgCount - 1].queueOffset - this.msgList[0].queueOffset;
}
return 0;
}
get msgCount() {
return this.msgList.length;
}
get isPullExpired() {
return Date.now() - this.lastPullTimestamp > pullMaxIdleTime;
}
putMessage(msgs) {
for (const msg of msgs) {
bsInsert(this.msgList, comparator, msg);
}
this.queueOffsetMax = this.msgList[this.msgCount - 1].queueOffset;
}
remove(count = 1) {
this.msgList.splice(0, count);
return this.msgCount ? this.msgList[0].queueOffset : this.queueOffsetMax + 1;
}
clear() {
this.msgList = [];
}
}
module.exports = ProcessQueue;
================================================
FILE: lib/producer/mq_producer.js
================================================
'use strict';
/* eslint no-bitwise: 0 */
const utils = require('../utils');
const logger = require('../logger');
const MixAll = require('../mix_all');
const MQClient = require('../mq_client');
const SendStatus = require('./send_status');
const ClientConfig = require('../client_config');
// const PermName = require('../protocol/perm_name');
const MessageConst = require('../message/message_const');
const TopicPublishInfo = require('./topic_publish_info');
const ResponseCode = require('../protocol/response_code');
const MessageSysFlag = require('../utils/message_sys_flag');
const MessageDecoder = require('../message/message_decoder');
const defaultOptions = {
logger,
producerGroup: MixAll.DEFAULT_PRODUCER_GROUP,
createTopicKey: MixAll.DEFAULT_TOPIC,
defaultTopicQueueNums: 4,
sendMsgTimeout: 3000,
compressMsgBodyOverHowmuch: 1024 * 4,
retryTimesWhenSendFailed: 3,
retryAnotherBrokerWhenNotStoreOK: false,
maxMessageSize: 1024 * 128,
connectTimeout: 10000,
};
class MQProducer extends ClientConfig {
/**
* metaq message producer
* @param {Object} options -
* @class
*/
constructor(options) {
super(Object.assign({ initMethod: 'init' }, defaultOptions, options));
this._topicPublishInfoTable = new Map();
this._inited = false;
if (this.producerGroup !== MixAll.CLIENT_INNER_PRODUCER_GROUP) {
this.changeInstanceNameToPID();
}
this._mqClient = MQClient.getAndCreateMQClient(this);
this._mqClient.on('error', err => this._error(err));
}
get logger() {
return this.options.logger;
}
get producerGroup() {
if (this.namespace) {
return `${this.namespace}%${this.options.producerGroup}`;
}
return this.options.producerGroup;
}
get createTopicKey() {
return this.options.createTopicKey;
}
get defaultTopicQueueNums() {
return this.options.defaultTopicQueueNums;
}
/**
* topic list
* @property {Array} MQProducer#publishTopicList
*/
get publishTopicList() {
return this._topicPublishInfoTable.keys();
}
/**
* start the producer
* @return {void}
*/
async init() {
// this._topicPublishInfoTable.set(this.createTopicKey, new TopicPublishInfo());
await MQProducer.getDefaultProducer(this.options);
this._mqClient.registerProducer(this.producerGroup, this);
await this._mqClient.ready();
this.logger.info('[mq:producer] producer started');
this._inited = true;
}
/**
* close the producer
*/
async close() {
this._topicPublishInfoTable.clear();
await this._mqClient.unregisterProducer(this.producerGroup);
await this._mqClient.close();
this.removeAllListeners();
this._inited = false;
this.logger.info('[mq:producer] producer close');
}
/**
* default error handler
* @param {Error} err - error
* @return {void}
*/
_error(err) {
setImmediate(() => {
err.message = 'MQPushConsumer occurred an error' + err.message;
this.emit('error', err);
});
}
/**
* update topic info
* @param {String} topic - topic
* @param {Object} info - route info
* @return {void}
*/
updateTopicPublishInfo(topic, info) {
if (topic && info) {
const prev = this._topicPublishInfoTable.get(topic);
this._topicPublishInfoTable.set(topic, info);
if (prev) {
info.sendWhichQueue = prev.sendWhichQueue;
this.logger.warn('[mq:producer] updateTopicPublishInfo() prev is not null, %j', prev);
}
}
}
/**
* is publish topic info need update
* @param {String} topic - topic
* @return {Boolean} need update?
*/
isPublishTopicNeedUpdate(topic) {
const info = this._topicPublishInfoTable.get(topic);
return !info || !info.ok;
}
/**
* create topic
* @param {String} key - TBW102
* @param {String} newTopic -
* @param {Number} queueNum - queue number
* @param {Number} topicSysFlag - 0
* @return {void}
*/
// * createTopic(key, newTopic, queueNum, topicSysFlag) {
// await this.ready();
// topicSysFlag = topicSysFlag || 0;
// const topicRouteData = await this._mqClient.getTopicRouteInfoFromNameServer(key, 1000 * 3);
// const brokerDataList = topicRouteData.brokerDatas;
// if (brokerDataList && brokerDataList.length) {
// // 排序原因：即使没有配置顺序消息模式，默认队列的顺序同配置的一致。
// brokerDataList.sort(compare);
// await brokerDataList.map(brokerData => {
// const addr = brokerData.brokerAddrs[MixAll.MASTER_ID];
// if (addr) {
// const topicConfig = {
// topicName: newTopic,
// readQueueNums: queueNum,
// writeQueueNums: queueNum,
// topicSysFlag,
// perm: PermName.PERM_READ | PermName.PERM_WRITE,
// topicFilterType: 'SINGLE_TAG',
// order: false,
// };
// this.logger.info('[mq:producer] execute createTopic at broker: %s, topic: %s', addr, newTopic);
// return this._mqClient.createTopic(addr, key, topicConfig, 1000 * 3);
// }
// return null;
// });
// } else {
// throw new Error('Not found broker, maybe key is wrong');
// }
// }
/**
* send message
* @param {Message} msg - message object
* @return {Object} sendResult
*/
async send(msg) {
await this.ready();
msg.topic = this.formatTopic(msg.topic);
const topicPublishInfo = await this.tryToFindTopicPublishInfo(msg.topic);
if (!topicPublishInfo) {
throw new Error(`no publish router data for topic: ${msg.topic}`);
}
const maxTimeout = this.options.sendMsgTimeout + 1000;
const timesTotal = 1 + this.options.retryTimesWhenSendFailed;
const beginTimestamp = Date.now();
const brokersSent = [];
let times = 0;
let lastBrokerName;
let sendResult;
for (; times < timesTotal && Date.now() - beginTimestamp < maxTimeout; times++) {
const messageQueue = topicPublishInfo.selectOneMessageQueue(lastBrokerName, msg);
if (!messageQueue) {
continue;
}
lastBrokerName = messageQueue.brokerName;
brokersSent[times] = lastBrokerName;
try {
sendResult = await this.sendKernelImpl(msg, messageQueue);
if (sendResult.sendStatus === SendStatus.SEND_OK || !this.options.retryAnotherBrokerWhenNotStoreOK) {
break;
}
} catch (err) {
if (err.name === 'MQClientException') {
switch (err.code) {
case ResponseCode.TOPIC_NOT_EXIST:
case ResponseCode.SERVICE_NOT_AVAILABLE:
case ResponseCode.SYSTEM_ERROR:
case ResponseCode.NO_PERMISSION:
case ResponseCode.NO_BUYER_ID:
case ResponseCode.NOT_IN_CURRENT_UNIT:
break;
default:
throw err;
}
}
}
}
if (!sendResult) {
throw new Error(`Send [${times}] times, still failed, cost [${Date.now() - beginTimestamp}]ms, Topic: ${msg.topic}, BrokersSent: ${JSON.stringify(brokersSent)}`);
}
return sendResult;
}
/**
* try to find topic info from name server
* @param {String} topic - topic
* @return {Object} route info
*/
async tryToFindTopicPublishInfo(topic) {
let topicPublishInfo = this._topicPublishInfoTable.get(topic);
if (!topicPublishInfo || !topicPublishInfo.ok) {
this._topicPublishInfoTable.set(topic, new TopicPublishInfo());
await this._mqClient.updateTopicRouteInfoFromNameServer(topic);
topicPublishInfo = this._topicPublishInfoTable.get(topic);
}
if (topicPublishInfo.haveTopicRouterInfo || topicPublishInfo && topicPublishInfo.ok) {
return topicPublishInfo;
}
await this._mqClient.updateTopicRouteInfoFromNameServer(topic, true, this);
return this._topicPublishInfoTable.get(topic);
}
async sendKernelImpl(msg, messageQueue) {
let brokerAddr = this._mqClient.findBrokerAddressInPublish(messageQueue.brokerName);
if (!brokerAddr) {
await this.tryToFindTopicPublishInfo(messageQueue.topic);
brokerAddr = this._mqClient.findBrokerAddressInPublish(messageQueue.brokerName);
if (!brokerAddr) {
const err = new Error(`The broker[${messageQueue.brokerName}] not exist`);
err.name = 'MQClientException';
throw err;
}
}
let sysFlag = 0;
if (this.tryToCompressMessage(msg)) {
sysFlag |= MessageSysFlag.CompressedFlag;
}
const tranMsg = msg.properties[MessageConst.PROPERTY_TRANSACTION_PREPARED];
if (tranMsg) {
sysFlag |= MessageSysFlag.TransactionPreparedType;
}
const requestHeader = {
producerGroup: this.producerGroup,
topic: msg.topic,
defaultTopic: this.createTopicKey,
defaultTopicQueueNums: this.defaultTopicQueueNums,
queueId: messageQueue.queueId,
sysFlag,
bornTimestamp: Date.now(),
flag: msg.flag,
properties: MessageDecoder.messageProperties2String(msg.properties),
reconsumeTimes: 0,
unitMode: this.unitMode,
};
if (requestHeader.topic.indexOf(MixAll.RETRY_GROUP_TOPIC_PREFIX) === 0) {
const reconsumeTimes = msg.properties[MessageConst.PROPERTY_RECONSUME_TIME];
if (reconsumeTimes) {
requestHeader.reconsumeTimes = parseInt(reconsumeTimes, 10);
delete msg.properties[MessageConst.PROPERTY_RECONSUME_TIME];
}
const maxReconsumeTimes = msg.properties[MessageConst.PROPERTY_MAX_RECONSUME_TIMES];
if (maxReconsumeTimes) {
requestHeader.maxReconsumeTimes = parseInt(maxReconsumeTimes, 10);
delete msg.properties[MessageConst.PROPERTY_MAX_RECONSUME_TIMES];
}
}
return await this._mqClient.sendMessage(brokerAddr, messageQueue.brokerName, msg, requestHeader, this.options.sendMsgTimeout);
}
/**
* 尝试压缩消息
* @param {Message} msg - 消息对象
* @return {Boolean} 是否压缩
*/
tryToCompressMessage(msg) {
const body = msg.body;
if (body) {
if (body.length >= this.options.compressMsgBodyOverHowmuch) {
try {
const data = utils.compress(body);
if (data) {
msg.body = data;
return true;
}
} catch (err) {
err.name = 'MetaQCompressError';
this._error(err);
}
}
}
return false;
}
/**
* 获取默认 Producer 实例
* @param {Object} options 配置
* @return {MQProducer} 默认 Producer 实例
*/
static async getDefaultProducer(options = {}) {
if (!MQProducer.defaultProducer) {
const opts = Object.assign({}, options);
opts.producerGroup = MixAll.CLIENT_INNER_PRODUCER_GROUP;
MQProducer.defaultProducer = new MQProducer(opts);
await MQProducer.defaultProducer.ready();
return MQProducer.defaultProducer;
}
return MQProducer.defaultProducer;
}
}
module.exports = MQProducer;
// // Helper
// // ---------------
// function compare(routerA, routerB) {
// if (routerA.brokerName > routerB.brokerName) {
// return 1;
// } else if (routerA.brokerName < routerB.brokerName) {
// return -1;
// }
// return 0;
// }
================================================
FILE: lib/producer/send_status.js
================================================
'use strict';
module.exports = {
// 消息发送成功
SEND_OK: 'SEND_OK',
// 消息发送成功，但是服务器刷盘超时，消息已经进入服务器队列，只有此时服务器宕机，消息才会丢失
FLUSH_DISK_TIMEOUT: 'FLUSH_DISK_TIMEOUT',
// 消息发送成功，但是服务器同步到Slave时超时，消息已经进入服务器队列，只有此时服务器宕机，消息才会丢失
FLUSH_SLAVE_TIMEOUT: 'FLUSH_SLAVE_TIMEOUT',
// 消息发送成功，但是此时slave不可用，消息已经进入服务器队列，只有此时服务器宕机，消息才会丢失
SLAVE_NOT_AVAILABLE: 'SLAVE_NOT_AVAILABLE',
};
================================================
FILE: lib/producer/topic_publish_info.js
================================================
'use strict';
const crypto = require('crypto');
class TopicPublishInfo {
constructor() {
this.orderTopic = false;
this.haveTopicRouterInfo = false;
this.messageQueueList = [];
this.sendWhichQueue = Math.floor(Math.random() * Math.floor(Math.pow(2, 31))); // 打散 queue 选择起点
}
/**
* 是否可用
*/
get ok() {
return this.messageQueueList && this.messageQueueList.length;
}
/**
* 如果lastBrokerName不为null，则寻找与其不同的MessageQueue
* @param {String} lastBrokerName - last broker name
* @param {Object} message - message
* @return {Object} queue
*/
selectOneMessageQueue(lastBrokerName, message) {
let index,
pos,
mq;
const shardingKey = message && message.shardingKey;
let queueId = '';
if (shardingKey) {
// 这里不应该是所有可用队列数字，应该是queue总数
// 但这里的 messageQueueList 是所有可写队列， golang和java客户端均为在queue总对恶劣中寻找位置
// 通过寻找所有可写queueId总数计算应该写入的queueId
const queueIds = [];
this.messageQueueList.forEach(queue => {
if (!queueIds.includes(queue.queueId)) queueIds.push(queue.queueId);
});
// 参考golang和java的sdk，没有规范确定的shardingKey找queueId的实现标准
// 若引入其他hash算法就显得比较笨重, 可以使用 nodejs 原生hash方法生成可计算的hash值
// Reference: https://github.com/apache/rocketmq/blob/c11ed78eeb73c23da4cf9a36a6bad493a1279210/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java#L382
// Reference: https://github.com/apache/rocketmq-client-go/blob/896a8a3453be4bcfcb5bfb5bb9139c4df6d4cdab/producer/selector.go#L112
const queueIndex = crypto
.createHash('md5')
.update(shardingKey)
.digest()
.readUInt32BE() % queueIds.length;
queueId = queueIds[queueIndex] || '';
}
if (!Number.isSafeInteger(this.sendWhichQueue)) {
this.sendWhichQueue = 0; // 超出安全范围，重置为 0
}
index = this.sendWhichQueue++;
for (let i = 0, len = this.messageQueueList.length; i < len; i++) {
pos = Math.abs(index++) % len;
mq = this.messageQueueList[pos];
if (lastBrokerName && mq.brokerName === lastBrokerName) {
continue;
}
if (queueId && mq.queueId !== queueId) {
continue;
}
return mq;
}
return null;
}
}
module.exports = TopicPublishInfo;
================================================
FILE: lib/protocol/command/opaque_generator.js
================================================
'use strict';
const MAX_INT_31 = Math.pow(2, 31) - 10;
let opaque = 0;
exports.getNextOpaque = () => {
if (opaque >= MAX_INT_31) {
this.resetOpaque();
}
return ++opaque;
};
exports.resetOpaque = () => { opaque = 0; };
exports.getCurrentOpaque = () => opaque;
================================================
FILE: lib/protocol/command/remoting_command.js
================================================
'use strict';
/* eslint no-bitwise: 0*/
const bytes = require('bytes');
const is = require('is-type-of');
const ByteBuffer = require('byte');
const OpaqueGenerator = require('./opaque_generator');
const REQUEST_COMMAND = 'REQUEST_COMMAND';
const RESPONSE_COMMAND = 'RESPONSE_COMMAND';
// 0, REQUEST_COMMAND
// 1, RESPONSE_COMMAND
const RPC_TYPE = 0;
// 0, RPC
// 1, Oneway
const RPC_ONEWAY = 1;
// V3_4_9 支持设置重试的 maxReconsumeTimes
const MQ_VERSION = 121;
const buf = new ByteBuffer({ size: bytes('1m') });
class RemotingCommand {
/**
* metaq 命令
* @param {Object} options
* - code {Number} 命令代号
* - customHeader {Object} 自定义头部
* - flag {Number} 标识命令的属性
* - opaque {Number} 关联请求、响应的字段
* - remark {String}
* - extFields {Object} 扩展字段
*/
constructor(options) {
this.code = options.code;
this.customHeader = options.customHeader;
this.flag = options.flag || 0;
this.opaque = options.opaque || OpaqueGenerator.getNextOpaque();
this.remark = options.remark;
this.extFields = options.extFields || {};
this.body = null;
}
get language() {
return 'JAVA';
}
get version() {
return MQ_VERSION;
}
get type() {
return this.isResponseType ? RESPONSE_COMMAND : REQUEST_COMMAND;
}
get isResponseType() {
const bits = 1 << RPC_TYPE;
return (this.flag & bits) === bits;
}
get isOnewayRPC() {
const bits = 1 << RPC_ONEWAY;
return (this.flag & bits) === bits;
}
markResponseType() {
const bits = 1 << RPC_TYPE;
this.flag |= bits;
}
markOnewayRPC() {
const bits = 1 << RPC_ONEWAY;
this.flag |= bits;
}
makeCustomHeaderToNet() {
const customHeader = this.customHeader;
if (!customHeader) {
return;
}
this.extFields = this.extFields || {};
for (const key in customHeader) {
const field = customHeader[key];
if (is.nullOrUndefined(field) || is.function(field)) {
continue;
}
this.extFields[key] = field;
}
}
decodeCommandCustomHeader() {
return JSON.parse(JSON.stringify(this.extFields)); // 深拷贝
}
buildHeader() {
this.makeCustomHeaderToNet();
return Buffer.from(JSON.stringify({
code: this.code,
language: this.language,
opaque: this.opaque,
flag: this.flag,
version: this.version,
remark: this.remark,
extFields: this.extFields,
}), 'utf8');
}
encode() {
buf.reset();
let length = 4;
const headerData = this.buildHeader();
length += headerData.length;
if (this.body) {
length += this.body.length;
}
buf.putInt(length);
buf.putInt(headerData.length);
buf.put(headerData);
if (this.body) {
buf.put(this.body);
}
return buf.copy();
}
/**
* 创建 request 命令
* @param {Number} code - 命令代号
* @param {Object} customHeader -
* @return {RemotingCommand} command
*/
static createRequestCommand(code, customHeader) {
return new RemotingCommand({
code,
customHeader,
});
}
/**
* 创建 request 命令
* @param {Number} code - 命令代号
* @param {Number} opaque -
* @param {String} remark -
* @return {RemotingCommand} command
*/
static createResponseCommand(code, opaque, remark) {
const command = new RemotingCommand({
code,
opaque,
remark: remark || 'not set any response code',
});
command.markResponseType();
return command;
}
/**
* 反序列化报文
* @param {Buffer} packet -
* @return {RemotingCommand} command
*/
static decode(packet) {
const customHeaderLength = packet.readInt32BE(4);
const customHeaderData = packet.slice(8, 8 + customHeaderLength);
let body;
if (packet.length - 8 - customHeaderLength > 0) {
body = packet.slice(8 + customHeaderLength);
}
const command = new RemotingCommand(JSON.parse(customHeaderData));
command.body = body;
return command;
}
}
module.exports = RemotingCommand;
================================================
FILE: lib/protocol/consume_type.js
================================================
'use strict';
module.exports = {
CONSUME_ACTIVELY: 'CONSUME_ACTIVELY',
CONSUME_PASSIVELY: 'CONSUME_PASSIVELY',
};
================================================
FILE: lib/protocol/message_model.js
================================================
'use strict';
module.exports = {
BROADCASTING: 'BROADCASTING',
CLUSTERING: 'CLUSTERING',
};
================================================
FILE: lib/protocol/perm_name.js
================================================
'use strict';
/* eslint no-bitwise: 0 */
// var PERM_PRIORITY = 0x1 << 3;
const PERM_READ = 0x1 << 2;
const PERM_WRITE = 0x1 << 1;
const PERM_INHERIT = 0x1 << 0;
exports.PERM_READ = PERM_READ;
exports.PERM_WRITE = PERM_WRITE;
const isReadable = exports.isReadable = function(perm) {
return (perm & PERM_READ) === PERM_READ;
};
const isWriteable = exports.isWriteable = function(perm) {
return (perm & PERM_WRITE) === PERM_WRITE;
};
const isInherited = exports.isInherited = function(perm) {
return (perm & PERM_INHERIT) === PERM_INHERIT;
};
exports.perm2String = function(perm) {
let str = '';
str += isReadable(perm) ? 'R' : '-';
str += isWriteable(perm) ? 'W' : '-';
str += isInherited(perm) ? 'X' : '-';
return str;
};
================================================
FILE: lib/protocol/request_code.js
================================================
'use strict';
// Broker 发送消息
exports.SEND_MESSAGE = 10;
// Broker 订阅消息
exports.PULL_MESSAGE = 11;
// Broker 查询消息
exports.QUERY_MESSAGE = 12;
// Broker 查询Broker Offset
exports.QUERY_BROKER_OFFSET = 13;
// Broker 查询Consumer Offset
exports.QUERY_CONSUMER_OFFSET = 14;
// Broker 更新Consumer Offset
exports.UPDATE_CONSUMER_OFFSET = 15;
// Broker 更新或者增加一个Topic
exports.UPDATE_AND_CREATE_TOPIC = 17;
// Broker 获取所有Topic的配置（Slave和Namesrv都会向Master请求此配置）
exports.GET_ALL_TOPIC_CONFIG = 21;
// Broker 获取所有Topic配置（Slave和Namesrv都会向Master请求此配置）
exports.GET_TOPIC_CONFIG_LIST = 22;
// Broker 获取所有Topic名称列表
exports.GET_TOPIC_NAME_LIST = 23;
// Broker 更新Broker上的配置
exports.UPDATE_BROKER_CONFIG = 25;
// Broker 获取Broker上的配置
exports.GET_BROKER_CONFIG = 26;
// Broker 触发Broker删除文件
exports.TRIGGER_DELETE_FILES = 27;
// Broker 获取Broker运行时信息
exports.GET_BROKER_RUNTIME_INFO = 28;
// Broker 根据时间查询队列的Offset
exports.SEARCH_OFFSET_BY_TIMESTAMP = 29;
// Broker 查询队列最大Offset
exports.GET_MAX_OFFSET = 30;
// Broker 查询队列最小Offset
exports.GET_MIN_OFFSET = 31;
// Broker 查询队列最早消息对应时间
exports.GET_EARLIEST_MSG_STORETIME = 32;
// Broker 根据消息ID来查询消息
exports.VIEW_MESSAGE_BY_ID = 33;
// Broker Client向Client发送心跳，并注册自身
exports.HEART_BEAT = 34;
// Broker Client注销
exports.UNREGISTER_CLIENT = 35;
// Broker Consumer将处理不了的消息发回服务器
exports.CONSUMER_SEND_MSG_BACK = 36;
// Broker Commit或者Rollback事务
exports.END_TRANSACTION = 37;
// Broker 获取ConsumerId列表通过GroupName
exports.GET_CONSUMER_LIST_BY_GROUP = 38;
// Broker 主动向Producer回查事务状态
exports.CHECK_TRANSACTION_STATE = 39;
// Broker Broker通知Consumer列表变化
exports.NOTIFY_CONSUMER_IDS_CHANGED = 40;
// Broker Consumer向Master锁定队列
exports.LOCK_BATCH_MQ = 41;
// Broker Consumer向Master解锁队列
exports.UNLOCK_BATCH_MQ = 42;
// Broker 获取所有Consumer Offset
exports.GET_ALL_CONSUMER_OFFSET = 43;
// Broker 获取所有定时进度
exports.GET_ALL_DELAY_OFFSET = 45;
// Namesrv 向Namesrv追加KV配置
exports.PUT_KV_CONFIG = 100;
// Namesrv 从Namesrv获取KV配置
exports.GET_KV_CONFIG = 101;
// Namesrv 从Namesrv获取KV配置
exports.DELETE_KV_CONFIG = 102;
// Namesrv 注册一个Broker，数据都是持久化的，如果存在则覆盖配置
exports.REGISTER_BROKER = 103;
// Namesrv 卸载一个Broker，数据都是持久化的
exports.UNREGISTER_BROKER = 104;
// Namesrv 根据Topic获取Broker Name、队列数(包含读队列与写队列)
exports.GET_ROUTEINTO_BY_TOPIC = 105;
// Namesrv 获取注册到Name Server的所有Broker集群信息
exports.GET_BROKER_CLUSTER_INFO = 106;
exports.UPDATE_AND_CREATE_SUBSCRIPTIONGROUP = 200;
exports.GET_ALL_SUBSCRIPTIONGROUP_CONFIG = 201;
exports.GET_TOPIC_STATS_INFO = 202;
exports.GET_CONSUMER_CONNECTION_LIST = 203;
exports.GET_PRODUCER_CONNECTION_LIST = 204;
exports.WIPE_WRITE_PERM_OF_BROKER = 205;
// 从Name Server获取完整Topic列表
exports.GET_ALL_TOPIC_LIST_FROM_NAMESERVER = 206;
// 从Broker删除订阅组
exports.DELETE_SUBSCRIPTIONGROUP = 207;
// 从Broker获取消费状态（进度）
exports.GET_CONSUME_STATS = 208;
// Suspend Consumer消费过程
exports.SUSPEND_CONSUMER = 209;
// Resume Consumer消费过程
exports.RESUME_CONSUMER = 210;
// 重置Consumer Offset
exports.RESET_CONSUMER_OFFSET_IN_CONSUMER = 211;
// 重置Consumer Offset
exports.RESET_CONSUMER_OFFSET_IN_BROKER = 212;
// 调整Consumer线程池数量
exports.ADJUST_CONSUMER_THREAD_POOL = 213;
// 查询消息被哪些消费组消费
exports.WHO_CONSUME_THE_MESSAGE = 214;
// 从Broker删除Topic配置
exports.DELETE_TOPIC_IN_BROKER = 215;
// 从Namesrv删除Topic配置
exports.DELETE_TOPIC_IN_NAMESRV = 216;
// Namesrv 通过 project 获取所有的 server ip 信息
exports.GET_KV_CONFIG_BY_VALUE = 217;
// Namesrv 删除指定 project group 下的所有 server ip 信息
exports.DELETE_KV_CONFIG_BY_VALUE = 218;
// 通过NameSpace获取所有的KV List
exports.GET_KVLIST_BY_NAMESPACE = 219;
// offset 重置
exports.RESET_CONSUMER_CLIENT_OFFSET = 220;
// 客户端订阅消息
exports.GET_CONSUMER_STATUS_FROM_CLIENT = 221;
// 通知 broker 调用 offset 重置处理
exports.INVOKE_BROKER_TO_RESET_OFFSET = 222;
// 通知 broker 调用客户端订阅消息处理
exports.INVOKE_BROKER_TO_GET_CONSUMER_STATUS = 223;
// Broker 查询topic被谁消费
// 2014-03-21 Add By shijia
exports.QUERY_TOPIC_CONSUME_BY_WHO = 300;
// 获取指定集群下的所有 topic
// 2014-03-26
exports.GET_TOPICS_BY_CLUSTER = 224;
// 向Broker注册Filter Server
// 2014-04-06 Add By shijia
exports.REGISTER_FILTER_SERVER = 301;
// 向Filter Server注册Class
// 2014-04-06 Add By shijia
exports.REGISTER_MESSAGE_FILTER_CLASS = 302;
// 根据 topic 和 group 获取消息的时间跨度
exports.QUERY_CONSUME_TIME_SPAN = 303;
// 获取所有系统内置 Topic 列表
exports.GET_SYSTEM_TOPIC_LIST_FROM_NS = 304;
exports.GET_SYSTEM_TOPIC_LIST_FROM_BROKER = 305;
// 清理失效队列
exports.CLEAN_EXPIRED_CONSUMEQUEUE = 306;
// 通过Broker查询Consumer内存数据
// 2014-07-19 Add By shijia
exports.GET_CONSUMER_RUNNING_INFO = 307;
// 查找被修正 offset (转发组件）
exports.QUERY_CORRECTION_OFFSET = 308;
// 通过Broker直接向某个Consumer发送一条消息，并立刻消费，返回结果给broker，再返回给调用方
// 2014-08-11 Add By shijia
exports.CONSUME_MESSAGE_DIRECTLY = 309;
// Broker 发送消息，优化网络数据包
exports.SEND_MESSAGE_V2 = 310;
// 单元化相关 topic
exports.GET_UNIT_TOPIC_LIST = 311;
// 获取含有单元化订阅组的 Topic 列表
exports.GET_HAS_UNIT_SUB_TOPIC_LIST = 312;
// 获取含有单元化订阅组的非单元化 Topic 列表
exports.GET_HAS_UNIT_SUB_UNUNIT_TOPIC_LIST = 313;
// 克隆某一个组的消费进度到新的组
exports.CLONE_GROUP_OFFSET = 314;
// 查看Broker上的各种统计信息
exports.VIEW_BROKER_STATS_DATA = 315;
================================================
FILE: lib/protocol/response_code.js
================================================
'use strict';
// 成功
exports.SUCCESS = 0;
// 发生了未捕获异常
exports.SYSTEM_ERROR = 1;
// 由于线程池拥堵，系统繁忙
exports.SYSTEM_BUSY = 2;
// 请求代码不支持
exports.REQUEST_CODE_NOT_SUPPORTED = 3;
// Broker 刷盘超时
exports.FLUSH_DISK_TIMEOUT = 10;
// Broker 同步双写，Slave不可用
exports.SLAVE_NOT_AVAILABLE = 11;
// Broker 同步双写，等待Slave应答超时
exports.FLUSH_SLAVE_TIMEOUT = 12;
// Broker 消息非法
exports.MESSAGE_ILLEGAL = 13;
// Broker, Namesrv 服务不可用，可能是正在关闭或者权限问题
exports.SERVICE_NOT_AVAILABLE = 14;
// Broker, Namesrv 版本号不支持
exports.VERSION_NOT_SUPPORTED = 15;
// Broker, Namesrv 无权限执行此操作，可能是发、收、或者其他操作
exports.NO_PERMISSION = 16;
// Broker, Topic不存在
exports.TOPIC_NOT_EXIST = 17;
// Broker, Topic已经存在，创建Topic
exports.TOPIC_EXIST_ALREADY = 18;
// Broker 拉消息未找到（请求的Offset等于最大Offset，最大Offset无对应消息）
exports.PULL_NOT_FOUND = 19;
// Broker 可能被过滤，或者误通知等
exports.PULL_RETRY_IMMEDIATELY = 20;
// Broker 拉消息请求的Offset不合法，太小或太大
exports.PULL_OFFSET_MOVED = 21;
// Broker 查询消息未找到
exports.QUERY_NOT_FOUND = 22;
// Broker 订阅关系解析失败
exports.SUBSCRIPTION_PARSE_FAILED = 23;
// Broker 订阅关系不存在
exports.SUBSCRIPTION_NOT_EXIST = 24;
// Broker 订阅关系不是最新的
exports.SUBSCRIPTION_NOT_LATEST = 25;
// Broker 订阅组不存在
exports.SUBSCRIPTION_GROUP_NOT_EXIST = 26;
// Producer 事务应该被提交
exports.TRANSACTION_SHOULD_COMMIT = 200;
// Producer 事务应该被回滚
exports.TRANSACTION_SHOULD_ROLLBACK = 201;
// Producer 事务状态未知
exports.TRANSACTION_STATE_UNKNOW = 202;
// Producer ProducerGroup错误
exports.TRANSACTION_STATE_GROUP_WRONG = 203;
// 单元化消息，需要设置 buyerId
exports.NO_BUYER_ID = 204;
// 单元化消息，非本单元消息
exports.NOT_IN_CURRENT_UNIT = 205;
// Consumer不在线
exports.CONSUMER_NOT_ONLINE = 206;
// Consumer消费消息超时
exports.CONSUME_MSG_TIMEOUT = 207;
================================================
FILE: lib/remoting_client.js
================================================
'use strict';
const assert = require('assert');
const Base = require('sdk-base');
const logger = require('./logger');
const Channel = require('./channel');
const defaultOptions = {
logger,
responseTimeout: 30000,
};
class RemotingClient extends Base {
/**
* rocketmq remoting client
* @param {Object} options
* - {HttpClient} httpclient - http request client
* - {Object} [logger] - log module
* - {Number} [responseTimeout] - tcp response timeout
* @class
*/
constructor(options) {
assert(options.onsAddr || options.nameSrv, '[RemotingClient] options.onsAddr or options.nameSrv one of them is required');
assert(options.httpclient, '[RemotingClient] options.httpclient is required');
super(Object.assign({ initMethod: 'init' }, defaultOptions, options));
this._index = 0;
this._inited = false;
this._namesrvAddrList = [];
this._channels = new Map();
}
/**
* @property {HttpClient} RemotingClient#httpclient
*/
get httpclient() {
return this.options.httpclient;
}
/**
* @property {Object} RemotingClient#logger
*/
get logger() {
return this.options.logger;
}
/**
* @property {Number} RemotingClient#responseTimeout
*/
get responseTimeout() {
return this.options.responseTimeout;
}
get unitName() {
return this.options.unitName;
}
/**
* start the client
* @return {void}
*/
async init() {
// get name server address at first
await this.updateNameServerAddressList();
this._inited = true;
this.logger.info('[mq:remoting_client] remoting client started');
}
/**
* close the client
* @return {void}
*/
async close() {
if (!this._inited) {
return;
}
// wait all channel close
await Promise.all(Array.from(this._channels.keys()).map(addr => {
return new Promise(resolve => {
const channel = this._channels.get(addr);
if (channel && channel.isOK) {
channel.once('close', resolve);
channel.close();
} else {
resolve();
}
this._channels.delete(addr);
});
}));
this._inited = false;
this.emit('close');
this.removeAllListeners();
this.logger.info('[mq:remoting_client] remoting client is closed');
}
/**
* default error handler
* @param {Error} err - error object
* @return {void}
*/
error(err) {
this.emit('error', err);
}
async handleClose(addr, channel) {
if (this._channels.get(addr) && this._channels.get(addr).clientId === channel.clientId) {
this._channels.delete(addr);
}
await this.updateNameServerAddressListThrottle();
}
async updateNameServerAddressListThrottle() {
if (this._updatingNameServerAddressList) {
// if it is in the process of updating and there is a new update task
// it needs to be recorded at this time and update again at the end to avoid missing the last update.
this._needToUpdatingNameServerAddressList = true;
return;
}
this._updatingNameServerAddressList = true;
let updateError;
try {
await this.updateNameServerAddressList();
} catch (e) {
updateError = e;
}
this._updatingNameServerAddressList = false;
if (this._needToUpdatingNameServerAddressList) {
this._needToUpdatingNameServerAddressList = false;
this.updateNameServerAddressListThrottle();
}
if (updateError) {
throw updateError;
}
}
/**
* fetch name server address list
* @return {Promise<void>}
*/
async updateNameServerAddressList() {
if (this.options.nameSrv) {
this._namesrvAddrList = Array.isArray(this.options.nameSrv) ? this.options.nameSrv : [ this.options.nameSrv ];
return;
}
let url = this.options.onsAddr;
if (this.unitName) {
url = url + '-' + this.unitName + '?nofix=1';
}
let retries = 3;
let ret;
while (retries-- > 0) {
ret = await this.httpclient.request(url, {
timeout: this.options.connectTimeout || 10000,
});
if (ret.status === 200) {
const addrs = ret.data.toString().trim();
const newList = addrs.split(';');
if (newList.length) {
this._namesrvAddrList = newList;
}
this.logger.info('[mq:remoting_client] fetch name server addresses successfully, address: %s', addrs);
return;
}
}
throw new Error('[mq:remoting_client] fetch name server addresses failed, ret.statusCode: ' + ret.status);
}
/**
* invoke command
* @param {String} addr - server address
* @param {RemotingCommand} command - remoting command
* @param {Number} [timeout] - response timeout
* @return {Promise<Object>} response
*/
async invoke(addr, command, timeout) {
if (!this._inited) {
await this.ready();
}
return await this.getAndCreateChannel(addr)
.invoke(command, timeout || this.responseTimeout);
}
/**
* invoke command for namesrv, return success at least once success
* @param {RemotingCommand} command - remoting command
* @param {Number} [timeout] - response timeout
* @return {Object} response
*/
async invokeForNameSrvAtLeastOnce(command, timeout) {
let response = {};
let count = this._namesrvAddrList.length;
let err = null;
while (count--) {
try {
response = await this.invoke(null, command, timeout);
// 有一个成功即无需抛 invoke 错误
err = null;
break;
} catch (e) {
err = e;
this.logger.warn(err);
}
}
if (err) {
throw err;
}
return response;
}
/**
* invoke command without response
* @param {String} addr - server address
* @param {RemotingCommand} command - remoting command
* @return {Promise<void>}
*/
async invokeOneway(addr, command) {
if (!this._inited) {
await this.ready();
}
await this.getAndCreateChannel(addr).invokeOneway(command);
}
/**
* get request channel from address
* @param {String} [addr] - server address
* @return {Promise<Channel>} request channel
*/
getAndCreateChannel(addr) {
if (!addr) {
this._index = ++this._index % this._namesrvAddrList.length;
addr = this._namesrvAddrList[this._index];
}
let channel = this._channels.get(addr);
if (channel && channel.isOK) {
return channel;
}
channel = new Channel(addr, this.options);
this._channels.set(addr, channel);
channel.once('close', this.handleClose.bind(this, addr, channel));
channel.on('error', err => this.error(err));
channel.on('request', (request, address) => this.emit('request', request, address));
return channel;
}
}
module.exports = RemotingClient;
================================================
FILE: lib/store/index.js
================================================
'use strict';
exports.LocalFileOffsetStore = require('./local_file');
exports.LocalMemoryOffsetStore = require('./local_memory');
exports.RemoteBrokerOffsetStore = require('./remote_broker');
================================================
FILE: lib/store/local_file.js
================================================
'use strict';
const fs = require('fs');
const path = require('path');
const osenv = require('osenv');
const mkdirp = require('mkdirp');
const is = require('is-type-of');
const Base = require('sdk-base');
const ReadOffsetType = require('./read_offset_type');
const localOffsetStoreDir = path.join(osenv.home(), '.rocketmq_offsets_node');
class LocalFileOffsetStore extends Base {
/**
* 消费进度存储到Consumer本地
* @param {MQClient} mqClient -
* @param {String} groupName 分组名
* @class
*/
constructor(mqClient, groupName) {
super();
this.mqClient = mqClient;
this.groupName = groupName;
this.offsetTable = new Map();
this.storePath = path.join(localOffsetStoreDir, mqClient.clientId, groupName, 'offsets.json');
mkdirp.sync(path.dirname(this.storePath));
}
get logger() {
return this.mqClient.logger;
}
/**
* 加载Offset
*/
async load() {
const data = await this.readLocalOffset();
if (data && data.offsetTable) {
for (const key in data.offsetTable) {
if (is.number(data.offsetTable[key])) {
this.offsetTable.set(key, data.offsetTable[key]);
}
}
}
}
/**
* 更新消费进度，存储到内存
* @param {MessageQueue} messageQueue -
* @param {Number} offset 新的进度
* @param {Boolean} increaseOnly 是否强制覆盖
* @return {void}
*/
updateOffset(messageQueue, offset, increaseOnly) {
increaseOnly = increaseOnly || false;
if (!messageQueue) {
return;
}
let prev = -1;
if (this.offsetTable.has(messageQueue.key)) {
prev = this.offsetTable.get(messageQueue.key);
}
if (prev >= offset && increaseOnly) {
return;
}
this.offsetTable.set(messageQueue.key, offset);
this.logger.info('[mq:LocalFileOffsetStore] update offset for messageQueue: %s, current offset: %d, prev offset: %s, increaseOnly: %s', messageQueue.key, offset, prev, increaseOnly);
}
/**
* 从本地缓存读取消费进度
* @param {MessageQueue} messageQueue -
* @param {String} type 消费类型（从哪里开始消费）
* @return {void}
*/
async readOffset(messageQueue, type) {
if (!messageQueue) {
return -1;
}
switch (type) {
case ReadOffsetType.MEMORY_FIRST_THEN_STORE:
case ReadOffsetType.READ_FROM_MEMORY:
{
const offset = this.offsetTable.get(messageQueue.key);
if (!is.nullOrUndefined(offset)) {
return offset;
}
break;
}
case ReadOffsetType.READ_FROM_STORE:
{
const data = await this.readLocalOffset();
if (data && data.offsetTable) {
const offset = data.offsetTable[messageQueue.key];
if (is.number(offset)) {
this.updateOffset(messageQueue, offset, false);
return offset;
}
}
break;
}
default:
break;
}
return -1;
}
/**
* 持久化全部消费进度，可能持久化本地或者远端Broker
* @param {Array} mqs -
* @return {void}
*/
async persistAll(mqs) {
if (mqs && mqs.length) {
const data = {
offsetTable: {},
};
const newTable = new Map();
for (const messageQueue of mqs) {
const offset = this.offsetTable.get(messageQueue.key) || -1;
data.offsetTable[messageQueue.key] = offset;
newTable.set(messageQueue.key, offset);
}
this.offsetTable = newTable;
fs.writeFileSync(this.storePath, JSON.stringify(data));
}
}
/* eslint-disable no-empty-function */
/**
* 持久化消费进度
* @return {void}
*/
async persist() {}
/* eslint-enable no-empty-function */
/**
* 删除不必要的MessageQueue offset
* @param {Object} messageQueue 消息队列
* @return {void}
*/
removeOffset(messageQueue) {
if (messageQueue) {
this.offsetTable.delete(messageQueue.key);
}
}
/**
* 从本地文件读消费进度
* @return {void}
*/
async readLocalOffset() {
if (!fs.existsSync(this.storePath)) {
return null;
}
const content = fs.readFileSync(this.storePath);
if (!content) {
return null;
}
try {
return JSON.parse(content.toString());
} catch (e) {
e.message = 'readLocalOffset() failed' + e.message;
this.emit('error', e);
return null;
}
}
}
module.exports = LocalFileOffsetStore;
================================================
FILE: lib/store/local_memory.js
================================================
'use strict';
const is = require('is-type-of');
const Base = require('sdk-base');
const ReadOffsetType = require('./read_offset_type');
class LocalMemoryOffsetStore extends Base {
/**
* 消费进度存储到Consumer本地
* @param {MQClient} mqClient -
* @param {String} groupName 分组名
* @class
*/
constructor(mqClient, groupName) {
super();
this.mqClient = mqClient;
this.groupName = groupName;
this.offsetTable = new Map();
}
get logger() {
return this.mqClient.logger;
}
/**
* 更新消费进度，存储到内存
* @param {MessageQueue} messageQueue -
* @param {Number} offset 新的进度
* @param {Boolean} increaseOnly 是否强制覆盖
* @return {void}
*/
updateOffset(messageQueue, offset, increaseOnly) {
increaseOnly = increaseOnly || false;
if (!messageQueue) {
return;
}
let prev = -1;
if (this.offsetTable.has(messageQueue.key)) {
prev = this.offsetTable.get(messageQueue.key);
}
if (prev >= offset && increaseOnly) {
return;
}
this.offsetTable.set(messageQueue.key, offset);
this.logger.info('[mq:LocalMemoryOffsetStore] update offset for messageQueue: %s, current offset: %d, prev offset: %s, increaseOnly: %s', messageQueue.key, offset, prev, increaseOnly);
}
/**
* 从本地缓存读取消费进度
* @param {MessageQueue} messageQueue -
* @param {String} type 消费类型（从哪里开始消费）
* @return {void}
*/
async readOffset(messageQueue, type) {
if (!messageQueue) {
return -1;
}
switch (type) {
case ReadOffsetType.MEMORY_FIRST_THEN_STORE:
case ReadOffsetType.READ_FROM_MEMORY:
case ReadOffsetType.READ_FROM_STORE:
{
const offset = this.offsetTable.get(messageQueue.key);
if (!is.nullOrUndefined(offset)) {
return offset;
}
break;
}
default:
break;
}
return -1;
}
/* eslint-disable no-empty-function */
async load() {}
async persistAll() {}
async persist() {}
/* eslint-enable no-empty-function */
/**
* 删除不必要的MessageQueue offset
* @param {Object} messageQueue 消息队列
* @return {void}
*/
removeOffset(messageQueue) {
if (messageQueue) {
this.offsetTable.delete(messageQueue.key);
}
}
}
module.exports = LocalMemoryOffsetStore;
================================================
FILE: lib/store/read_offset_type.js
================================================
'use strict';
module.exports = {
// 只从Memory读取
READ_FROM_MEMORY: 'READ_FROM_MEMORY',
// 只从存储层读取（本地或者远端）
READ_FROM_STORE: 'READ_FROM_STORE',
// 先从内存读，内存不存在再从存储层读
MEMORY_FIRST_THEN_STORE: 'MEMORY_FIRST_THEN_STORE',
};
================================================
FILE: lib/store/remote_broker.js
================================================
'use strict';
const is = require('is-type-of');
const Base = require('sdk-base');
const gather = require('co-gather');
const ReadOffsetType = require('./read_offset_type');
class RemoteBrokerOffsetStore extends Base {
/**
* consume offset store at remote server
* @param {MQClient} mqClient - mq client
* @param {String} groupName - group name
* @class
*/
constructor(mqClient, groupName) {
super();
this.mqClient = mqClient;
this.groupName = groupName;
this.offsetTable = new Map();
}
get logger() {
return this.mqClient.logger;
}
/* eslint-disable no-empty-function */
/**
* load the offset
* @return {void}
*/
async load() {}
/* eslint-enable no-empty-function */
/**
* update consume offset
* @param {MessageQueue} messageQueue - message queue
* @param {Number} offset - new offset
* @param {Boolean} increaseOnly - force override
* @return {void}
*/
updateOffset(messageQueue, offset, increaseOnly) {
increaseOnly = increaseOnly || false;
if (!messageQueue) {
return;
}
let prev = -1;
if (this.offsetTable.has(messageQueue.key)) {
prev = this.offsetTable.get(messageQueue.key);
}
if (prev >= offset && increaseOnly) {
return;
}
this.offsetTable.set(messageQueue.key, offset);
this.logger.info('[mq:RemoteBrokerOffsetStore] update offset for messageQueue: %s, current offset: %d, prev offset: %s, increaseOnly: %s', messageQueue.key, offset, prev, increaseOnly);
}
/**
* read consume offset
* @param {MessageQueue} messageQueue - message queue
* @param {String} type - consume from where
* @return {Promise<Number>} offset
*/
async readOffset(messageQueue, type) {
if (!messageQueue) {
return -1;
}
try {
switch (type) {
case ReadOffsetType.MEMORY_FIRST_THEN_STORE:
case ReadOffsetType.READ_FROM_MEMORY:
{
const offset = this.offsetTable.get(messageQueue.key);
if (!is.nullOrUndefined(offset)) {
return offset;
}
break;
}
case ReadOffsetType.READ_FROM_STORE:
{
const brokerOffset = await this.fetchConsumeOffsetFromBroker(messageQueue);
this.updateOffset(messageQueue, brokerOffset, false);
return brokerOffset;
}
default:
break;
}
} catch (err) {
err.message = `RemoteBrokerOffsetStore.readOffset failed, topic: ${messageQueue.topic}, group: ${this.groupName}, queueId: ${messageQueue.queueId}, type: ${type} ` + err.message;
this.emit('error', err);
}
return -1;
}
/**
* persist all consume offset
* @param {Array} mqs - all queues
* @return {void}
*/
async persistAll(mqs) {
if (!mqs || !mqs.length) {
return;
}
const newTable = new Map();
const tasks = [];
let info = 'persist all consume offset\n';
for (const messageQueue of mqs) {
const offset = this.offsetTable.get(messageQueue.key) || -1;
newTable.set(messageQueue.key, offset);
info += `\t- topic="${messageQueue.topic}", brokerName="${messageQueue.brokerName}", queueId="${messageQueue.queueId}", consumer offset => ${offset}\n`;
tasks.push(() => this.updateConsumeOffsetToBroker(messageQueue, offset));
}
this.logger.info('[mq:RemoteBrokerOffsetStore] %s', info);
this.offsetTable = newTable;
const ret = await gather(tasks);
const errors = ret.filter(item => item.isError);
for (const err of errors) {
this.emit('error', err);
}
}
/**
* persist consume offset
* @param {MessageQueue} messageQueue - message queue
* @return {void}
*/
async persist(messageQueue) {
const offset = this.offsetTable.get(messageQueue.key);
this.logger.info('[mq:RemoteBrokerOffsetStore] persist consume offset of [%s], offset: %d', messageQueue.key, offset);
if (is.number(offset)) {
await this.updateConsumeOffsetToBroker(messageQueue, offset);
}
}
/**
* remove message queue
* @param {MessageQueue} messageQueue - message queue
* @return {void}
*/
removeOffset(messageQueue) {
if (messageQueue) {
this.offsetTable.delete(messageQueue.key);
}
}
/**
* fetch consume offset from broker
* @param {MessageQueue} messageQueue - message queue
* @return {Promise<Number>} brokerOffset
*/
async fetchConsumeOffsetFromBroker(messageQueue) {
const requestHeader = {
topic: messageQueue.topic,
consumerGroup: this.groupName,
queueId: messageQueue.queueId,
};
const findBrokerResult = await this.findBrokerAddressInAdmin(messageQueue);
return await this.mqClient.queryConsumerOffset(findBrokerResult.brokerAddr, requestHeader, 1000 * 5);
}
/**
* update consume offset to broker
* @param {MessageQueue} messageQueue - message queue
* @param {Number} offset - offset
* @return {void}
*/
async updateConsumeOffsetToBroker(messageQueue, offset) {
const requestHeader = {
topic: messageQueue.topic,
consumerGroup: this.groupName,
queueId: messageQueue.queueId,
commitOffset: offset,
};
const findBrokerResult = await this.findBrokerAddressInAdmin(messageQueue);
// use oneway, cause this action may cost time.
await this.mqClient.updateConsumerOffsetOneway(findBrokerResult.brokerAddr, requestHeader);
}
async findBrokerAddressInAdmin(messageQueue) {
let findBrokerResult = this.mqClient.findBrokerAddressInAdmin(messageQueue.brokerName);
if (!findBrokerResult) {
await this.mqClient.updateTopicRouteInfoFromNameServer(messageQueue.topic);
findBrokerResult = this.mqClient.findBrokerAddressInAdmin(messageQueue.brokerName);
if (!findBrokerResult) {
throw new Error(`The broker[${messageQueue.brokerName}] not exist`);
}
}
return findBrokerResult;
}
}
module.exports = RemoteBrokerOffsetStore;
================================================
FILE: lib/utils/index.js
================================================
'use strict';
/* eslint no-bitwise:0 */
const zlib = require('zlib');
const MixAll = require('../mix_all');
const is = require('is-type-of');
// 压缩
exports.compress = function(bs) {
return zlib.deflateSync(bs);
};
// 解压
exports.uncompress = function(bs) {
return zlib.inflateSync(bs);
};
// 解析日期（注意时区为当地时区）
exports.parseDate = function(str) {
return new Date(
Number(str.slice(0, 4)),
Number(str.slice(4, 6)) - 1,
Number(str.slice(6, 8)),
Number(str.slice(8, 10)),
Number(str.slice(10, 12)),
Number(str.slice(12)));
};
// 返回日期时间格式，精度到秒
// 格式如下：20131223051900
exports.timeMillisToHumanString = function(time) {
const date = is.date(time) ? time : new Date(time);
return '' +
date.getFullYear() +
zeroize(date.getMonth() + 1) +
zeroize(date.getDate()) +
zeroize(date.getHours()) +
zeroize(date.getMinutes()) +
zeroize(date.getSeconds());
};
// 获取字符串的哈希值
exports.hashCode = function(str) {
// s[0]*31^(n-1) + s[1]*31^(n-2) + ... + s[n-1]
let hash = 0;
const len = str.length;
for (let i = 0; i < len; i++) {
hash = 31 * hash + str.charCodeAt(i) << 0;
}
return hash & 4294967295;
};
// 重置重试消息的 Topic
exports.resetRetryTopic = function(msg, consumerGroup) {
const groupTopic = MixAll.getRetryTopic(consumerGroup);
if (msg.topic === groupTopic) {
msg.topic = msg.retryTopic;
}
};
function zeroize(value, length) {
if (!length) {
length = 2;
}
value = String(value);
let zeros = '';
for (let i = 0; i < length - value.length; i++) {
zeros += '0';
}
return zeros + value;
}
================================================
FILE: lib/utils/message_sys_flag.js
================================================
'use strict';
/* eslint no-bitwise: 0 */
exports.CompressedFlag = 1 << 0;
exports.MultiTagsFlag = 1 << 1;
/**
* 7 6 5 4 3 2 1 0
* SysFlag 事务相关，从左属，2与3
*/
exports.TransactionNotType = 0 << 2;
exports.TransactionPreparedType = 1 << 2;
exports.TransactionCommitType = 2 << 2;
exports.TransactionRollbackType = 3 << 2;
exports.getTransactionValue = function(flag) {
return flag & exports.TransactionRollbackType;
};
exports.resetTransactionValue = function(flag, type) {
return flag & ~exports.TransactionRollbackType | type;
};
exports.clearCompressedFlag = function(flag) {
return flag & ~exports.CompressedFlag;
};
================================================
FILE: lib/utils/pull_sys_flag.js
================================================
'use strict';
/* eslint no-bitwise: 0 */
const FLAG_COMMIT_OFFSET = 1 << 0;
const FLAG_SUSPEND = 1 << 1;
const FLAG_SUBSCRIPTION = 1 << 2;
const FLAG_CLASS_FILTER = 1 << 3;
exports.buildSysFlag = function(commitOffset, suspend, subscription, classFilter) {
let flag = 0;
if (commitOffset) {
flag |= FLAG_COMMIT_OFFSET;
}
if (suspend) {
flag |= FLAG_SUSPEND;
}
if (subscription) {
flag |= FLAG_SUBSCRIPTION;
}
if (classFilter) {
flag |= FLAG_CLASS_FILTER;
}
return flag;
};
exports.clearCommitOffsetFlag = function(sysFlag) {
return sysFlag & ~FLAG_COMMIT_OFFSET;
};
exports.hasCommitOffsetFlag = function(sysFlag) {
return (sysFlag & FLAG_COMMIT_OFFSET) === FLAG_COMMIT_OFFSET;
};
exports.hasSuspendFlag = function(sysFlag) {
return (sysFlag & FLAG_SUSPEND) === FLAG_SUSPEND;
};
exports.hasSubscriptionFlag = function(sysFlag) {
return (sysFlag & FLAG_SUBSCRIPTION) === FLAG_SUBSCRIPTION;
};
exports.hasClassFilterFlag = function(sysFlag) {
return (sysFlag & FLAG_CLASS_FILTER) === FLAG_CLASS_FILTER;
};
================================================
FILE: package.json
================================================
{
"name": "ali-ons",
"version": "3.12.2",
"description": "Aliyun Open Notification Service Client",
"main": "lib/index.js",
"files": [
"lib"
],
"scripts": {
"pkgfiles": "egg-bin pkgfiles --check",
"lint": "eslint . --ext .js",
"test": "npm run lint && npm run test-local",
"test-local": "TEST_TIMEOUT=120000 egg-bin test",
"cov": "TEST_TIMEOUT=120000 egg-bin cov",
"ci": "npm run pkgfiles && npm run lint && npm run cov"
},
"repository": {
"type": "git",
"url": "git+https://github.com/ali-sdk/ali-ons.git"
},
"keywords": [
"ons",
"aliyun",
"ali-sdk"
],
"author": "gxcsoccer <gxcsoccer@126.com>",
"license": "MIT",
"bugs": {
"url": "https://github.com/ali-sdk/ali-ons/issues"
},
"homepage": "https://github.com/ali-sdk/ali-ons#readme",
"dependencies": {
"address": "^1.1.2",
"binary-search-insert": "^1.0.3",
"byte": "^2.0.0",
"bytes": "^3.1.0",
"co-gather": "^1.0.1",
"debug": "^4.3.1",
"is-type-of": "^1.2.1",
"JSON2": "^0.1.0",
"long": "^4.0.0",
"mkdirp": "^1.0.4",
"mz-modules": "^2.1.0",
"osenv": "^0.1.5",
"sdk-base": "^3.6.0",
"tcp-base": "^3.1.0",
"utility": "^1.17.0"
},
"devDependencies": {
"egg-bin": "^4.16.1",
"eslint": "^7.25.0",
"eslint-config-egg": "^9.0.0",
"mm": "^3.2.0",
"nyc": "^17.1.0",
"urllib": "^2.37.1"
},
"engines": {
"node": ">= 8.0.0"
}
}
================================================
FILE: test/allocate_message_queue_averagely.test.js
================================================
'use strict';
const assert = require('assert');
const AllocateMessageQueueAveragely = require('../lib/consumer/rebalance/allocate_message_queue_averagely');
describe('test/allocate_message_queue_averagely.test.js', function() {
const allocator = new AllocateMessageQueueAveragely();
it('should allocate queue averagely', function() {
assert(allocator.name === 'AVG');
let result = allocator.allocate('test', '1', [ 1, 2, 3, 4, 5, 6, 7, 8 ], [ '1', '2' ]);
assert.deepEqual(result, [ 1, 2, 3, 4 ]);
result = allocator.allocate('test', '2', [ 1, 2, 3, 4, 5, 6, 7, 8 ], [ '1', '2' ]);
assert.deepEqual(result, [ 5, 6, 7, 8 ]);
});
it('should allocate ok, indivisible', function() {
let result = allocator.allocate('test', '1', [ 1, 2, 3, 4, 5, 6, 7, 8 ], [ '1', '2', '3' ]);
assert.deepEqual(result, [ 1, 2, 3 ]);
result = allocator.allocate('test', '2', [ 1, 2, 3, 4, 5, 6, 7, 8 ], [ '1', '2', '3' ]);
assert.deepEqual(result, [ 4, 5, 6 ]);
result = allocator.allocate('test', '3', [ 1, 2, 3, 4, 5, 6, 7, 8 ], [ '1', '2', '3' ]);
assert.deepEqual(result, [ 7, 8 ]);
});
it('should allocate ok, indivisible 2', function() {
let result = allocator.allocate('test', '1', [ 1, 2, 3, 4, 5 ], [ '1', '2' ]);
assert.deepEqual(result, [ 1, 2, 3 ]);
result = allocator.allocate('test', '2', [ 1, 2, 3, 4, 5 ], [ '1', '2' ]);
assert.deepEqual(result, [ 4, 5 ]);
});
it('should allocate ok, indivisible 3', function() {
let result = allocator.allocate('test', '1', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3' ]);
assert.deepEqual(result, [ 1, 2 ]);
result = allocator.allocate('test', '2', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3' ]);
assert.deepEqual(result, [ 3, 4 ]);
result = allocator.allocate('test', '3', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3' ]);
assert.deepEqual(result, [ 5 ]);
});
it('should allocate ok, indivisible 4', function() {
let result = allocator.allocate('test', '1', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4' ]);
assert.deepEqual(result, [ 1, 2 ]);
result = allocator.allocate('test', '2', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4' ]);
assert.deepEqual(result, [ 3 ]);
result = allocator.allocate('test', '3', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4' ]);
assert.deepEqual(result, [ 4 ]);
result = allocator.allocate('test', '4', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4' ]);
assert.deepEqual(result, [ 5 ]);
});
it('should allocate ok, indivisible 5', function() {
let result = allocator.allocate('test', '1', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5' ]);
assert.deepEqual(result, [ 1 ]);
result = allocator.allocate('test', '2', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5' ]);
assert.deepEqual(result, [ 2 ]);
result = allocator.allocate('test', '3', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5' ]);
assert.deepEqual(result, [ 3 ]);
result = allocator.allocate('test', '4', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5' ]);
assert.deepEqual(result, [ 4 ]);
result = allocator.allocate('test', '5', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5' ]);
assert.deepEqual(result, [ 5 ]);
});
it('should allocate ok, indivisible 6', function() {
let result = allocator.allocate('test', '1', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5', '6' ]);
assert.deepEqual(result, [ 1 ]);
result = allocator.allocate('test', '2', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5', '6' ]);
assert.deepEqual(result, [ 2 ]);
result = allocator.allocate('test', '3', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5', '6' ]);
assert.deepEqual(result, [ 3 ]);
result = allocator.allocate('test', '4', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5', '6' ]);
assert.deepEqual(result, [ 4 ]);
result = allocator.allocate('test', '5', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5', '6' ]);
assert.deepEqual(result, [ 5 ]);
result = allocator.allocate('test', '6', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5', '6' ]);
assert.deepEqual(result, []);
});
it('should throw error if currentId is null', function() {
assert.throws(() => {
allocator.allocate('test', null, [ 1, 2, 3, 4, 5, 6, 7, 8 ], [ '1', '2' ]);
}, /currentCID is empty/);
});
it('should throw error if mqAll is empty', function() {
assert.throws(() => {
allocator.allocate('test', '1', [], [ '1', '2' ]);
}, /mqAll is null or mqAll empty/);
});
it('should throw error if cidAll is empty', function() {
assert.throws(() => {
allocator.allocate('test', '1', [ 1, 2, 3, 4, 5, 6, 7, 8 ], []);
}, /cidAll is null or cidAll empty/);
});
it('should get empty array if currentId not exists in cidAll', function() {
const result = allocator.allocate('test', 'not exists', [ 1, 2, 3, 4, 5, 6, 7, 8 ], [ '1', '2' ]);
assert.deepEqual(result, []);
});
});
================================================
FILE: test/channel.test.js
================================================
'use strict';
const mm = require('mm');
const net = require('net');
const assert = require('assert');
const urllib = require('urllib');
const sleep = require('mz-modules/sleep');
const Channel = require('../lib/channel');
const config = require('../example/config');
const RequestCode = require('../lib/protocol/request_code');
const ResponseCode = require('../lib/protocol/response_code');
const RemotingCommand = require('../lib/protocol/command/remoting_command');
const localIp = require('address').ip();
const { nameSrv } = require('../example/config');
const TOPIC = config.topic;
describe('test/channel.test.js', function() {
const onsAddr = 'http://onsaddr-internet.aliyun.com/rocketmq/nsaddr4client-internet';
let address;
before(function(done) {
urllib.request(onsAddr, function(err, data, result) {
if (err) {
return done(err);
}
if (result && result.status === 200) {
address = data.toString().trim().split(';')[0];
done();
} else {
done(new Error('no addresses'));
}
});
});
afterEach(mm.restore);
it('should connect ok', () => {
const channel = new Channel(nameSrv);
return channel.ready();
});
it('should close ok', done => {
const channel = new Channel(nameSrv);
channel.ready(err => {
if (err) {
done(err);
return;
}
assert(channel._socket);
channel.close();
});
channel.on('close', () => {
assert(!channel._socket);
done();
});
});
it('should invoke ok', async () => {
const channel = new Channel(nameSrv, config);
await channel.ready();
const res = await channel.invoke(new RemotingCommand({
code: RequestCode.GET_ROUTEINTO_BY_TOPIC,
customHeader: {
topic: TOPIC,
},
}), 5000);
assert(res);
assert(res.body);
channel.close();
});
it('should invoke ok though channel not ready', async () => {
const channel = new Channel(address);
const res = await channel.invoke(new RemotingCommand({
code: RequestCode.GET_ROUTEINTO_BY_TOPIC,
customHeader: {
topic: TOPIC,
},
}), 5000);
assert(res);
assert(res.remark && res.remark.includes('__accessKey is blank'));
channel.close();
});
it('should invoke oneway ok', async () => {
const channel = new Channel(nameSrv);
await channel.invokeOneway(new RemotingCommand({
code: RequestCode.GET_KV_CONFIG_BY_VALUE,
customHeader: {
namespace: 'PROJECT_CONFIG',
key: localIp,
},
}));
await channel.ready();
});
it('should close if timeout', async () => {
const server = net.createServer();
server.listen();
const { port } = server.address();
await sleep(100);
const channel = new Channel(`127.0.0.1:${port}`);
await assert.rejects(async () => {
await channel.invoke(new RemotingCommand({
code: RequestCode.GET_ROUTEINTO_BY_TOPIC,
customHeader: {
topic: TOPIC,
},
}), 5000);
}, {
name: 'ResponseTimeoutError',
});
await sleep(100);
assert(!channel._socket);
server.close();
});
// it('should emit error if connect failed', function(done) {
// done = pedding(done, 2);
// const channel = new Channel('100.100.100.100:9876');
// channel.once('close', function() {
// console.log('channel closed')
// done();
// });
// channel.once('error', function(err) {
// console.log('channel error', err);
// should.exist(err);
// done();
// });
// });
it('should get error response', async () => {
const channel = new Channel('100.100.100.100:9876');
try {
await channel.invoke(new RemotingCommand({
code: RequestCode.GET_KV_CONFIG_BY_VALUE,
customHeader: {
namespace: 'PROJECT_CONFIG',
key: localIp,
},
}));
} catch (err) {
console.log(err);
}
channel.close();
});
// it.only('should throw error if command is invalid', function*() {
// const channel = new Channel(nameSrv);
// let isError = false;
// try {
// yield channel.invoke('invalid command', 5000);
// } catch (err) {
// isError = true;
// err.name.should.equal('MQInvalidCommandError');
// err.message.should.equal('Invalid command');
// }
// isError.should.be.ok();
// channel.close();
// channel.close();
// });
it('should clearupInvokes after connection close', async () => {
const channel = new Channel(nameSrv);
await channel.ready();
channel.close();
let isError = false;
try {
await channel.invoke(new RemotingCommand({
code: RequestCode.GET_ROUTEINTO_BY_TOPIC,
customHeader: {
topic: 'NOT_EXISTS',
},
}), 5000);
} catch (err) {
isError = true;
assert(err.message === `The socket was closed. (address: ${nameSrv})`);
}
assert(isError);
});
// it('should throw error if invoke after close', function(done) {
// const channel = new Channel(nameSrv);
// channel.on('close', function() {
// channel.invoke(new RemotingCommand({
// code: RequestCode.GET_ROUTEINTO_BY_TOPIC,
// customHeader: {
// topic: 'NOT_EXISTS'
// }
// }), 5000, function(err, res) {
// should.exist(err);
// err.message.should.equal(`The socket is already closed`);
// done();
// });
// });
// channel.close();
// });
it('should invoke response ok', async () => {
const channel = new Channel(nameSrv);
await channel.ready();
await channel.invokeOneway(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, 1, ''));
});
it('should throw error if invoke timeout', async () => {
const channel = new Channel(nameSrv);
let isError = false;
try {
await channel.invoke(new RemotingCommand({
code: RequestCode.GET_KV_CONFIG_BY_VALUE,
customHeader: {
namespace: 'PROJECT_CONFIG',
key: localIp,
},
}), 1);
} catch (err) {
isError = true;
assert(err.name === 'ResponseTimeoutError');
assert(err.message === `Server no response in 1ms, address#${nameSrv}`);
}
channel.close();
assert(isError);
});
// it('should throw error if connection close by remote server', function(done) {
// const channel = new Channel(nameSrv);
// yield channel.ready();
// channel.invoke(new RemotingCommand({
// code: RequestCode.GET_KV_CONFIG_BY_VALUE,
// customHeader: {
// namespace: 'PROJECT_CONFIG',
// key: localIp,
// },
// }), 1, function(err, res) {
// should.exist(err);
// channel.invokes.size.should.equal(0);
// err.name.should.equal('MQSocketClosedError');
// err.message.should.containEql(`${address} conn closed by remote server`);
// done();
// });
// channel.close();
// });
});
================================================
FILE: test/consumer/rebalance/allocate_message_queue_averagely.test.js
================================================
'use strict';
const assert = require('assert');
const AllocateMessageQueueAveragely = require('../../../lib/consumer/rebalance/allocate_message_queue_averagely');
describe('test/allocate_message_queue_averagely.test.js', function() {
const allocator = new AllocateMessageQueueAveragely();
it('should allocate queue averagely', function() {
assert(allocator.name === 'AVG');
let result = allocator.allocate('test', '1', [ 1, 2, 3, 4, 5, 6, 7, 8 ], [ '1', '2' ]);
assert.deepEqual(result, [ 1, 2, 3, 4 ]);
result = allocator.allocate('test', '2', [ 1, 2, 3, 4, 5, 6, 7, 8 ], [ '1', '2' ]);
assert.deepEqual(result, [ 5, 6, 7, 8 ]);
});
it('should allocate ok, indivisible', function() {
let result = allocator.allocate('test', '1', [ 1, 2, 3, 4, 5, 6, 7, 8 ], [ '1', '2', '3' ]);
assert.deepEqual(result, [ 1, 2, 3 ]);
result = allocator.allocate('test', '2', [ 1, 2, 3, 4, 5, 6, 7, 8 ], [ '1', '2', '3' ]);
assert.deepEqual(result, [ 4, 5, 6 ]);
result = allocator.allocate('test', '3', [ 1, 2, 3, 4, 5, 6, 7, 8 ], [ '1', '2', '3' ]);
assert.deepEqual(result, [ 7, 8 ]);
});
it('should allocate ok, indivisible 2', function() {
let result = allocator.allocate('test', '1', [ 1, 2, 3, 4, 5 ], [ '1', '2' ]);
assert.deepEqual(result, [ 1, 2, 3 ]);
result = allocator.allocate('test', '2', [ 1, 2, 3, 4, 5 ], [ '1', '2' ]);
assert.deepEqual(result, [ 4, 5 ]);
});
it('should allocate ok, indivisible 3', function() {
let result = allocator.allocate('test', '1', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3' ]);
assert.deepEqual(result, [ 1, 2 ]);
result = allocator.allocate('test', '2', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3' ]);
assert.deepEqual(result, [ 3, 4 ]);
result = allocator.allocate('test', '3', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3' ]);
assert.deepEqual(result, [ 5 ]);
});
it('should allocate ok, indivisible 4', function() {
let result = allocator.allocate('test', '1', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4' ]);
assert.deepEqual(result, [ 1, 2 ]);
result = allocator.allocate('test', '2', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4' ]);
assert.deepEqual(result, [ 3 ]);
result = allocator.allocate('test', '3', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4' ]);
assert.deepEqual(result, [ 4 ]);
result = allocator.allocate('test', '4', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4' ]);
assert.deepEqual(result, [ 5 ]);
});
it('should allocate ok, indivisible 5', function() {
let result = allocator.allocate('test', '1', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5' ]);
assert.deepEqual(result, [ 1 ]);
result = allocator.allocate('test', '2', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5' ]);
assert.deepEqual(result, [ 2 ]);
result = allocator.allocate('test', '3', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5' ]);
assert.deepEqual(result, [ 3 ]);
result = allocator.allocate('test', '4', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5' ]);
assert.deepEqual(result, [ 4 ]);
result = allocator.allocate('test', '5', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5' ]);
assert.deepEqual(result, [ 5 ]);
});
it('should allocate ok, indivisible 6', function() {
let result = allocator.allocate('test', '1', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5', '6' ]);
assert.deepEqual(result, [ 1 ]);
result = allocator.allocate('test', '2', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5', '6' ]);
assert.deepEqual(result, [ 2 ]);
result = allocator.allocate('test', '3', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5', '6' ]);
assert.deepEqual(result, [ 3 ]);
result = allocator.allocate('test', '4', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5', '6' ]);
assert.deepEqual(result, [ 4 ]);
result = allocator.allocate('test', '5', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5', '6' ]);
assert.deepEqual(result, [ 5 ]);
result = allocator.allocate('test', '6', [ 1, 2, 3, 4, 5 ], [ '1', '2', '3', '4', '5', '6' ]);
assert.deepEqual(result, []);
});
it('should throw error if currentId is null', function() {
assert.throws(() => {
allocator.allocate('test', null, [ 1, 2, 3, 4, 5, 6, 7, 8 ], [ '1', '2' ]);
}, /currentCID is empty/);
});
it('should throw error if mqAll is empty', function() {
assert.throws(() => {
allocator.allocate('test', '1', [], [ '1', '2' ]);
}, /mqAll is null or mqAll empty/);
});
it('should throw error if cidAll is empty', function() {
assert.throws(() => {
allocator.allocate('test', '1', [ 1, 2, 3, 4, 5, 6, 7, 8 ], []);
}, /cidAll is null or cidAll empty/);
});
it('should get empty array if currentId not exists in cidAll', function() {
const result = allocator.allocate('test', 'not exists', [ 1, 2, 3, 4, 5, 6, 7, 8 ], [ '1', '2' ]);
assert.deepEqual(result, []);
});
});
================================================
FILE: test/index.test.js
================================================
'use strict';
const mm = require('mm');
const path = require('path');
const osenv = require('osenv');
const assert = require('assert');
const httpclient = require('urllib');
const utils = require('../lib/utils');
const Message = require('../').Message;
const Consumer = require('../').Consumer;
const Producer = require('../').Producer;
const sleep = require('mz-modules/sleep');
const rimraf = require('mz-modules/rimraf');
const config = require('../example/config');
const MixAll = require('../lib/mix_all');
const TOPIC = config.topic;
const localOffsetStoreDir = path.join(osenv.home(), '.rocketmq_offsets_node');
describe('test/index.test.js', () => {
describe('API', () => {
let producer;
let consumer;
before(async () => {
producer = new Producer(Object.assign({
httpclient,
retryAnotherBrokerWhenNotStoreOK: true,
compressMsgBodyOverHowmuch: 10,
}, config));
await producer.ready();
consumer = new Consumer(Object.assign({
httpclient,
rebalanceInterval: 1000,
isBroadcast: true,
}, config));
await consumer.ready();
consumer.subscribe(TOPIC, async msg => {
console.log('message receive ------------> ', msg.body.toString());
});
});
after(async () => {
await producer.close();
await consumer.close();
});
afterEach(mm.restore);
// it.skip('it should create topic ok', function*() {
// yield producer.createTopic('TBW102', 'XXX', 8);
// });
it('should select another broker if one failed', async () => {
mm(producer, 'sendKernelImpl', async () => {
mm.restore();
return {
sendStatus: 'FLUSH_DISK_TIMEOUT',
};
});
const msg = new Message(TOPIC, // topic
'TagA', // tag
'Hello TagA !!! ' // body
);
let sendResult = await producer.send(msg);
assert(sendResult && sendResult.msgId);
mm(producer, 'sendKernelImpl', async () => {
mm.restore();
const err = new Error('mock err');
err.name = 'MQClientException';
err.code = 205;
throw err;
});
sendResult = await producer.send(msg);
assert(sendResult && sendResult.msgId);
});
it('should tryToFindTopicPublishInfo if brokerAddr not found', async () => {
const msg = new Message(TOPIC, // topic
'TagA', // tag
'Hello TagA !!! ' // body
);
let sendResult = await producer.send(msg);
assert(sendResult && sendResult.msgId);
mm(producer._mqClient, 'findBrokerAddressInPublish', () => {
mm.restore();
return null;
});
sendResult = await producer.send(msg);
assert(sendResult && sendResult.msgId);
mm(producer._mqClient, 'findBrokerAddressInPublish', () => {
return null;
});
let isError = false;
try {
await producer.send(msg);
} catch (err) {
isError = true;
assert(/The broker\[.+\] not exist/i.test(err.message));
assert(err.name === 'MQClientException');
}
assert(isError);
mm.restore();
isError = false;
try {
await producer.send(new Message('%RETRY%' + TOPIC, // topic
'TagA', // tag
'Hello TagA !!! ' // body
));
} catch (err) {
isError = true;
assert(err.name === 'MQClientException');
}
assert(isError);
});
it('should emit error if compress failed', done => {
mm(utils, 'compress', function() {
throw new Error('mock error');
});
const msg = new Message(TOPIC, // topic
'TagA', // tag
'Hello TagA !!! sds' // body
);
producer.on('error', err => {
assert(err.name === 'MetaQCompressError');
done();
});
(async () => {
try {
const sendResult = await producer.send(msg);
assert(sendResult && sendResult.msgId);
console.log('over');
} catch (err) {
console.log(err);
}
})();
});
it('should updateProcessQueueTableInRebalance ok', async () => {
await sleep(3000);
await consumer.rebalanceByTopic(TOPIC);
const size = consumer.processQueueTable.size;
assert(size > 0);
const key = Array.from(consumer.processQueueTable.keys())[0];
const obj = consumer.processQueueTable.get(key);
const processQueue = obj.processQueue;
processQueue.lastPullTimestamp = 10000;
await consumer.rebalanceByTopic(TOPIC);
assert(consumer.processQueueTable.size === size);
await consumer.updateProcessQueueTableInRebalance(TOPIC, []);
assert(consumer.processQueueTable.size === 0);
await consumer.updateProcessQueueTableInRebalance(MixAll.getRetryTopic(consumer.consumerGroup), []);
assert(consumer.processQueueTable.size === 0);
});
it('should computePullFromWhere ok', async () => {
mm(consumer._offsetStore, 'readOffset', async () => {
return -1;
});
mm(consumer, 'consumeFromWhere', 'CONSUME_FROM_LAST_OFFSET');
let offset = await consumer.computePullFromWhere({
topic: '%RETRY%__',
});
assert(offset === 0);
mm(consumer, 'consumeFromWhere', 'CONSUME_FROM_TIMESTAMP');
mm(consumer._mqClient, 'maxOffset', async () => {
return 1000;
});
offset = await consumer.computePullFromWhere({
topic: '%RETRY%__',
});
assert(offset === 1000);
mm.error(consumer._mqClient, 'maxOffset');
offset = await consumer.computePullFromWhere({
topic: '%RETRY%__',
});
assert(offset === -1);
mm(consumer._offsetStore, 'readOffset', async () => {
return 100;
});
offset = await consumer.computePullFromWhere({
topic: 'TP',
});
assert(offset === 100);
mm(consumer, 'consumeFromWhere', 'CONSUME_FROM_FIRST_OFFSET');
offset = await consumer.computePullFromWhere({
topic: 'TP',
});
assert(offset === 100);
mm(consumer, 'consumeFromWhere', 'NOT_EXISTS');
offset = await consumer.computePullFromWhere({
topic: 'TP',
});
assert(offset === -1);
});
it('should support namespace', () => {
mm(producer, 'namespace', 'xxx');
mm(consumer, 'namespace', 'xxx');
assert(consumer.consumerGroup === `xxx%${config.consumerGroup}`);
assert(producer.producerGroup === `xxx%${config.producerGroup}`);
assert(consumer.formatTopic(`%RETRY%${consumer.consumerGroup}`) === `%RETRY%${consumer.consumerGroup}`);
assert(producer.formatTopic('TEST_TOPIC') === 'xxx%TEST_TOPIC');
assert(consumer.formatTopic('TEST_TOPIC') === 'xxx%TEST_TOPIC');
});
});
// 广播消费
describe('broadcast', () => {
[
'CONSUME_FROM_LAST_OFFSET_AND_FROM_MIN_WHEN_BOOT_FIRST',
'CONSUME_FROM_MIN_OFFSET',
'CONSUME_FROM_MAX_OFFSET',
'CONSUME_FROM_LAST_OFFSET',
'CONSUME_FROM_FIRST_OFFSET',
'CONSUME_FROM_TIMESTAMP',
].forEach(consumeFromWhere => {
let consumer;
let producer;
describe(`consumeFromWhere => ${consumeFromWhere}`, () => {
before(() => {
return rimraf(localOffsetStoreDir);
});
before(async () => {
consumer = new Consumer(Object.assign({
httpclient,
consumeFromWhere,
isBroadcast: true,
persistent: true,
rebalanceInterval: 2000,
}, config));
producer = new Producer(Object.assign({
httpclient,
}, config));
await consumer.ready();
await producer.ready();
});
after(async () => {
await producer.close();
const originFn = consumer._offsetStore.persistAll;
mm(consumer._offsetStore, 'persistAll', async mqs => {
assert(mqs && mqs.length);
return await originFn.call(consumer._offsetStore, mqs);
});
await consumer.close();
});
afterEach(mm.restore);
it('should subscribe message ok', async () => {
let msgId;
const received = new Set();
consumer.subscribe(TOPIC, 'TagA', async msg => {
console.log('message receive ------------> ', msg.msgId, msg.body.toString());
assert(msg.tags !== 'TagB');
received.add(msg.msgId);
if (msgId && received.has(msgId)) {
assert(msg.body.toString() === 'Hello MetaQ !!! ');
consumer.emit('TagA');
}
});
await sleep(5000);
let msg = new Message(TOPIC, // topic
'TagB', // tag
'Hello MetaQ !!! ' // body
);
let sendResult = await producer.send(msg);
assert(sendResult && sendResult.msgId);
msg = new Message(TOPIC, // topic
'TagA', // tag
'Hello MetaQ !!! ' // body
);
sendResult = await producer.send(msg);
assert(sendResult && sendResult.msgId);
msgId = sendResult.msgId;
console.log('send message success,', sendResult.msgId);
if (!received.has(msgId)) {
await consumer.await('TagA');
}
});
it.skip('should viewMessage ok', async () => {
const msg = new Message(TOPIC, // topic
'TagA', // tag
'Hello MetaQ !!! ' // body
);
const sendResult = await producer.send(msg);
assert(sendResult && sendResult.msgId);
await sleep(3000);
const message = await consumer.viewMessage(sendResult.msgId);
assert(message.body.toString() === 'Hello MetaQ !!! ');
});
});
});
});
// 集群消费
describe('cluster', () => {
let consumer;
let producer;
beforeEach(async () => {
consumer = new Consumer(Object.assign({
httpclient,
isBroadcast: false,
}, config));
await consumer.ready();
producer = new Producer(Object.assign({
httpclient,
}, config));
await producer.ready();
});
afterEach(async () => {
await consumer.close();
await producer.close();
});
afterEach(mm.restore);
it('should subscribe message ok', async () => {
await sleep(3000);
const msg = new Message(TOPIC, // topic
'TagA', // tag
'Hello MetaQ !!! ' // body
);
const sendResult = await producer.send(msg);
assert(sendResult && sendResult.msgId);
const msgId = sendResult.msgId;
console.log(sendResult);
await new Promise(r => {
consumer.subscribe(TOPIC, '*', async msg => {
if (msg.msgId === msgId) {
assert(msg.body.toString() === 'Hello MetaQ !!! ');
r();
}
});
});
});
it.skip('should subscribe message with SQL92 expression type ok', async () => {
// 公有云未开启 SQL 过滤
await sleep(3000);
const msg = new Message(TOPIC, // topic
'TagA', // tag
'Hello MetaQ !!! ' // body
);
msg.properties.a = '1';
const sendResult = await producer.send(msg);
assert(sendResult && sendResult.msgId);
const msgId = sendResult.msgId;
console.log(sendResult);
await new Promise(r => {
consumer.subscribe(TOPIC, {
expressionType: 'SQL92',
subString: 'a IS NOT NULL',
}, async msg => {
if (msg.msgId === msgId) {
assert(msg.body.toString() === 'Hello MetaQ !!! ');
r();
}
});
});
});
});
describe('process exception', () => {
let consumer;
let producer;
const logger = {
info() {},
warn() {},
error(...args) {
console.error(...args);
},
debug() {},
};
beforeEach(async () => {
consumer = new Consumer(Object.assign({
httpclient,
logger,
rebalanceInterval: 2000,
maxReconsumeTimes: 2,
}, config));
producer = new Producer(Object.assign({
httpclient,
logger,
}, config));
await consumer.ready();
await producer.ready();
});
afterEach(async () => {
await producer.close();
await consumer.close();
});
it('should not correctTags if process queue not empty', () => {
let done = false;
mm(consumer._offsetStore, 'updateOffset', () => {
done = true;
});
consumer.correctTagsOffset({
processQueue: {
msgCount: 0,
},
});
assert(done);
done = true;
mm.restore();
mm(consumer._offsetStore, 'updateOffset', () => {
done = false;
});
consumer.correctTagsOffset({
processQueue: {
msgCount: 1,
},
});
assert(done);
mm.restore();
});
it('should retry(consume later) if process failed', async () => {
let msgId;
consumer.subscribe(TOPIC, '*', async msg => {
console.warn('message receive ------------> ', msg.body.toString());
if (msg.msgId === msgId || msg.originMessageId === msgId) {
assert(msg.body.toString() === 'Hello MetaQ !!! ');
if (msg.reconsumeTimes === 0) {
throw new Error('mock error');
}
consumer.emit('*');
}
});
await sleep(5000);
const msg = new Message(TOPIC, // topic
'TagA', // tag
'Hello MetaQ !!! ' // body
);
const sendResult = await producer.send(msg);
gitextract_g5wtfrkh/
├── .eslintignore
├── .eslintrc
├── .github/
│ └── workflows/
│ ├── ci-actions.yml
│ └── release.yml
├── .gitignore
├── AUTHORS
├── CHANGELOG.md
├── History.md
├── LICENSE
├── README.md
├── example/
│ ├── config.js
│ ├── consumer.js
│ ├── logger.js
│ └── producer.js
├── lib/
│ ├── channel.js
│ ├── client_config.js
│ ├── consumer/
│ │ ├── consume_from_where.js
│ │ ├── mq_push_consumer.js
│ │ ├── pull_status.js
│ │ └── rebalance/
│ │ ├── allocate_message_queue_averagely.js
│ │ └── allocate_message_queue_strategy.js
│ ├── index.js
│ ├── logger.js
│ ├── message/
│ │ ├── message.js
│ │ ├── message_const.js
│ │ └── message_decoder.js
│ ├── message_queue.js
│ ├── mix_all.js
│ ├── mq_client.js
│ ├── mq_client_api.js
│ ├── process_queue.js
│ ├── producer/
│ │ ├── mq_producer.js
│ │ ├── send_status.js
│ │ └── topic_publish_info.js
│ ├── protocol/
│ │ ├── command/
│ │ │ ├── opaque_generator.js
│ │ │ └── remoting_command.js
│ │ ├── consume_type.js
│ │ ├── message_model.js
│ │ ├── perm_name.js
│ │ ├── request_code.js
│ │ └── response_code.js
│ ├── remoting_client.js
│ ├── store/
│ │ ├── index.js
│ │ ├── local_file.js
│ │ ├── local_memory.js
│ │ ├── read_offset_type.js
│ │ └── remote_broker.js
│ └── utils/
│ ├── index.js
│ ├── message_sys_flag.js
│ └── pull_sys_flag.js
├── package.json
└── test/
├── allocate_message_queue_averagely.test.js
├── channel.test.js
├── consumer/
│ └── rebalance/
│ └── allocate_message_queue_averagely.test.js
├── index.test.js
├── index_namesrv.test.js
├── message/
│ └── message_decoder.test.js
├── mq_client.test.js
├── mq_client_api.test.js
├── protocol/
│ └── command/
│ ├── opaque_generator.test.js
│ └── remoting_command.test.js
├── remoting_client.test.js
├── store/
│ ├── local_file.test.js
│ ├── local_memory.test.js
│ └── remote_broker.test.js
├── utils/
│ ├── index.test.js
│ ├── message_sys_flag.test.js
│ └── pull_sys_flag.test.js
└── utils.js
SYMBOL INDEX (309 symbols across 31 files)
FILE: example/logger.js
method info (line 4) | info() {}
method warn (line 5) | warn() {}
method error (line 6) | error(...args) {
method debug (line 9) | debug() {}
FILE: lib/channel.js
class Channel (line 9) | class Channel extends Base {
method constructor (line 19) | constructor(address, options = {}) {
method accessKey (line 34) | get accessKey() {
method secretKey (line 38) | get secretKey() {
method onsChannel (line 42) | get onsChannel() {
method getBodyLength (line 51) | getBodyLength(header) {
method decode (line 55) | decode(body, header) {
method beforeRequest (line 64) | beforeRequest(command) {
method invoke (line 106) | invoke(command, timeout) {
method invokeOneway (line 126) | invokeOneway(command) {
FILE: lib/client_config.js
class ClientConfig (line 28) | class ClientConfig extends Base {
method constructor (line 39) | constructor(options) {
method clientId (line 44) | get clientId() {
method pollNameServerInteval (line 50) | get pollNameServerInteval() {
method heartbeatBrokerInterval (line 54) | get heartbeatBrokerInterval() {
method persistConsumerOffsetInterval (line 58) | get persistConsumerOffsetInterval() {
method rebalanceInterval (line 62) | get rebalanceInterval() {
method unitMode (line 66) | get unitMode() {
method unitName (line 70) | get unitName() {
method namespace (line 74) | get namespace() {
method formatTopic (line 78) | formatTopic(topic) {
method changeInstanceNameToPID (line 89) | changeInstanceNameToPID() {
FILE: lib/consumer/mq_push_consumer.js
class MQPushConsumer (line 53) | class MQPushConsumer extends ClientConfig {
method constructor (line 54) | constructor(options) {
method logger (line 84) | get logger() {
method subscriptions (line 88) | get subscriptions() {
method processQueueTable (line 92) | get processQueueTable() {
method parallelConsumeLimit (line 96) | get parallelConsumeLimit() {
method consumerGroup (line 100) | get consumerGroup() {
method messageModel (line 107) | get messageModel() {
method consumeType (line 111) | get consumeType() {
method consumeFromWhere (line 115) | get consumeFromWhere() {
method allocateMessageQueueStrategy (line 119) | get allocateMessageQueueStrategy() {
method init (line 123) | async init() {
method close (line 154) | async close() {
method newOffsetStoreInstance (line 168) | newOffsetStoreInstance() {
method subscribe (line 185) | subscribe(topic, subExpression, handler) {
method _consumeMessageLoop (line 227) | async _consumeMessageLoop(topic, needFilter, tagsSet, subExpression) {
method consumeSingleMsg (line 277) | async consumeSingleMsg(handler, msg, mq, pq) {
method buildSubscriptionData (line 330) | buildSubscriptionData(consumerGroup, topic, subString) {
method persistConsumerOffset (line 357) | async persistConsumerOffset() {
method pullMessageQueue (line 373) | pullMessageQueue(messageQueue, processQueue) {
method executePullRequestImmediately (line 406) | async executePullRequestImmediately(messageQueue) {
method pullKernelImpl (line 485) | async pullKernelImpl(messageQueue, subExpression, expressionType, subV...
method findBrokerAddress (line 521) | async findBrokerAddress(messageQueue) {
method recalculatePullFromWhichNode (line 533) | recalculatePullFromWhichNode(messageQueue) {
method correctTagsOffset (line 541) | correctTagsOffset(pullRequest) {
method updatePullFromWhichNode (line 548) | updatePullFromWhichNode(messageQueue, brokerId) {
method updateTopicSubscribeInfo (line 558) | updateTopicSubscribeInfo(topic, info) {
method isSubscribeTopicNeedUpdate (line 569) | isSubscribeTopicNeedUpdate(topic) {
method doRebalance (line 580) | async doRebalance() {
method rebalanceByTopic (line 586) | async rebalanceByTopic(topic) {
method updateProcessQueueTableInRebalance (line 623) | async updateProcessQueueTableInRebalance(topic, mqSet) {
method computePullFromWhere (line 683) | async computePullFromWhere(messageQueue) {
method removeProcessQueue (line 746) | async removeProcessQueue(messageQueue) {
method removeUnnecessaryMessageQueue (line 761) | async removeUnnecessaryMessageQueue(messageQueue) {
method sendMessageBack (line 768) | async sendMessageBack(msg, delayLevel, brokerName, consumerGroup) {
method _handleError (line 809) | _handleError(err) {
method _sleep (line 814) | _sleep(timeout) {
function compare (line 825) | function compare(mqA, mqB) {
FILE: lib/consumer/rebalance/allocate_message_queue_averagely.js
class AllocateMessageQueueAveragely (line 6) | class AllocateMessageQueueAveragely extends AllocateMessageQueueStrategy {
method name (line 7) | get name() {
method allocate (line 11) | allocate(consumerGroup, currentCID, mqAll, cidAll) {
FILE: lib/consumer/rebalance/allocate_message_queue_strategy.js
class AllocateMessageQueueStrategy (line 5) | class AllocateMessageQueueStrategy {
method allocate (line 13) | allocate(consumerGroup, currentCID, mqAll, cidAll) { /* eslint no-unus...
method name (line 20) | get name() {
FILE: lib/logger.js
method info (line 4) | info() {}
method warn (line 5) | warn(...args) {
method error (line 8) | error(...args) {
method debug (line 11) | debug() {}
FILE: lib/message/message.js
class Message (line 6) | class Message {
method constructor (line 15) | constructor(topic, tags, body) {
method tags (line 50) | get tags() {
method tags (line 54) | set tags(val) {
method keys (line 62) | get keys() {
method keys (line 66) | set keys(val) {
method originMessageId (line 74) | get originMessageId() {
method originMessageId (line 78) | set originMessageId(val) {
method retryTopic (line 86) | get retryTopic() {
method retryTopic (line 90) | set retryTopic(val) {
method delayTimeLevel (line 98) | get delayTimeLevel() {
method delayTimeLevel (line 106) | set delayTimeLevel(val) {
method waitStoreMsgOK (line 114) | get waitStoreMsgOK() {
method waitStoreMsgOK (line 119) | set waitStoreMsgOK(val) {
method buyerId (line 127) | get buyerId() {
method buyerId (line 131) | set buyerId(val) {
method setStartDeliverTime (line 139) | setStartDeliverTime(delayTime) {
method getStartDeliverTime (line 147) | getStartDeliverTime() {
method shardingKey (line 151) | get shardingKey() {
method shardingKey (line 154) | set shardingKey(val) {
FILE: lib/message/message_decoder.js
constant MSG_ID_LENGTH (line 12) | const MSG_ID_LENGTH = 8 + 8;
constant NAME_VALUE_SEPARATOR (line 13) | const NAME_VALUE_SEPARATOR = String.fromCharCode(1);
constant PROPERTY_SEPARATOR (line 14) | const PROPERTY_SEPARATOR = String.fromCharCode(2);
function innerDecode (line 80) | function innerDecode(byteBuffer, readBody, deCompressBody) {
function string2bytes (line 181) | function string2bytes(hexString) {
function bytes2string (line 188) | function bytes2string(src) {
function string2messageProperties (line 195) | function string2messageProperties(properties) {
FILE: lib/message_queue.js
class MessageQueue (line 5) | class MessageQueue {
method constructor (line 6) | constructor(topic, brokerName, queueId) {
FILE: lib/mq_client.js
class MQClient (line 15) | class MQClient extends MQClientAPI {
method constructor (line 22) | constructor(clientConfig) {
method clientId (line 36) | get clientId() {
method pollNameServerInteval (line 43) | get pollNameServerInteval() {
method heartbeatBrokerInterval (line 50) | get heartbeatBrokerInterval() {
method persistConsumerOffsetInterval (line 57) | get persistConsumerOffsetInterval() {
method rebalanceInterval (line 64) | get rebalanceInterval() {
method init (line 71) | async init() {
method close (line 83) | async close() {
method startScheduledTask (line 99) | startScheduledTask(name, interval, delay) {
method registerConsumer (line 120) | registerConsumer(group, consumer) {
method unregisterConsumer (line 134) | async unregisterConsumer(group) {
method registerProducer (line 146) | registerProducer(group, producer) {
method unregisterProducer (line 160) | async unregisterProducer(group) {
method unregister (line 172) | async unregister(producerGroup, consumerGroup) {
method updateAllTopicRouterInfo (line 193) | async updateAllTopicRouterInfo() {
method updateTopicRouteInfoFromNameServer (line 232) | async updateTopicRouteInfoFromNameServer(topic, isDefault, defaultMQPr...
method _refreshTopicRouteInfo (line 254) | _refreshTopicRouteInfo(topic, topicRouteData) {
method _isRouteDataChanged (line 317) | _isRouteDataChanged(prev, current) {
method _isNeedUpdateTopicRouteInfo (line 325) | _isNeedUpdateTopicRouteInfo(topic) {
method _topicRouteData2TopicPublishInfo (line 344) | _topicRouteData2TopicPublishInfo(topic, topicRouteData) {
method _topicRouteData2TopicSubscribeInfo (line 376) | _topicRouteData2TopicSubscribeInfo(topic, topicRouteData) {
method sendHeartbeatToAllBroker (line 392) | async sendHeartbeatToAllBroker() {
method _prepareHeartbeatData (line 428) | _prepareHeartbeatData() {
method _cleanOfflineBroker (line 470) | _cleanOfflineBroker() {
method _isBrokerAddrExistInTopicRouteTable (line 492) | _isBrokerAddrExistInTopicRouteTable(addr) {
method doRebalance (line 510) | async doRebalance() {
method findConsumerIdList (line 525) | async findConsumerIdList(topic, group) {
method findBrokerAddrByTopic (line 542) | findBrokerAddrByTopic(topic) {
method findBrokerAddressInAdmin (line 566) | findBrokerAddressInAdmin(brokerName) {
method findBrokerAddressInSubscribe (line 588) | findBrokerAddressInSubscribe(brokerName, brokerId, onlyThisBroker) {
method maxOffset (line 624) | async maxOffset(messageQueue) {
method findBrokerAddressInPublish (line 641) | findBrokerAddressInPublish(brokerName) {
method persistAllConsumerOffset (line 646) | async persistAllConsumerOffset() {
method searchOffset (line 655) | async searchOffset(messageQueue, timestamp) {
method getAndCreateMQClient (line 671) | static getAndCreateMQClient(clientConfig) {
FILE: lib/mq_client_api.js
constant JSON2 (line 3) | const JSON2 = require('JSON2');
constant NAMESPACE_PROJECT_CONFIG (line 19) | const NAMESPACE_PROJECT_CONFIG = 'PROJECT_CONFIG';
constant VIRTUAL_APPGROUP_PREFIX (line 20) | const VIRTUAL_APPGROUP_PREFIX = '%%PROJECT_%s%%';
class MQClientAPI (line 26) | class MQClientAPI extends RemotingClient {
method constructor (line 36) | constructor(options) {
method init (line 47) | async init() {
method getProjectGroupByIp (line 58) | async getProjectGroupByIp(ip, timeoutMillis) {
method getKVConfigByValue (line 75) | async getKVConfigByValue(namespace, value, timeoutMillis) {
method getDefaultTopicRouteInfoFromNameServer (line 100) | async getDefaultTopicRouteInfoFromNameServer(topic, timeoutMillis) {
method unregisterClient (line 139) | async unregisterClient(addr, clientId, producerGroup, consumerGroup, t...
method getTopicRouteInfoFromNameServer (line 166) | async getTopicRouteInfoFromNameServer(topic, timeoutMillis) {
method sendHearbeat (line 200) | async sendHearbeat(addr, heartbeatData, timeout) {
method updateConsumerOffsetOneway (line 226) | async updateConsumerOffsetOneway(brokerAddr, requestHeader) {
method queryConsumerOffset (line 241) | async queryConsumerOffset(brokerAddr, requestHeader, timeoutMillis) {
method getMaxOffset (line 267) | async getMaxOffset(addr, topic, queueId, timeoutMillis) {
method searchOffset (line 298) | async searchOffset(addr, topic, queueId, timestamp, timeoutMillis) {
method getConsumerIdListByGroup (line 327) | async getConsumerIdListByGroup(addr, consumerGroup, timeoutMillis) {
method pullMessage (line 354) | async pullMessage(brokerAddr, requestHeader, timeoutMillis) {
method createTopic (line 411) | async createTopic(addr, defaultTopic, topicConfig, timeoutMillis) {
method sendMessage (line 443) | async sendMessage(brokerAddr, brokerName, msg, requestHeader, timeoutM...
method consumerSendMessageBack (line 504) | async consumerSendMessageBack(brokerAddr, msg, consumerGroup, delayLev...
method _defaultHandler (line 549) | _defaultHandler(request, response) {
method _buildWithProjectGroup (line 556) | _buildWithProjectGroup(origin) {
method _clearProjectGroup (line 567) | _clearProjectGroup(origin) {
function compare (line 580) | function compare(routerA, routerB) {
FILE: lib/process_queue.js
class ProcessQueue (line 9) | class ProcessQueue extends Base {
method constructor (line 10) | constructor(options = {}) {
method maxSpan (line 22) | get maxSpan() {
method msgCount (line 30) | get msgCount() {
method isPullExpired (line 34) | get isPullExpired() {
method putMessage (line 38) | putMessage(msgs) {
method remove (line 45) | remove(count = 1) {
method clear (line 50) | clear() {
FILE: lib/producer/mq_producer.js
class MQProducer (line 30) | class MQProducer extends ClientConfig {
method constructor (line 37) | constructor(options) {
method logger (line 51) | get logger() {
method producerGroup (line 55) | get producerGroup() {
method createTopicKey (line 62) | get createTopicKey() {
method defaultTopicQueueNums (line 66) | get defaultTopicQueueNums() {
method publishTopicList (line 74) | get publishTopicList() {
method init (line 82) | async init() {
method close (line 94) | async close() {
method _error (line 109) | _error(err) {
method updateTopicPublishInfo (line 122) | updateTopicPublishInfo(topic, info) {
method isPublishTopicNeedUpdate (line 138) | isPublishTopicNeedUpdate(topic) {
method send (line 188) | async send(msg) {
method tryToFindTopicPublishInfo (line 245) | async tryToFindTopicPublishInfo(topic) {
method sendKernelImpl (line 261) | async sendKernelImpl(msg, messageQueue) {
method tryToCompressMessage (line 319) | tryToCompressMessage(msg) {
method getDefaultProducer (line 343) | static async getDefaultProducer(options = {}) {
FILE: lib/producer/topic_publish_info.js
class TopicPublishInfo (line 5) | class TopicPublishInfo {
method constructor (line 6) | constructor() {
method ok (line 16) | get ok() {
method selectOneMessageQueue (line 26) | selectOneMessageQueue(lastBrokerName, message) {
FILE: lib/protocol/command/opaque_generator.js
constant MAX_INT_31 (line 3) | const MAX_INT_31 = Math.pow(2, 31) - 10;
FILE: lib/protocol/command/remoting_command.js
constant REQUEST_COMMAND (line 9) | const REQUEST_COMMAND = 'REQUEST_COMMAND';
constant RESPONSE_COMMAND (line 10) | const RESPONSE_COMMAND = 'RESPONSE_COMMAND';
constant RPC_TYPE (line 14) | const RPC_TYPE = 0;
constant RPC_ONEWAY (line 17) | const RPC_ONEWAY = 1;
constant MQ_VERSION (line 20) | const MQ_VERSION = 121;
class RemotingCommand (line 24) | class RemotingCommand {
method constructor (line 36) | constructor(options) {
method language (line 46) | get language() {
method version (line 50) | get version() {
method type (line 54) | get type() {
method isResponseType (line 58) | get isResponseType() {
method isOnewayRPC (line 63) | get isOnewayRPC() {
method markResponseType (line 68) | markResponseType() {
method markOnewayRPC (line 73) | markOnewayRPC() {
method makeCustomHeaderToNet (line 78) | makeCustomHeaderToNet() {
method decodeCommandCustomHeader (line 94) | decodeCommandCustomHeader() {
method buildHeader (line 98) | buildHeader() {
method encode (line 111) | encode() {
method createRequestCommand (line 136) | static createRequestCommand(code, customHeader) {
method createResponseCommand (line 150) | static createResponseCommand(code, opaque, remark) {
method decode (line 165) | static decode(packet) {
FILE: lib/protocol/perm_name.js
constant PERM_READ (line 5) | const PERM_READ = 0x1 << 2;
constant PERM_WRITE (line 6) | const PERM_WRITE = 0x1 << 1;
constant PERM_INHERIT (line 7) | const PERM_INHERIT = 0x1 << 0;
FILE: lib/remoting_client.js
class RemotingClient (line 13) | class RemotingClient extends Base {
method constructor (line 23) | constructor(options) {
method httpclient (line 36) | get httpclient() {
method logger (line 43) | get logger() {
method responseTimeout (line 50) | get responseTimeout() {
method unitName (line 54) | get unitName() {
method init (line 62) | async init() {
method close (line 73) | async close() {
method error (line 104) | error(err) {
method handleClose (line 108) | async handleClose(addr, channel) {
method updateNameServerAddressListThrottle (line 115) | async updateNameServerAddressListThrottle() {
method updateNameServerAddressList (line 143) | async updateNameServerAddressList() {
method invoke (line 178) | async invoke(addr, command, timeout) {
method invokeForNameSrvAtLeastOnce (line 192) | async invokeForNameSrvAtLeastOnce(command, timeout) {
method invokeOneway (line 219) | async invokeOneway(addr, command) {
method getAndCreateChannel (line 231) | getAndCreateChannel(addr) {
FILE: lib/store/local_file.js
class LocalFileOffsetStore (line 12) | class LocalFileOffsetStore extends Base {
method constructor (line 20) | constructor(mqClient, groupName) {
method logger (line 30) | get logger() {
method load (line 37) | async load() {
method updateOffset (line 56) | updateOffset(messageQueue, offset, increaseOnly) {
method readOffset (line 79) | async readOffset(messageQueue, type) {
method persistAll (line 117) | async persistAll(mqs) {
method persist (line 140) | async persist() {}
method removeOffset (line 149) | removeOffset(messageQueue) {
method readLocalOffset (line 159) | async readLocalOffset() {
FILE: lib/store/local_memory.js
class LocalMemoryOffsetStore (line 7) | class LocalMemoryOffsetStore extends Base {
method constructor (line 14) | constructor(mqClient, groupName) {
method logger (line 21) | get logger() {
method updateOffset (line 32) | updateOffset(messageQueue, offset, increaseOnly) {
method readOffset (line 55) | async readOffset(messageQueue, type) {
method load (line 78) | async load() {}
method persistAll (line 80) | async persistAll() {}
method persist (line 82) | async persist() {}
method removeOffset (line 91) | removeOffset(messageQueue) {
FILE: lib/store/remote_broker.js
class RemoteBrokerOffsetStore (line 8) | class RemoteBrokerOffsetStore extends Base {
method constructor (line 16) | constructor(mqClient, groupName) {
method logger (line 24) | get logger() {
method load (line 33) | async load() {}
method updateOffset (line 44) | updateOffset(messageQueue, offset, increaseOnly) {
method readOffset (line 67) | async readOffset(messageQueue, type) {
method persistAll (line 104) | async persistAll(mqs) {
method persist (line 132) | async persist(messageQueue) {
method removeOffset (line 145) | removeOffset(messageQueue) {
method fetchConsumeOffsetFromBroker (line 156) | async fetchConsumeOffsetFromBroker(messageQueue) {
method updateConsumeOffsetToBroker (line 172) | async updateConsumeOffsetToBroker(messageQueue, offset) {
method findBrokerAddressInAdmin (line 184) | async findBrokerAddressInAdmin(messageQueue) {
FILE: lib/utils/index.js
function zeroize (line 65) | function zeroize(value, length) {
FILE: lib/utils/pull_sys_flag.js
constant FLAG_COMMIT_OFFSET (line 4) | const FLAG_COMMIT_OFFSET = 1 << 0;
constant FLAG_SUSPEND (line 5) | const FLAG_SUSPEND = 1 << 1;
constant FLAG_SUBSCRIPTION (line 6) | const FLAG_SUBSCRIPTION = 1 << 2;
constant FLAG_CLASS_FILTER (line 7) | const FLAG_CLASS_FILTER = 1 << 3;
FILE: test/channel.test.js
constant TOPIC (line 16) | const TOPIC = config.topic;
FILE: test/index.test.js
constant TOPIC (line 17) | const TOPIC = config.topic;
method info (line 404) | info() {}
method warn (line 405) | warn() {}
method error (line 406) | error(...args) {
method debug (line 409) | debug() {}
FILE: test/index_namesrv.test.js
constant TOPIC (line 20) | const TOPIC = config.topic;
FILE: test/message/message_decoder.test.js
constant NAME_VALUE_SEPARATOR (line 9) | const NAME_VALUE_SEPARATOR = String.fromCharCode(1);
constant PROPERTY_SEPARATOR (line 10) | const PROPERTY_SEPARATOR = String.fromCharCode(2);
constant SYSTEM_PROP_KEY_STARTDELIVERTIME (line 11) | const SYSTEM_PROP_KEY_STARTDELIVERTIME = '__STARTDELIVERTIME';
FILE: test/mq_client.test.js
method updateTopicSubscribeInfo (line 28) | updateTopicSubscribeInfo() {}
method isSubscribeTopicNeedUpdate (line 29) | isSubscribeTopicNeedUpdate() {}
method doRebalance (line 30) | doRebalance() {}
method updateTopicPublishInfo (line 42) | updateTopicPublishInfo() {}
method isPublishTopicNeedUpdate (line 43) | isPublishTopicNeedUpdate() {}
method updateTopicSubscribeInfo (line 55) | updateTopicSubscribeInfo() {}
method isSubscribeTopicNeedUpdate (line 56) | isSubscribeTopicNeedUpdate() {}
method doRebalance (line 57) | doRebalance() {}
method updateTopicPublishInfo (line 64) | updateTopicPublishInfo() {}
method isPublishTopicNeedUpdate (line 65) | isPublishTopicNeedUpdate() {}
method updateTopicSubscribeInfo (line 83) | updateTopicSubscribeInfo() {}
method isSubscribeTopicNeedUpdate (line 84) | isSubscribeTopicNeedUpdate() {}
method doRebalance (line 85) | doRebalance() {}
method updateTopicPublishInfo (line 90) | updateTopicPublishInfo() {}
method isPublishTopicNeedUpdate (line 91) | isPublishTopicNeedUpdate() {}
method updateTopicSubscribeInfo (line 110) | updateTopicSubscribeInfo() {}
method isSubscribeTopicNeedUpdate (line 111) | isSubscribeTopicNeedUpdate() {}
method doRebalance (line 112) | doRebalance() {}
method updateTopicPublishInfo (line 116) | updateTopicPublishInfo() {}
method isPublishTopicNeedUpdate (line 117) | isPublishTopicNeedUpdate() {}
method updateTopicSubscribeInfo (line 206) | updateTopicSubscribeInfo() {}
method isSubscribeTopicNeedUpdate (line 207) | isSubscribeTopicNeedUpdate() {}
method doRebalance (line 208) | doRebalance() {}
method updateTopicPublishInfo (line 212) | updateTopicPublishInfo() {}
method isPublishTopicNeedUpdate (line 213) | isPublishTopicNeedUpdate() {}
method persistConsumerOffset (line 230) | async persistConsumerOffset() { console.log('persistConsumerOffset'); }
method doRebalance (line 231) | async doRebalance() { console.log('doRebalance'); }
FILE: test/mq_client_api.test.js
constant TOPIC (line 9) | const TOPIC = config.topic;
FILE: test/store/remote_broker.test.js
constant TOPIC (line 11) | const TOPIC = config.topic;
Condensed preview — 69 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (269K chars).
[
{
"path": ".eslintignore",
"chars": 73,
"preview": "test/fixtures\ntest/benchmark\nnode_modules\ncoverage\nbenchmark\n_lib\nexample"
},
{
"path": ".eslintrc",
"chars": 37,
"preview": "{\n \"extends\": \"eslint-config-egg\"\n}\n"
},
{
"path": ".github/workflows/ci-actions.yml",
"chars": 1780,
"preview": "name: ci-actions\non:\n push:\n branches: [ master ]\n pull_request:\n branches: [ master ]\n merge_group:\njobs:\n te"
},
{
"path": ".github/workflows/release.yml",
"chars": 322,
"preview": "name: Release\non:\n push:\n branches: [ master ]\n\npermissions:\n contents: write\n deployments: write\n issues: write\n"
},
{
"path": ".gitignore",
"chars": 576,
"preview": ".DS_Store\n\n# Logs\nlogs\n*.log\n\n# Runtime data\npids\n*.pid\n*.seed\n\n# Directory for instrumented libs generated by jscoverag"
},
{
"path": "AUTHORS",
"chars": 242,
"preview": "# Ordered by date of first contribution.\n# Auto-generated by 'contributors' on Fri, 29 Sep 2017 01:42:55 GMT.\n# https://"
},
{
"path": "CHANGELOG.md",
"chars": 547,
"preview": "# Changelog\n\n## <small>3.12.2 (2025-12-08)</small>\n\n* fix: make auto release work again ([5b43dda](https://github.com/al"
},
{
"path": "History.md",
"chars": 9277,
"preview": "\n3.12.0 / 2023-07-31\n==================\n\n**features**\n * [[`32d4d09`](http://github.com/ali-sdk/ali-ons/commit/32d4d096"
},
{
"path": "LICENSE",
"chars": 1070,
"preview": "The MIT License (MIT)\n\nCopyright (c) ali-sdk\n\nPermission is hereby granted, free of charge, to any person obtaining a co"
},
{
"path": "README.md",
"chars": 3721,
"preview": "# ali-ons\n\n[![NPM version][npm-image]][npm-url]\n[![ci-actions][ci-image]][ci-url]\n[![npm download][download-image]][down"
},
{
"path": "example/config.js",
"chars": 508,
"preview": "'use strict';\n\nconst env = process.env;\n\n// export ALI_SDK_ONS_ID=your-accesskey\n// export ALI_SDK_ONS_SECRET=your-secre"
},
{
"path": "example/consumer.js",
"chars": 602,
"preview": "'use strict';\n\nconst httpclient = require('urllib');\nconst logger = require('./logger');\nconst Consumer = require('../')"
},
{
"path": "example/logger.js",
"chars": 129,
"preview": "'use strict';\n\nmodule.exports = {\n info() {},\n warn() {},\n error(...args) {\n console.error(...args);\n },\n debug("
},
{
"path": "example/producer.js",
"chars": 577,
"preview": "'use strict';\n\nconst logger = require('./logger');\nconst config = require('./config');\nconst httpclient = require('urlli"
},
{
"path": "lib/channel.js",
"chars": 3490,
"preview": "'use strict';\n\nconst is = require('is-type-of');\nconst crypto = require('crypto');\nconst Base = require('tcp-base');\ncon"
},
{
"path": "lib/client_config.js",
"chars": 2633,
"preview": "'use strict';\n\nconst Base = require('sdk-base');\nconst address = require('address');\nconst MixAll = require('./mix_all')"
},
{
"path": "lib/consumer/consume_from_where.js",
"chars": 691,
"preview": "'use strict';\n\nmodule.exports = {\n /**\n * 一个新的订阅组第一次启动从队列的最后位置开始消费<br>\n * 后续再启动接着上次消费的进度开始消费\n */\n CONSUME_FROM_L"
},
{
"path": "lib/consumer/mq_push_consumer.js",
"chars": 30111,
"preview": "'use strict';\n\nconst assert = require('assert');\nconst is = require('is-type-of');\nconst utils = require('../utils');\nco"
},
{
"path": "lib/consumer/pull_status.js",
"chars": 273,
"preview": "'use strict';\n\nmodule.exports = {\n /**\n * 找到消息\n */\n FOUND: 'FOUND',\n /**\n * 没有新的消息可以被拉取\n */\n NO_NEW_MSG: 'NO"
},
{
"path": "lib/consumer/rebalance/allocate_message_queue_averagely.js",
"chars": 1510,
"preview": "'use strict';\n\nconst debug = require('debug')('mq:allocate');\nconst AllocateMessageQueueStrategy = require('./allocate_m"
},
{
"path": "lib/consumer/rebalance/allocate_message_queue_strategy.js",
"chars": 622,
"preview": "/* istanbul ignore next */\n/* eslint valid-jsdoc:0 */\n'use strict';\n\nclass AllocateMessageQueueStrategy {\n /**\n * 给当前"
},
{
"path": "lib/index.js",
"chars": 176,
"preview": "'use strict';\n\nexports.Message = require('./message/message');\nexports.Producer = require('./producer/mq_producer');\nexp"
},
{
"path": "lib/logger.js",
"chars": 166,
"preview": "'use strict';\n\nmodule.exports = {\n info() {},\n warn(...args) {\n console.warn(...args);\n },\n error(...args) {\n "
},
{
"path": "lib/message/message.js",
"chars": 3640,
"preview": "'use strict';\n\nconst is = require('is-type-of');\nconst MessageConst = require('./message_const');\n\nclass Message {\n\n /*"
},
{
"path": "lib/message/message_const.js",
"chars": 1173,
"preview": "'use strict';\n\n// 消息关键词，多个Key用KEY_SEPARATOR隔开（查询消息使用）\nexports.PROPERTY_KEYS = 'KEYS';\n// 消息标签，只支持设置一个Tag（服务端消息过滤使用）\nexpo"
},
{
"path": "lib/message/message_decoder.js",
"chars": 5543,
"preview": "'use strict';\n/* eslint no-bitwise: 0 */\n\nconst debug = require('debug')('mq:decoder');\nconst Long = require('long');\nco"
},
{
"path": "lib/message_queue.js",
"chars": 352,
"preview": "'use strict';\n\nconst fmt = require('util').format;\n\nclass MessageQueue {\n constructor(topic, brokerName, queueId) {\n "
},
{
"path": "lib/mix_all.js",
"chars": 914,
"preview": "'use strict';\n\nexports.DEFAULT_TOPIC = 'TBW102';\nexports.DEFAULT_PRODUCER_GROUP = 'DEFAULT_PRODUCER';\nexports.DEFAULT_CO"
},
{
"path": "lib/mq_client.js",
"chars": 20425,
"preview": "'use strict';\n\nconst is = require('is-type-of');\nconst gather = require('co-gather');\nconst sleep = require('mz-modules/"
},
{
"path": "lib/mq_client_api.js",
"chars": 20071,
"preview": "'use strict';\n\nconst JSON2 = require('JSON2');\nconst bytes = require('bytes');\nconst fmt = require('util').format;\nconst"
},
{
"path": "lib/process_queue.js",
"chars": 1213,
"preview": "'use strict';\n\nconst Base = require('sdk-base');\nconst bsInsert = require('binary-search-insert');\nconst comparator = (a"
},
{
"path": "lib/producer/mq_producer.js",
"chars": 11093,
"preview": "'use strict';\n/* eslint no-bitwise: 0 */\n\nconst utils = require('../utils');\nconst logger = require('../logger');\nconst "
},
{
"path": "lib/producer/send_status.js",
"chars": 370,
"preview": "'use strict';\n\nmodule.exports = {\n // 消息发送成功\n SEND_OK: 'SEND_OK',\n // 消息发送成功，但是服务器刷盘超时，消息已经进入服务器队列，只有此时服务器宕机，消息才会丢失\n "
},
{
"path": "lib/producer/topic_publish_info.js",
"chars": 2271,
"preview": "'use strict';\n\nconst crypto = require('crypto');\n\nclass TopicPublishInfo {\n constructor() {\n this.orderTopic = false"
},
{
"path": "lib/protocol/command/opaque_generator.js",
"chars": 273,
"preview": "'use strict';\n\nconst MAX_INT_31 = Math.pow(2, 31) - 10;\nlet opaque = 0;\n\nexports.getNextOpaque = () => {\n if (opaque >="
},
{
"path": "lib/protocol/command/remoting_command.js",
"chars": 3963,
"preview": "'use strict';\n/* eslint no-bitwise: 0*/\n\nconst bytes = require('bytes');\nconst is = require('is-type-of');\nconst ByteBuf"
},
{
"path": "lib/protocol/consume_type.js",
"chars": 119,
"preview": "'use strict';\n\nmodule.exports = {\n CONSUME_ACTIVELY: 'CONSUME_ACTIVELY',\n CONSUME_PASSIVELY: 'CONSUME_PASSIVELY',\n};\n"
},
{
"path": "lib/protocol/message_model.js",
"chars": 97,
"preview": "'use strict';\n\nmodule.exports = {\n BROADCASTING: 'BROADCASTING',\n CLUSTERING: 'CLUSTERING',\n};\n"
},
{
"path": "lib/protocol/perm_name.js",
"chars": 744,
"preview": "'use strict';\n/* eslint no-bitwise: 0 */\n\n// var PERM_PRIORITY = 0x1 << 3;\nconst PERM_READ = 0x1 << 2;\nconst PERM_WRITE "
},
{
"path": "lib/protocol/request_code.js",
"chars": 4988,
"preview": "'use strict';\n\n// Broker 发送消息\nexports.SEND_MESSAGE = 10;\n// Broker 订阅消息\nexports.PULL_MESSAGE = 11;\n// Broker 查询消息\nexport"
},
{
"path": "lib/protocol/response_code.js",
"chars": 1652,
"preview": "'use strict';\n\n// 成功\nexports.SUCCESS = 0;\n// 发生了未捕获异常\nexports.SYSTEM_ERROR = 1;\n// 由于线程池拥堵，系统繁忙\nexports.SYSTEM_BUSY = 2;"
},
{
"path": "lib/remoting_client.js",
"chars": 6739,
"preview": "'use strict';\n\nconst assert = require('assert');\nconst Base = require('sdk-base');\nconst logger = require('./logger');\nc"
},
{
"path": "lib/store/index.js",
"chars": 193,
"preview": "'use strict';\n\nexports.LocalFileOffsetStore = require('./local_file');\nexports.LocalMemoryOffsetStore = require('./local"
},
{
"path": "lib/store/local_file.js",
"chars": 4237,
"preview": "'use strict';\n\nconst fs = require('fs');\nconst path = require('path');\nconst osenv = require('osenv');\nconst mkdirp = re"
},
{
"path": "lib/store/local_memory.js",
"chars": 2263,
"preview": "'use strict';\n\nconst is = require('is-type-of');\nconst Base = require('sdk-base');\nconst ReadOffsetType = require('./rea"
},
{
"path": "lib/store/read_offset_type.js",
"chars": 229,
"preview": "'use strict';\n\nmodule.exports = {\n // 只从Memory读取\n READ_FROM_MEMORY: 'READ_FROM_MEMORY',\n // 只从存储层读取（本地或者远端）\n READ_FR"
},
{
"path": "lib/store/remote_broker.js",
"chars": 5964,
"preview": "'use strict';\n\nconst is = require('is-type-of');\nconst Base = require('sdk-base');\nconst gather = require('co-gather');\n"
},
{
"path": "lib/utils/index.js",
"chars": 1585,
"preview": "'use strict';\n\n/* eslint no-bitwise:0 */\n\nconst zlib = require('zlib');\nconst MixAll = require('../mix_all');\nconst is ="
},
{
"path": "lib/utils/message_sys_flag.js",
"chars": 628,
"preview": "'use strict';\n/* eslint no-bitwise: 0 */\n\nexports.CompressedFlag = 1 << 0;\nexports.MultiTagsFlag = 1 << 1;\n\n/**\n * 7 6 5"
},
{
"path": "lib/utils/pull_sys_flag.js",
"chars": 1060,
"preview": "'use strict';\n/* eslint no-bitwise: 0 */\n\nconst FLAG_COMMIT_OFFSET = 1 << 0;\nconst FLAG_SUSPEND = 1 << 1;\nconst FLAG_SUB"
},
{
"path": "package.json",
"chars": 1454,
"preview": "{\n \"name\": \"ali-ons\",\n \"version\": \"3.12.2\",\n \"description\": \"Aliyun Open Notification Service Client\",\n \"main\": \"lib"
},
{
"path": "test/allocate_message_queue_averagely.test.js",
"chars": 4869,
"preview": "'use strict';\n\nconst assert = require('assert');\nconst AllocateMessageQueueAveragely = require('../lib/consumer/rebalanc"
},
{
"path": "test/channel.test.js",
"chars": 7001,
"preview": "'use strict';\n\nconst mm = require('mm');\nconst net = require('net');\nconst assert = require('assert');\nconst urllib = re"
},
{
"path": "test/consumer/rebalance/allocate_message_queue_averagely.test.js",
"chars": 4873,
"preview": "'use strict';\n\nconst assert = require('assert');\nconst AllocateMessageQueueAveragely = require('../../../lib/consumer/re"
},
{
"path": "test/index.test.js",
"chars": 20498,
"preview": "'use strict';\n\nconst mm = require('mm');\nconst path = require('path');\nconst osenv = require('osenv');\nconst assert = re"
},
{
"path": "test/index_namesrv.test.js",
"chars": 15790,
"preview": "'use strict';\n\nconst mm = require('mm');\nconst path = require('path');\nconst osenv = require('osenv');\nconst assert = re"
},
{
"path": "test/message/message_decoder.test.js",
"chars": 4076,
"preview": "'use strict';\n\nconst assert = require('assert');\nconst ByteBuffer = require('byte');\nconst utils = require('../utils');\n"
},
{
"path": "test/mq_client.test.js",
"chars": 8023,
"preview": "'use strict';\n\nconst assert = require('assert');\nconst httpclient = require('urllib');\nconst ClientConfig = require('../"
},
{
"path": "test/mq_client_api.test.js",
"chars": 1964,
"preview": "'use strict';\n\nconst assert = require('assert');\nconst address = require('address');\nconst httpclient = require('urllib'"
},
{
"path": "test/protocol/command/opaque_generator.test.js",
"chars": 410,
"preview": "'use strict';\n\nconst assert = require('assert');\nconst OpaqueGenerator = require('../../../lib/protocol/command/opaque_g"
},
{
"path": "test/protocol/command/remoting_command.test.js",
"chars": 3256,
"preview": "'use strict';\n\nconst assert = require('assert');\nconst RequestCode = require('../../../lib/protocol/request_code');\ncons"
},
{
"path": "test/remoting_client.test.js",
"chars": 5235,
"preview": "'use strict';\n\nconst mm = require('mm');\nconst assert = require('assert');\nconst httpclient = require('urllib');\nconst c"
},
{
"path": "test/store/local_file.test.js",
"chars": 2674,
"preview": "'use strict';\n\nconst assert = require('assert');\nconst httpclient = require('urllib');\nconst MQClient = require('../../l"
},
{
"path": "test/store/local_memory.test.js",
"chars": 3210,
"preview": "'use strict';\n\nconst mm = require('mm');\nconst assert = require('assert');\nconst MessageQueue = require('../../lib/messa"
},
{
"path": "test/store/remote_broker.test.js",
"chars": 2841,
"preview": "'use strict';\n\nconst assert = require('assert');\nconst httpclient = require('urllib');\nconst MQClient = require('../../l"
},
{
"path": "test/utils/index.test.js",
"chars": 493,
"preview": "'use strict';\n\nconst assert = require('assert');\nconst util = require('../../lib/utils/index');\n\ndescribe('test/utils/in"
},
{
"path": "test/utils/message_sys_flag.test.js",
"chars": 1079,
"preview": "'use strict';\n\nconst assert = require('assert');\nconst MessageSysFlag = require('../../lib/utils/message_sys_flag');\n\nde"
},
{
"path": "test/utils/pull_sys_flag.test.js",
"chars": 1489,
"preview": "'use strict';\n\nconst assert = require('assert');\nconst PullSysFlag = require('../../lib/utils/pull_sys_flag');\n\ndescribe"
},
{
"path": "test/utils.js",
"chars": 294,
"preview": "'use strict';\n\nconst fs = require('fs');\nconst path = require('path');\nconst fixtures = path.join(__dirname, 'fixtures')"
}
]
About this extraction
This page contains the full source code of the ali-sdk/ali-ons GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 69 files (244.6 KB), approximately 67.9k tokens, and a symbol index with 309 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.